Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/black/comments.py: 44%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 06:15 +0000

1import re 

2import sys 

3from dataclasses import dataclass 

4from functools import lru_cache 

5from typing import Iterator, List, Optional, Union 

6 

7if sys.version_info >= (3, 8): 

8 from typing import Final 

9else: 

10 from typing_extensions import Final 

11 

12from black.nodes import ( 

13 CLOSING_BRACKETS, 

14 STANDALONE_COMMENT, 

15 WHITESPACE, 

16 container_of, 

17 first_leaf_of, 

18 preceding_leaf, 

19 syms, 

20) 

21from blib2to3.pgen2 import token 

22from blib2to3.pytree import Leaf, Node 

23 

24# types 

25LN = Union[Leaf, Node] 

26 

27FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"} 

28FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"} 

29FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP} 

30FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"} 

31 

32COMMENT_EXCEPTIONS = " !:#'" 

33 

34 

35@dataclass 

36class ProtoComment: 

37 """Describes a piece of syntax that is a comment. 

38 

39 It's not a :class:`blib2to3.pytree.Leaf` so that: 

40 

41 * it can be cached (`Leaf` objects should not be reused more than once as 

42 they store their lineno, column, prefix, and parent information); 

43 * `newlines` and `consumed` fields are kept separate from the `value`. This 

44 simplifies handling of special marker comments like ``# fmt: off/on``. 

45 """ 

46 

47 type: int # token.COMMENT or STANDALONE_COMMENT 

48 value: str # content of the comment 

49 newlines: int # how many newlines before the comment 

50 consumed: int # how many characters of the original leaf's prefix did we consume 

51 

52 

53def generate_comments(leaf: LN) -> Iterator[Leaf]: 

54 """Clean the prefix of the `leaf` and generate comments from it, if any. 

55 

56 Comments in lib2to3 are shoved into the whitespace prefix. This happens 

57 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation 

58 move because it does away with modifying the grammar to include all the 

59 possible places in which comments can be placed. 

60 

61 The sad consequence for us though is that comments don't "belong" anywhere. 

62 This is why this function generates simple parentless Leaf objects for 

63 comments. We simply don't know what the correct parent should be. 

64 

65 No matter though, we can live without this. We really only need to 

66 differentiate between inline and standalone comments. The latter don't 

67 share the line with any code. 

68 

69 Inline comments are emitted as regular token.COMMENT leaves. Standalone 

70 are emitted with a fake STANDALONE_COMMENT token identifier. 

71 """ 

72 for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): 

73 yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) 

74 

75 

76@lru_cache(maxsize=4096) 

77def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: 

78 """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" 

79 result: List[ProtoComment] = [] 

80 if not prefix or "#" not in prefix: 

81 return result 

82 

83 consumed = 0 

84 nlines = 0 

85 ignored_lines = 0 

86 for index, line in enumerate(re.split("\r?\n", prefix)): 

87 consumed += len(line) + 1 # adding the length of the split '\n' 

88 line = line.lstrip() 

89 if not line: 

90 nlines += 1 

91 if not line.startswith("#"): 

92 # Escaped newlines outside of a comment are not really newlines at 

93 # all. We treat a single-line comment following an escaped newline 

94 # as a simple trailing comment. 

95 if line.endswith("\\"): 

96 ignored_lines += 1 

97 continue 

98 

99 if index == ignored_lines and not is_endmarker: 

100 comment_type = token.COMMENT # simple trailing comment 

101 else: 

102 comment_type = STANDALONE_COMMENT 

103 comment = make_comment(line) 

104 result.append( 

105 ProtoComment( 

106 type=comment_type, value=comment, newlines=nlines, consumed=consumed 

107 ) 

108 ) 

109 nlines = 0 

110 return result 

111 

112 

113def make_comment(content: str) -> str: 

114 """Return a consistently formatted comment from the given `content` string. 

115 

116 All comments (except for "##", "#!", "#:", '#'") should have a single 

117 space between the hash sign and the content. 

118 

119 If `content` didn't start with a hash sign, one is provided. 

120 """ 

121 content = content.rstrip() 

122 if not content: 

123 return "#" 

124 

125 if content[0] == "#": 

126 content = content[1:] 

127 NON_BREAKING_SPACE = " " 

128 if ( 

129 content 

130 and content[0] == NON_BREAKING_SPACE 

131 and not content.lstrip().startswith("type:") 

132 ): 

133 content = " " + content[1:] # Replace NBSP by a simple space 

134 if content and content[0] not in COMMENT_EXCEPTIONS: 

135 content = " " + content 

136 return "#" + content 

137 

138 

139def normalize_fmt_off(node: Node) -> None: 

140 """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" 

141 try_again = True 

142 while try_again: 

143 try_again = convert_one_fmt_off_pair(node) 

144 

145 

146def convert_one_fmt_off_pair(node: Node) -> bool: 

147 """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. 

148 

149 Returns True if a pair was converted. 

150 """ 

151 for leaf in node.leaves(): 

152 previous_consumed = 0 

153 for comment in list_comments(leaf.prefix, is_endmarker=False): 

154 if comment.value not in FMT_PASS: 

155 previous_consumed = comment.consumed 

156 continue 

157 # We only want standalone comments. If there's no previous leaf or 

158 # the previous leaf is indentation, it's a standalone comment in 

159 # disguise. 

160 if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: 

161 prev = preceding_leaf(leaf) 

162 if prev: 

163 if comment.value in FMT_OFF and prev.type not in WHITESPACE: 

164 continue 

165 if comment.value in FMT_SKIP and prev.type in WHITESPACE: 

166 continue 

167 

168 ignored_nodes = list(generate_ignored_nodes(leaf, comment)) 

169 if not ignored_nodes: 

170 continue 

171 

172 first = ignored_nodes[0] # Can be a container node with the `leaf`. 

173 parent = first.parent 

174 prefix = first.prefix 

175 if comment.value in FMT_OFF: 

176 first.prefix = prefix[comment.consumed :] 

177 if comment.value in FMT_SKIP: 

178 first.prefix = "" 

179 standalone_comment_prefix = prefix 

180 else: 

181 standalone_comment_prefix = ( 

182 prefix[:previous_consumed] + "\n" * comment.newlines 

183 ) 

184 hidden_value = "".join(str(n) for n in ignored_nodes) 

185 if comment.value in FMT_OFF: 

186 hidden_value = comment.value + "\n" + hidden_value 

187 if comment.value in FMT_SKIP: 

188 hidden_value += " " + comment.value 

189 if hidden_value.endswith("\n"): 

190 # That happens when one of the `ignored_nodes` ended with a NEWLINE 

191 # leaf (possibly followed by a DEDENT). 

192 hidden_value = hidden_value[:-1] 

193 first_idx: Optional[int] = None 

194 for ignored in ignored_nodes: 

195 index = ignored.remove() 

196 if first_idx is None: 

197 first_idx = index 

198 assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" 

199 assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" 

200 parent.insert_child( 

201 first_idx, 

202 Leaf( 

203 STANDALONE_COMMENT, 

204 hidden_value, 

205 prefix=standalone_comment_prefix, 

206 fmt_pass_converted_first_leaf=first_leaf_of(first), 

207 ), 

208 ) 

209 return True 

210 

211 return False 

212 

213 

214def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: 

215 """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. 

216 

217 If comment is skip, returns leaf only. 

218 Stops at the end of the block. 

219 """ 

220 if comment.value in FMT_SKIP: 

221 yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) 

222 return 

223 container: Optional[LN] = container_of(leaf) 

224 while container is not None and container.type != token.ENDMARKER: 

225 if is_fmt_on(container): 

226 return 

227 

228 # fix for fmt: on in children 

229 if children_contains_fmt_on(container): 

230 for index, child in enumerate(container.children): 

231 if isinstance(child, Leaf) and is_fmt_on(child): 

232 if child.type in CLOSING_BRACKETS: 

233 # This means `# fmt: on` is placed at a different bracket level 

234 # than `# fmt: off`. This is an invalid use, but as a courtesy, 

235 # we include this closing bracket in the ignored nodes. 

236 # The alternative is to fail the formatting. 

237 yield child 

238 return 

239 if ( 

240 child.type == token.INDENT 

241 and index < len(container.children) - 1 

242 and children_contains_fmt_on(container.children[index + 1]) 

243 ): 

244 # This means `# fmt: on` is placed right after an indentation 

245 # level, and we shouldn't swallow the previous INDENT token. 

246 return 

247 if children_contains_fmt_on(child): 

248 return 

249 yield child 

250 else: 

251 if container.type == token.DEDENT and container.next_sibling is None: 

252 # This can happen when there is no matching `# fmt: on` comment at the 

253 # same level as `# fmt: on`. We need to keep this DEDENT. 

254 return 

255 yield container 

256 container = container.next_sibling 

257 

258 

259def _generate_ignored_nodes_from_fmt_skip( 

260 leaf: Leaf, comment: ProtoComment 

261) -> Iterator[LN]: 

262 """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.""" 

263 prev_sibling = leaf.prev_sibling 

264 parent = leaf.parent 

265 # Need to properly format the leaf prefix to compare it to comment.value, 

266 # which is also formatted 

267 comments = list_comments(leaf.prefix, is_endmarker=False) 

268 if not comments or comment.value != comments[0].value: 

269 return 

270 if prev_sibling is not None: 

271 leaf.prefix = "" 

272 siblings = [prev_sibling] 

273 while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None: 

274 prev_sibling = prev_sibling.prev_sibling 

275 siblings.insert(0, prev_sibling) 

276 yield from siblings 

277 elif ( 

278 parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE 

279 ): 

280 # The `# fmt: skip` is on the colon line of the if/while/def/class/... 

281 # statements. The ignored nodes should be previous siblings of the 

282 # parent suite node. 

283 leaf.prefix = "" 

284 ignored_nodes: List[LN] = [] 

285 parent_sibling = parent.prev_sibling 

286 while parent_sibling is not None and parent_sibling.type != syms.suite: 

287 ignored_nodes.insert(0, parent_sibling) 

288 parent_sibling = parent_sibling.prev_sibling 

289 # Special case for `async_stmt` where the ASYNC token is on the 

290 # grandparent node. 

291 grandparent = parent.parent 

292 if ( 

293 grandparent is not None 

294 and grandparent.prev_sibling is not None 

295 and grandparent.prev_sibling.type == token.ASYNC 

296 ): 

297 ignored_nodes.insert(0, grandparent.prev_sibling) 

298 yield from iter(ignored_nodes) 

299 

300 

301def is_fmt_on(container: LN) -> bool: 

302 """Determine whether formatting is switched on within a container. 

303 Determined by whether the last `# fmt:` comment is `on` or `off`. 

304 """ 

305 fmt_on = False 

306 for comment in list_comments(container.prefix, is_endmarker=False): 

307 if comment.value in FMT_ON: 

308 fmt_on = True 

309 elif comment.value in FMT_OFF: 

310 fmt_on = False 

311 return fmt_on 

312 

313 

314def children_contains_fmt_on(container: LN) -> bool: 

315 """Determine if children have formatting switched on.""" 

316 for child in container.children: 

317 leaf = first_leaf_of(child) 

318 if leaf is not None and is_fmt_on(leaf): 

319 return True 

320 

321 return False 

322 

323 

324def contains_pragma_comment(comment_list: List[Leaf]) -> bool: 

325 """ 

326 Returns: 

327 True iff one of the comments in @comment_list is a pragma used by one 

328 of the more common static analysis tools for python (e.g. mypy, flake8, 

329 pylint). 

330 """ 

331 for comment in comment_list: 

332 if comment.value.startswith(("# type:", "# noqa", "# pylint:")): 

333 return True 

334 

335 return False