Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/comments.py: 17%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import re
2from collections.abc import Collection, Iterator
3from dataclasses import dataclass
4from functools import lru_cache
5from typing import Final, Optional, Union
7from black.mode import Mode, Preview
8from black.nodes import (
9 CLOSING_BRACKETS,
10 STANDALONE_COMMENT,
11 WHITESPACE,
12 container_of,
13 first_leaf_of,
14 make_simple_prefix,
15 preceding_leaf,
16 syms,
17)
18from blib2to3.pgen2 import token
19from blib2to3.pytree import Leaf, Node
21# types
22LN = Union[Leaf, Node]
24FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
25FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
26FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}
28COMMENT_EXCEPTIONS = " !:#'"
29_COMMENT_PREFIX = "# "
30_COMMENT_LIST_SEPARATOR = ";"
33@dataclass
34class ProtoComment:
35 """Describes a piece of syntax that is a comment.
37 It's not a :class:`blib2to3.pytree.Leaf` so that:
39 * it can be cached (`Leaf` objects should not be reused more than once as
40 they store their lineno, column, prefix, and parent information);
41 * `newlines` and `consumed` fields are kept separate from the `value`. This
42 simplifies handling of special marker comments like ``# fmt: off/on``.
43 """
45 type: int # token.COMMENT or STANDALONE_COMMENT
46 value: str # content of the comment
47 newlines: int # how many newlines before the comment
48 consumed: int # how many characters of the original leaf's prefix did we consume
49 form_feed: bool # is there a form feed before the comment
50 leading_whitespace: str # leading whitespace before the comment, if any
53def generate_comments(leaf: LN) -> Iterator[Leaf]:
54 """Clean the prefix of the `leaf` and generate comments from it, if any.
56 Comments in lib2to3 are shoved into the whitespace prefix. This happens
57 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
58 move because it does away with modifying the grammar to include all the
59 possible places in which comments can be placed.
61 The sad consequence for us though is that comments don't "belong" anywhere.
62 This is why this function generates simple parentless Leaf objects for
63 comments. We simply don't know what the correct parent should be.
65 No matter though, we can live without this. We really only need to
66 differentiate between inline and standalone comments. The latter don't
67 share the line with any code.
69 Inline comments are emitted as regular token.COMMENT leaves. Standalone
70 are emitted with a fake STANDALONE_COMMENT token identifier.
71 """
72 total_consumed = 0
73 for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):
74 total_consumed = pc.consumed
75 prefix = make_simple_prefix(pc.newlines, pc.form_feed)
76 yield Leaf(pc.type, pc.value, prefix=prefix)
77 normalize_trailing_prefix(leaf, total_consumed)
80@lru_cache(maxsize=4096)
81def list_comments(prefix: str, *, is_endmarker: bool) -> list[ProtoComment]:
82 """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
83 result: list[ProtoComment] = []
84 if not prefix or "#" not in prefix:
85 return result
87 consumed = 0
88 nlines = 0
89 ignored_lines = 0
90 form_feed = False
91 for index, full_line in enumerate(re.split("\r?\n|\r", prefix)):
92 consumed += len(full_line) + 1 # adding the length of the split '\n'
93 match = re.match(r"^(\s*)(\S.*|)$", full_line)
94 assert match
95 whitespace, line = match.groups()
96 if not line:
97 nlines += 1
98 if "\f" in full_line:
99 form_feed = True
100 if not line.startswith("#"):
101 # Escaped newlines outside of a comment are not really newlines at
102 # all. We treat a single-line comment following an escaped newline
103 # as a simple trailing comment.
104 if line.endswith("\\"):
105 ignored_lines += 1
106 continue
108 if index == ignored_lines and not is_endmarker:
109 comment_type = token.COMMENT # simple trailing comment
110 else:
111 comment_type = STANDALONE_COMMENT
112 comment = make_comment(line)
113 result.append(
114 ProtoComment(
115 type=comment_type,
116 value=comment,
117 newlines=nlines,
118 consumed=consumed,
119 form_feed=form_feed,
120 leading_whitespace=whitespace,
121 )
122 )
123 form_feed = False
124 nlines = 0
125 return result
128def normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None:
129 """Normalize the prefix that's left over after generating comments.
131 Note: don't use backslashes for formatting or you'll lose your voting rights.
132 """
133 remainder = leaf.prefix[total_consumed:]
134 if "\\" not in remainder:
135 nl_count = remainder.count("\n")
136 form_feed = "\f" in remainder and remainder.endswith("\n")
137 leaf.prefix = make_simple_prefix(nl_count, form_feed)
138 return
140 leaf.prefix = ""
143def make_comment(content: str) -> str:
144 """Return a consistently formatted comment from the given `content` string.
146 All comments (except for "##", "#!", "#:", '#'") should have a single
147 space between the hash sign and the content.
149 If `content` didn't start with a hash sign, one is provided.
150 """
151 content = content.rstrip()
152 if not content:
153 return "#"
155 if content[0] == "#":
156 content = content[1:]
157 if (
158 content
159 and content[0] == "\N{NO-BREAK SPACE}"
160 and not content.lstrip().startswith("type:")
161 ):
162 content = " " + content[1:] # Replace NBSP by a simple space
163 if content and content[0] not in COMMENT_EXCEPTIONS:
164 content = " " + content
165 return "#" + content
168def normalize_fmt_off(
169 node: Node, mode: Mode, lines: Collection[tuple[int, int]]
170) -> None:
171 """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
172 try_again = True
173 while try_again:
174 try_again = convert_one_fmt_off_pair(node, mode, lines)
177def convert_one_fmt_off_pair(
178 node: Node, mode: Mode, lines: Collection[tuple[int, int]]
179) -> bool:
180 """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
182 Returns True if a pair was converted.
183 """
184 for leaf in node.leaves():
185 previous_consumed = 0
186 for comment in list_comments(leaf.prefix, is_endmarker=False):
187 is_fmt_off = comment.value in FMT_OFF
188 is_fmt_skip = _contains_fmt_skip_comment(comment.value, mode)
189 if (not is_fmt_off and not is_fmt_skip) or (
190 # Invalid use when `# fmt: off` is applied before a closing bracket.
191 is_fmt_off
192 and leaf.type in CLOSING_BRACKETS
193 ):
194 previous_consumed = comment.consumed
195 continue
196 # We only want standalone comments. If there's no previous leaf or
197 # the previous leaf is indentation, it's a standalone comment in
198 # disguise.
199 if comment.type != STANDALONE_COMMENT:
200 prev = preceding_leaf(leaf)
201 if prev:
202 if is_fmt_off and prev.type not in WHITESPACE:
203 continue
204 if is_fmt_skip and prev.type in WHITESPACE:
205 continue
207 ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode))
208 if not ignored_nodes:
209 continue
211 first = ignored_nodes[0] # Can be a container node with the `leaf`.
212 parent = first.parent
213 prefix = first.prefix
214 if comment.value in FMT_OFF:
215 first.prefix = prefix[comment.consumed :]
216 if is_fmt_skip:
217 first.prefix = ""
218 standalone_comment_prefix = prefix
219 else:
220 standalone_comment_prefix = (
221 prefix[:previous_consumed] + "\n" * comment.newlines
222 )
223 hidden_value = "".join(str(n) for n in ignored_nodes)
224 comment_lineno = leaf.lineno - comment.newlines
225 if comment.value in FMT_OFF:
226 fmt_off_prefix = ""
227 if len(lines) > 0 and not any(
228 line[0] <= comment_lineno <= line[1] for line in lines
229 ):
230 # keeping indentation of comment by preserving original whitespaces.
231 fmt_off_prefix = prefix.split(comment.value)[0]
232 if "\n" in fmt_off_prefix:
233 fmt_off_prefix = fmt_off_prefix.split("\n")[-1]
234 standalone_comment_prefix += fmt_off_prefix
235 hidden_value = comment.value + "\n" + hidden_value
236 if is_fmt_skip:
237 hidden_value += comment.leading_whitespace + comment.value
238 if hidden_value.endswith("\n"):
239 # That happens when one of the `ignored_nodes` ended with a NEWLINE
240 # leaf (possibly followed by a DEDENT).
241 hidden_value = hidden_value[:-1]
242 first_idx: Optional[int] = None
243 for ignored in ignored_nodes:
244 index = ignored.remove()
245 if first_idx is None:
246 first_idx = index
247 assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
248 assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
249 parent.insert_child(
250 first_idx,
251 Leaf(
252 STANDALONE_COMMENT,
253 hidden_value,
254 prefix=standalone_comment_prefix,
255 fmt_pass_converted_first_leaf=first_leaf_of(first),
256 ),
257 )
258 return True
260 return False
263def generate_ignored_nodes(
264 leaf: Leaf, comment: ProtoComment, mode: Mode
265) -> Iterator[LN]:
266 """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
268 If comment is skip, returns leaf only.
269 Stops at the end of the block.
270 """
271 if _contains_fmt_skip_comment(comment.value, mode):
272 yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, mode)
273 return
274 container: Optional[LN] = container_of(leaf)
275 while container is not None and container.type != token.ENDMARKER:
276 if is_fmt_on(container):
277 return
279 # fix for fmt: on in children
280 if children_contains_fmt_on(container):
281 for index, child in enumerate(container.children):
282 if isinstance(child, Leaf) and is_fmt_on(child):
283 if child.type in CLOSING_BRACKETS:
284 # This means `# fmt: on` is placed at a different bracket level
285 # than `# fmt: off`. This is an invalid use, but as a courtesy,
286 # we include this closing bracket in the ignored nodes.
287 # The alternative is to fail the formatting.
288 yield child
289 return
290 if (
291 child.type == token.INDENT
292 and index < len(container.children) - 1
293 and children_contains_fmt_on(container.children[index + 1])
294 ):
295 # This means `# fmt: on` is placed right after an indentation
296 # level, and we shouldn't swallow the previous INDENT token.
297 return
298 if children_contains_fmt_on(child):
299 return
300 yield child
301 else:
302 if container.type == token.DEDENT and container.next_sibling is None:
303 # This can happen when there is no matching `# fmt: on` comment at the
304 # same level as `# fmt: on`. We need to keep this DEDENT.
305 return
306 yield container
307 container = container.next_sibling
310def _generate_ignored_nodes_from_fmt_skip(
311 leaf: Leaf, comment: ProtoComment, mode: Mode
312) -> Iterator[LN]:
313 """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`."""
314 prev_sibling = leaf.prev_sibling
315 parent = leaf.parent
316 ignored_nodes: list[LN] = []
317 # Need to properly format the leaf prefix to compare it to comment.value,
318 # which is also formatted
319 comments = list_comments(leaf.prefix, is_endmarker=False)
320 if not comments or comment.value != comments[0].value:
321 return
322 if prev_sibling is not None:
323 leaf.prefix = leaf.prefix[comment.consumed :]
325 if Preview.fix_fmt_skip_in_one_liners not in mode:
326 siblings = [prev_sibling]
327 while (
328 "\n" not in prev_sibling.prefix
329 and prev_sibling.prev_sibling is not None
330 ):
331 prev_sibling = prev_sibling.prev_sibling
332 siblings.insert(0, prev_sibling)
333 yield from siblings
334 return
336 # Generates the nodes to be ignored by `fmt: skip`.
338 # Nodes to ignore are the ones on the same line as the
339 # `# fmt: skip` comment, excluding the `# fmt: skip`
340 # node itself.
342 # Traversal process (starting at the `# fmt: skip` node):
343 # 1. Move to the `prev_sibling` of the current node.
344 # 2. If `prev_sibling` has children, go to its rightmost leaf.
345 # 3. If there's no `prev_sibling`, move up to the parent
346 # node and repeat.
347 # 4. Continue until:
348 # a. You encounter an `INDENT` or `NEWLINE` node (indicates
349 # start of the line).
350 # b. You reach the root node.
352 # Include all visited LEAVES in the ignored list, except INDENT
353 # or NEWLINE leaves.
355 current_node = prev_sibling
356 ignored_nodes = [current_node]
357 if current_node.prev_sibling is None and current_node.parent is not None:
358 current_node = current_node.parent
359 while "\n" not in current_node.prefix and current_node.prev_sibling is not None:
360 leaf_nodes = list(current_node.prev_sibling.leaves())
361 current_node = leaf_nodes[-1] if leaf_nodes else current_node
363 if current_node.type in (token.NEWLINE, token.INDENT):
364 current_node.prefix = ""
365 break
367 ignored_nodes.insert(0, current_node)
369 if current_node.prev_sibling is None and current_node.parent is not None:
370 current_node = current_node.parent
371 yield from ignored_nodes
372 elif (
373 parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE
374 ):
375 # The `# fmt: skip` is on the colon line of the if/while/def/class/...
376 # statements. The ignored nodes should be previous siblings of the
377 # parent suite node.
378 leaf.prefix = ""
379 parent_sibling = parent.prev_sibling
380 while parent_sibling is not None and parent_sibling.type != syms.suite:
381 ignored_nodes.insert(0, parent_sibling)
382 parent_sibling = parent_sibling.prev_sibling
383 # Special case for `async_stmt` where the ASYNC token is on the
384 # grandparent node.
385 grandparent = parent.parent
386 if (
387 grandparent is not None
388 and grandparent.prev_sibling is not None
389 and grandparent.prev_sibling.type == token.ASYNC
390 ):
391 ignored_nodes.insert(0, grandparent.prev_sibling)
392 yield from iter(ignored_nodes)
395def is_fmt_on(container: LN) -> bool:
396 """Determine whether formatting is switched on within a container.
397 Determined by whether the last `# fmt:` comment is `on` or `off`.
398 """
399 fmt_on = False
400 for comment in list_comments(container.prefix, is_endmarker=False):
401 if comment.value in FMT_ON:
402 fmt_on = True
403 elif comment.value in FMT_OFF:
404 fmt_on = False
405 return fmt_on
408def children_contains_fmt_on(container: LN) -> bool:
409 """Determine if children have formatting switched on."""
410 for child in container.children:
411 leaf = first_leaf_of(child)
412 if leaf is not None and is_fmt_on(leaf):
413 return True
415 return False
418def contains_pragma_comment(comment_list: list[Leaf]) -> bool:
419 """
420 Returns:
421 True iff one of the comments in @comment_list is a pragma used by one
422 of the more common static analysis tools for python (e.g. mypy, flake8,
423 pylint).
424 """
425 for comment in comment_list:
426 if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
427 return True
429 return False
432def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool:
433 """
434 Checks if the given comment contains FMT_SKIP alone or paired with other comments.
435 Matching styles:
436 # fmt:skip <-- single comment
437 # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview)
438 # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview)
439 """
440 semantic_comment_blocks = [
441 comment_line,
442 *[
443 _COMMENT_PREFIX + comment.strip()
444 for comment in comment_line.split(_COMMENT_PREFIX)[1:]
445 ],
446 *[
447 _COMMENT_PREFIX + comment.strip()
448 for comment in comment_line.strip(_COMMENT_PREFIX).split(
449 _COMMENT_LIST_SEPARATOR
450 )
451 ],
452 ]
454 return any(comment in FMT_SKIP for comment in semantic_comment_blocks)