Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/comments.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import re
2from collections.abc import Collection, Iterator
3from dataclasses import dataclass
4from functools import lru_cache
5from typing import Final, Union
7from black.mode import Mode
8from black.nodes import (
9 CLOSING_BRACKETS,
10 OPENING_BRACKETS,
11 STANDALONE_COMMENT,
12 STATEMENT,
13 WHITESPACE,
14 container_of,
15 first_leaf_of,
16 is_type_comment_string,
17 last_leaf,
18 make_simple_prefix,
19 preceding_leaf,
20 syms,
21)
22from blib2to3.pgen2 import token
23from blib2to3.pytree import Leaf, Node
25# types
26LN = Union[Leaf, Node]
28FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
29FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
30FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}
32# Compound statements we care about for fmt: skip handling
33# (excludes except_clause and case_block which aren't standalone compound statements)
34_COMPOUND_STATEMENTS: Final = STATEMENT - {syms.except_clause, syms.case_block}
36COMMENT_EXCEPTIONS = " !:#'"
37_COMMENT_PREFIX = "# "
38_COMMENT_LIST_SEPARATOR = ";"
41@dataclass
42class ProtoComment:
43 """Describes a piece of syntax that is a comment.
45 It's not a :class:`blib2to3.pytree.Leaf` so that:
47 * it can be cached (`Leaf` objects should not be reused more than once as
48 they store their lineno, column, prefix, and parent information);
49 * `newlines` and `consumed` fields are kept separate from the `value`. This
50 simplifies handling of special marker comments like ``# fmt: off/on``.
51 """
53 type: int # token.COMMENT or STANDALONE_COMMENT
54 value: str # content of the comment
55 newlines: int # how many newlines before the comment
56 consumed: int # how many characters of the original leaf's prefix did we consume
57 form_feed: bool # is there a form feed before the comment
58 original_line: str # Includes source indentation and trailing whitespace
59 lines_before_leaf: int # Physical line distance from this comment to its leaf
62def generate_comments(
63 leaf: Leaf, mode: Mode, *, block_depth: int | None = None
64) -> Iterator[Leaf]:
65 """Clean the prefix of the `leaf` and generate comments from it, if any.
67 Comments in lib2to3 are shoved into the whitespace prefix. This happens
68 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
69 move because it does away with modifying the grammar to include all the
70 possible places in which comments can be placed.
72 The sad consequence for us though is that comments don't "belong" anywhere.
73 This is why this function generates simple parentless Leaf objects for
74 comments. We simply don't know what the correct parent should be.
76 No matter though, we can live without this. We really only need to
77 differentiate between inline and standalone comments. The latter don't
78 share the line with any code.
80 Inline comments are emitted as regular token.COMMENT leaves. Standalone
81 are emitted with a fake STANDALONE_COMMENT token identifier.
83 Pass the block depth to keep comments outside the selected lines as written,
84 accounting for the indentation the line renderer will add.
85 """
86 comments = list_comments(
87 leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER, mode=mode
88 )
89 selected = leaf.line_ranges_selected
90 for pc in comments:
91 prefix = make_simple_prefix(pc.newlines, pc.form_feed)
92 if (
93 block_depth is not None
94 and selected is not None
95 and leaf.line_ranges_first_lineno - pc.lines_before_leaf not in selected
96 ):
97 yield Leaf(
98 pc.type,
99 pc.original_line.removeprefix(" " * block_depth),
100 prefix=prefix,
101 line_ranges_selected=selected,
102 )
103 else:
104 yield Leaf(pc.type, pc.value, prefix=prefix)
105 normalize_trailing_prefix(leaf, comments[-1].consumed if comments else 0)
108@lru_cache(maxsize=4096)
109def list_comments(prefix: str, *, is_endmarker: bool, mode: Mode) -> list[ProtoComment]:
110 """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
111 result: list[ProtoComment] = []
112 if not prefix or "#" not in prefix:
113 return result
115 consumed = 0
116 nlines = 0
117 ignored_lines = 0
118 form_feed = False
119 lines = re.split("\r?\n|\r", prefix)
120 for index, full_line in enumerate(lines):
121 consumed += len(full_line) + 1 # adding the length of the split '\n'
122 line = full_line.lstrip()
123 if not line:
124 nlines += 1
125 if "\f" in full_line:
126 form_feed = True
127 if not line.startswith("#"):
128 # Escaped newlines outside of a comment are not really newlines at
129 # all. We treat a single-line comment following an escaped newline
130 # as a simple trailing comment.
131 if line.endswith("\\"):
132 ignored_lines += 1
133 continue
135 if index == ignored_lines and not is_endmarker:
136 comment_type = token.COMMENT # simple trailing comment
137 else:
138 comment_type = STANDALONE_COMMENT
139 comment = make_comment(line, mode=mode)
140 result.append(
141 ProtoComment(
142 type=comment_type,
143 value=comment,
144 newlines=nlines,
145 consumed=consumed,
146 form_feed=form_feed,
147 original_line=full_line,
148 lines_before_leaf=len(lines) - 1 - index,
149 )
150 )
151 form_feed = False
152 nlines = 0
153 return result
156def normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None:
157 """Normalize the prefix that's left over after generating comments.
159 Note: don't use backslashes for formatting or you'll lose your voting rights.
160 """
161 remainder = leaf.prefix[total_consumed:]
162 if "\\" not in remainder:
163 nl_count = remainder.count("\n")
164 form_feed = "\f" in remainder and remainder.endswith("\n")
165 leaf.prefix = make_simple_prefix(nl_count, form_feed)
166 return
168 leaf.prefix = ""
171def make_comment(content: str, mode: Mode) -> str:
172 """Return a consistently formatted comment from the given `content` string.
174 All comments (except for "##", "#!", "#:", '#'") should have a single
175 space between the hash sign and the content.
177 If `content` didn't start with a hash sign, one is provided.
179 Comments containing fmt directives are preserved exactly as-is to respect
180 user intent (e.g., `#no space # fmt: skip` stays as-is).
181 """
182 content = content.rstrip()
183 if not content:
184 return "#"
186 # Preserve comments with fmt directives exactly as-is
187 if content.startswith("#") and contains_fmt_directive(content):
188 return content
190 if content[0] == "#":
191 content = content[1:]
192 if (
193 content
194 and content[0] == "\N{NO-BREAK SPACE}"
195 and not is_type_comment_string("# " + content.lstrip(), mode=mode)
196 ):
197 content = " " + content[1:] # Replace NBSP by a simple space
198 if (
199 content
200 and "\N{NO-BREAK SPACE}" not in content
201 and is_type_comment_string("#" + content, mode=mode)
202 ):
203 type_part, value_part = content.split(":", 1)
204 content = type_part.strip() + ": " + value_part.strip()
206 if content and content[0] not in COMMENT_EXCEPTIONS:
207 content = " " + content
208 return "#" + content
211def normalize_fmt_off(
212 node: Node, mode: Mode, lines: Collection[tuple[int, int]]
213) -> None:
214 """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
215 # Walk the leaves once and resume scanning from the last converted position
216 # instead of restarting from the root for every pair. Converting a pair only
217 # mutates the tree at or after the converted leaf, so earlier leaves never
218 # need to be revisited.
219 leaves = list(node.leaves())
220 # Per-parent hint of where the previous conversion removed a child, so the
221 # left-to-right removals below don't rescan each parent's children list from
222 # index 0 every time (quadratic when a file has many `# fmt: off` blocks
223 # sharing one parent).
224 search_hints: dict[int, int] = {}
225 i = 0
226 while i < len(leaves):
227 i = convert_one_fmt_off_pair(node, leaves, i, mode, lines, search_hints)
230def _should_process_fmt_comment(
231 comment: ProtoComment, leaf: Leaf
232) -> tuple[bool, bool, bool]:
233 """Check if comment should be processed for fmt handling.
235 Returns (should_process, is_fmt_off, is_fmt_skip).
236 """
237 is_fmt_off = contains_fmt_directive(comment.value, FMT_OFF)
238 is_fmt_skip = contains_fmt_directive(comment.value, FMT_SKIP)
240 if not is_fmt_off and not is_fmt_skip:
241 return False, False, False
243 # Invalid use when `# fmt: off` is applied before a closing bracket
244 if is_fmt_off and leaf.type in CLOSING_BRACKETS:
245 return False, False, False
247 return True, is_fmt_off, is_fmt_skip
250def _is_valid_standalone_fmt_comment(
251 comment: ProtoComment, leaf: Leaf, is_fmt_off: bool, is_fmt_skip: bool
252) -> bool:
253 """Check if comment is a valid standalone fmt directive.
255 We only want standalone comments. If there's no previous leaf or if
256 the previous leaf is indentation, it's a standalone comment in disguise.
257 """
258 prev = preceding_leaf(leaf)
259 if comment.type == STANDALONE_COMMENT:
260 # `# fmt: skip` applies to the code on its line, so on a line of its own
261 # between statements it skips nothing, not the statement above either.
262 return not (is_fmt_skip and prev is not None and prev.type in WHITESPACE)
264 if not prev:
265 return True
267 # Treat STANDALONE_COMMENT nodes as whitespace for check
268 if is_fmt_off and prev.type not in WHITESPACE and prev.type != STANDALONE_COMMENT:
269 return False
270 if is_fmt_skip and prev.type in WHITESPACE:
271 return False
273 return True
276def _handle_comment_only_fmt_block(
277 leaf: Leaf,
278 comment: ProtoComment,
279 previous_consumed: int,
280 mode: Mode,
281) -> bool:
282 """Handle fmt:off/on blocks that contain only comments.
284 Returns True if a block was converted, False otherwise.
285 """
286 all_comments = list_comments(leaf.prefix, is_endmarker=False, mode=mode)
288 # Find the first fmt:off and its matching fmt:on
289 fmt_off_idx = None
290 fmt_on_idx = None
291 for idx, c in enumerate(all_comments):
292 if fmt_off_idx is None and contains_fmt_directive(c.value, FMT_OFF):
293 fmt_off_idx = idx
294 if (
295 fmt_off_idx is not None
296 and idx > fmt_off_idx
297 and contains_fmt_directive(c.value, FMT_ON)
298 ):
299 fmt_on_idx = idx
300 break
302 # Only proceed if we found both directives
303 if fmt_on_idx is None or fmt_off_idx is None:
304 return False
306 comment = all_comments[fmt_off_idx]
307 fmt_on_comment = all_comments[fmt_on_idx]
308 original_prefix = leaf.prefix
310 # Build the hidden value
311 start_pos = comment.consumed
312 end_pos = fmt_on_comment.consumed
313 content_between_and_fmt_on = original_prefix[start_pos:end_pos]
314 hidden_value = comment.value + "\n" + content_between_and_fmt_on
316 if hidden_value.endswith("\n"):
317 hidden_value = hidden_value[:-1]
319 # Build the standalone comment prefix - preserve all content before fmt:off
320 # including any comments that precede it
321 if fmt_off_idx == 0:
322 # No comments before fmt:off, use previous_consumed
323 pre_fmt_off_consumed = previous_consumed
324 else:
325 # Use the consumed position of the last comment before fmt:off
326 # This preserves all comments and content before the fmt:off directive
327 pre_fmt_off_consumed = all_comments[fmt_off_idx - 1].consumed
329 preceding_prefix = original_prefix[:pre_fmt_off_consumed]
330 if fmt_off_idx > 0:
331 preceding_prefix = preceding_prefix.lstrip("\r\n")
333 standalone_comment_prefix = preceding_prefix + "\n" * comment.newlines
335 fmt_off_prefix = original_prefix.split(comment.value)[0]
336 if "\n" in fmt_off_prefix:
337 fmt_off_prefix = fmt_off_prefix.split("\n")[-1]
338 standalone_comment_prefix += fmt_off_prefix
340 # Update leaf prefix
341 leaf.prefix = original_prefix[fmt_on_comment.consumed :]
343 # Insert the STANDALONE_COMMENT
344 parent = leaf.parent
345 assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (prefix only)"
347 leaf_idx = None
348 for idx, child in enumerate(parent.children):
349 if child is leaf:
350 leaf_idx = idx
351 break
353 assert leaf_idx is not None, "INTERNAL ERROR: fmt: on/off handling (leaf index)"
355 parent.insert_child(
356 leaf_idx,
357 Leaf(
358 STANDALONE_COMMENT,
359 hidden_value,
360 prefix=standalone_comment_prefix,
361 fmt_pass_converted_first_leaf=None,
362 ),
363 )
364 return True
367def convert_one_fmt_off_pair(
368 node: Node,
369 leaves: list[Leaf],
370 start: int,
371 mode: Mode,
372 lines: Collection[tuple[int, int]],
373 search_hints: dict[int, int] | None = None,
374) -> int:
375 """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
377 Scans `leaves` starting at `start`. Returns the index to resume scanning from:
378 the index of the leaf that was just converted (so further directives on the same
379 leaf are reconsidered), or ``len(leaves)`` when no further pair is found.
380 """
381 for idx in range(start, len(leaves)):
382 leaf = leaves[idx]
383 # A previous conversion may have detached this leaf from the tree (e.g. it
384 # was absorbed into a fmt: off block); such leaves are already handled.
385 if not _is_attached(leaf, node):
386 continue
388 # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on/skip processing
389 # to avoid reprocessing them in subsequent iterations
390 if leaf.type == STANDALONE_COMMENT and hasattr(
391 leaf, "fmt_pass_converted_first_leaf"
392 ):
393 continue
395 previous_consumed = 0
396 for comment in list_comments(leaf.prefix, is_endmarker=False, mode=mode):
397 should_process, is_fmt_off, is_fmt_skip = _should_process_fmt_comment(
398 comment, leaf
399 )
400 if not should_process:
401 previous_consumed = comment.consumed
402 continue
404 if not _is_valid_standalone_fmt_comment(
405 comment, leaf, is_fmt_off, is_fmt_skip
406 ):
407 previous_consumed = comment.consumed
408 continue
410 ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode))
412 # Handle comment-only blocks
413 if not ignored_nodes and is_fmt_off:
414 if _handle_comment_only_fmt_block(
415 leaf, comment, previous_consumed, mode
416 ):
417 return idx
418 continue
420 # Need actual nodes to process
421 if not ignored_nodes:
422 continue
424 # Handle regular fmt blocks
426 _handle_regular_fmt_block(
427 ignored_nodes,
428 comment,
429 previous_consumed,
430 is_fmt_skip,
431 lines,
432 leaf,
433 mode,
434 search_hints,
435 )
436 return idx
438 return len(leaves)
441def _is_attached(leaf: Leaf, root: Node) -> bool:
442 """Return whether `leaf` is still reachable from `root` through its parents."""
443 current: LN | None = leaf
444 while current is not None:
445 if current is root:
446 return True
447 current = current.parent
448 return False
451def _remove_preceding_newline_for_comment(result: str) -> str:
452 comment_line_start = result.rfind("\n") + 1
453 comment_start = result.find("#", comment_line_start)
454 if comment_start < 0:
455 return result
457 newline_before_comment = result.rfind("\n", 0, comment_start)
458 if newline_before_comment >= 0 and result[
459 newline_before_comment + 1 :
460 ].lstrip().startswith("#"):
461 return result[:newline_before_comment] + result[newline_before_comment + 1 :]
462 return result
465def _handle_regular_fmt_block(
466 ignored_nodes: list[LN],
467 comment: ProtoComment,
468 previous_consumed: int,
469 is_fmt_skip: bool,
470 lines: Collection[tuple[int, int]],
471 leaf: Leaf,
472 mode: Mode,
473 search_hints: dict[int, int] | None = None,
474) -> None:
475 """Handle fmt blocks with actual AST nodes."""
476 first = ignored_nodes[0] # Can be a container node with the `leaf`.
477 parent = first.parent
478 prefix = first.prefix
479 trailing_comments = (
480 "" if is_fmt_skip else _take_comments_ending_fmt_off(ignored_nodes[-1], mode)
481 )
483 if contains_fmt_directive(comment.value, FMT_OFF):
484 first.prefix = prefix[comment.consumed :]
485 if is_fmt_skip:
486 first.prefix = ""
487 standalone_comment_prefix = prefix
488 else:
489 standalone_comment_prefix = prefix[:previous_consumed] + "\n" * comment.newlines
491 # Ensure STANDALONE_COMMENT nodes have trailing newlines when stringified
492 # This prevents multiple fmt: skip comments from being concatenated on one line
493 def stringify_standalone_comment(node: Leaf, next_node: LN | None) -> str:
494 node_str = str(node)
495 if not node_str.endswith("\n") and (
496 next_node is None
497 # A comment-only fmt:off/on block ends with the `# fmt: on` comment
498 or node.fmt_pass_converted_first_leaf is None
499 or (isinstance(next_node, Leaf) and next_node.type == STANDALONE_COMMENT)
500 or "\n" in next_node.prefix
501 ):
502 node_str += "\n"
503 return node_str
505 parts = []
506 for node_index, node in enumerate(ignored_nodes):
507 if isinstance(node, Leaf) and node.type == STANDALONE_COMMENT:
508 next_node = (
509 ignored_nodes[node_index + 1]
510 if node_index + 1 < len(ignored_nodes)
511 else None
512 )
513 parts.append(stringify_standalone_comment(node, next_node))
514 elif isinstance(node, Node):
515 # For nodes that might contain STANDALONE_COMMENT leaves,
516 # we need custom stringify
517 has_standalone = any(
518 leaf.type == STANDALONE_COMMENT for leaf in node.leaves()
519 )
520 if has_standalone:
521 # Stringify node with STANDALONE_COMMENT leaves having trailing newlines
522 def stringify_node(n: LN, next_node: LN | None = None) -> str:
523 if isinstance(n, Leaf):
524 if n.type == STANDALONE_COMMENT:
525 result = n.prefix + n.value
526 if (
527 isinstance(next_node, Leaf)
528 and next_node.type in CLOSING_BRACKETS
529 and "\n" in n.value
530 ):
531 result = _remove_preceding_newline_for_comment(result)
532 # The newline ends a trailing comment. A skipped block
533 # that ends in code, like the `)` of an inner
534 # `( # fmt: skip` block, is followed by its own source.
535 last_line = result.rpartition("\n")[2]
536 if not result.endswith("\n") and "#" in last_line:
537 result += "\n"
538 return result
539 return str(n)
540 else:
541 # For nested nodes, recursively process children
542 children = n.children
543 next_children = [*children[1:], None]
544 return "".join(
545 stringify_node(child, next_child)
546 for child, next_child in zip(
547 children, next_children, strict=True
548 )
549 )
551 parts.append(stringify_node(node))
552 else:
553 parts.append(str(node))
554 else:
555 parts.append(str(node))
557 hidden_value = "".join(parts) + trailing_comments
558 comment_lineno = leaf.lineno - comment.newlines
559 leaf_is_ignored = any(
560 ignored is leaf
561 or (
562 isinstance(ignored, Node)
563 and any(child is leaf for child in ignored.leaves())
564 )
565 for ignored in ignored_nodes
566 )
568 if contains_fmt_directive(comment.value, FMT_OFF):
569 fmt_off_prefix = ""
570 if len(lines) > 0 and not any(
571 line[0] <= comment_lineno <= line[1] for line in lines
572 ):
573 # keeping indentation of comment by preserving original whitespaces.
574 fmt_off_prefix = prefix.split(comment.value)[0]
575 if "\n" in fmt_off_prefix:
576 fmt_off_prefix = fmt_off_prefix.split("\n")[-1]
577 standalone_comment_prefix += fmt_off_prefix
578 hidden_value = comment.value + "\n" + hidden_value
580 if is_fmt_skip and not leaf_is_ignored:
581 whitespace = comment.original_line.partition("#")[0]
582 hidden_value += whitespace + comment.value
584 if hidden_value.endswith("\n"):
585 # That happens when one of the `ignored_nodes` ended with a NEWLINE
586 # leaf (possibly followed by a DEDENT).
587 hidden_value = hidden_value[:-1]
589 first_idx: int | None = None
590 for ignored in ignored_nodes:
591 ignored_parent = ignored.parent
592 hint = (
593 search_hints.get(id(ignored_parent), 0)
594 if search_hints is not None and ignored_parent is not None
595 else 0
596 )
597 index = ignored.remove(hint)
598 if (
599 index is not None
600 and search_hints is not None
601 and ignored_parent is not None
602 ):
603 search_hints[id(ignored_parent)] = index
604 if first_idx is None:
605 first_idx = index
607 assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
608 assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
610 parent.insert_child(
611 first_idx,
612 Leaf(
613 STANDALONE_COMMENT,
614 hidden_value,
615 prefix=standalone_comment_prefix,
616 fmt_pass_converted_first_leaf=first_leaf_of(first),
617 ),
618 )
621def _take_comments_ending_fmt_off(last_ignored: LN, mode: Mode) -> str:
622 """Detach the comments that end a `# fmt: off` region inside brackets.
624 The comments between the last ignored node and `# fmt: on` live in the prefix of
625 the leaf after the region. So do all of its comments when that leaf is a closing
626 bracket and the region runs to the end of the bracket without a `# fmt: on`.
627 Return that part of the prefix and remove it from the leaf, so it is kept
628 verbatim instead of being formatted.
629 """
630 last = last_leaf(last_ignored)
631 if last is None or last.type in {token.NEWLINE, token.DEDENT}:
632 # The region ends at statement level, where comments before `# fmt: on`
633 # are left to the blank line handling.
634 return ""
636 node: LN | None = last_ignored
637 while node is not None and node.next_sibling is None:
638 node = node.parent
639 if node is None or node.next_sibling is None:
640 return ""
642 next_leaf = first_leaf_of(node.next_sibling)
643 if next_leaf is None:
644 return ""
646 consumed = 0
647 fmt_on = False
648 for comment in list_comments(next_leaf.prefix, is_endmarker=False, mode=mode):
649 if contains_fmt_directive(comment.value, FMT_ON):
650 fmt_on = True
651 break
652 consumed = comment.consumed
653 if not consumed or not (fmt_on or next_leaf.type in CLOSING_BRACKETS):
654 return ""
656 # A comment always ends its line before the next leaf. Leave that newline in the
657 # prefix, so whatever follows (`# fmt: on` or the leaf) stays on its own line.
658 consumed -= 1
659 trailing_comments = next_leaf.prefix[:consumed]
660 next_leaf.prefix = next_leaf.prefix[consumed:]
661 return trailing_comments
664def generate_ignored_nodes(
665 leaf: Leaf, comment: ProtoComment, mode: Mode
666) -> Iterator[LN]:
667 """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
669 If comment is skip, returns leaf only.
670 Stops at the end of the block.
671 """
672 if contains_fmt_directive(comment.value, FMT_SKIP):
673 yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, mode)
674 return
675 container: LN | None = container_of(leaf)
676 while container is not None and container.type != token.ENDMARKER:
677 if is_fmt_on(container, mode=mode):
678 return
680 # fix for fmt: on in children
681 if children_contains_fmt_on(container, mode=mode):
682 for index, child in enumerate(container.children):
683 if isinstance(child, Leaf) and is_fmt_on(child, mode=mode):
684 if child.type in CLOSING_BRACKETS:
685 # This means `# fmt: on` is placed at a different bracket level
686 # than `# fmt: off`. This is an invalid use, but as a courtesy,
687 # we include this closing bracket in the ignored nodes.
688 # The alternative is to fail the formatting.
689 yield child
690 return
691 if (
692 child.type == token.INDENT
693 and index < len(container.children) - 1
694 and children_contains_fmt_on(
695 container.children[index + 1], mode=mode
696 )
697 ):
698 # This means `# fmt: on` is placed right after an indentation
699 # level, and we shouldn't swallow the previous INDENT token.
700 return
701 if children_contains_fmt_on(child, mode=mode):
702 return
703 yield child
704 else:
705 if container.type == token.DEDENT and container.next_sibling is None:
706 # This can happen when there is no matching `# fmt: on` comment at the
707 # same level as `# fmt: on`. We need to keep this DEDENT.
708 return
709 yield container
710 container = container.next_sibling
713def _find_compound_statement_context(parent: Node) -> Node | None:
714 """Return the body node of a compound statement if we should respect fmt: skip.
716 This handles one-line compound statements like:
717 if condition: body # fmt: skip
719 When Black expands such statements, they temporarily look like:
720 if condition:
721 body # fmt: skip
723 In both cases, we want to return the body node (either the simple_stmt directly
724 or the suite containing it).
725 """
726 if parent.type != syms.simple_stmt:
727 return None
729 if not isinstance(parent.parent, Node):
730 return None
732 # Case 1: Expanded form after Black's initial formatting pass.
733 # The one-liner has been split across multiple lines:
734 # if True:
735 # print("a"); print("b") # fmt: skip
736 # Structure: compound_stmt -> suite -> simple_stmt
737 if (
738 parent.parent.type == syms.suite
739 and isinstance(parent.parent.parent, Node)
740 and parent.parent.parent.type in _COMPOUND_STATEMENTS
741 ):
742 return parent.parent
744 # Case 2: Original one-line form from the input source.
745 # The statement is still on a single line:
746 # if True: print("a"); print("b") # fmt: skip
747 # Structure: compound_stmt -> simple_stmt
748 if parent.parent.type in _COMPOUND_STATEMENTS:
749 return parent
751 return None
754def _should_keep_compound_statement_inline(
755 body_node: Node, simple_stmt_parent: Node
756) -> bool:
757 """Check if a compound statement should be kept on one line.
759 Returns True only for compound statements with semicolon-separated bodies,
760 like: if True: print("a"); print("b") # fmt: skip
761 """
762 # Narrow down to the single simple_stmt that may carry the semicolons before
763 # scanning any leaves. A compound statement's suite holds one child per body
764 # statement, so walking the whole suite here is O(n) and, called once per
765 # `# fmt: skip` line in the block, makes the pass O(n^2).
766 if body_node.type == syms.suite:
767 # After formatting: the suite must hold exactly one simple_stmt and it
768 # must be the one carrying the directive. Stop at the second simple_stmt.
769 target: LN | None = None
770 for child in body_node.children:
771 if child.type == syms.simple_stmt:
772 if target is not None:
773 return False
774 target = child
775 if target is None or target is not simple_stmt_parent:
776 return False
777 else:
778 # Original form: body_node IS the simple_stmt
779 if body_node is not simple_stmt_parent:
780 return False
781 target = body_node
783 return any(leaf.type == token.SEMI for leaf in target.leaves())
786def _get_compound_statement_header(
787 body_node: Node, simple_stmt_parent: Node, *, ignored: Collection[LN] = ()
788) -> list[LN]:
789 """Get header nodes for a compound statement that should be preserved inline.
791 Leaves of `ignored` are left out, so they aren't added a second time.
792 """
793 if not _should_keep_compound_statement_inline(body_node, simple_stmt_parent):
794 return []
796 # Get the compound statement (parent of body)
797 compound_stmt = body_node.parent
798 if compound_stmt is None or compound_stmt.type not in _COMPOUND_STATEMENTS:
799 return []
801 covered = {id(leaf) for node in ignored for leaf in node.leaves()}
803 # Collect all header leaves before the body
804 header_leaves: list[LN] = []
805 # `async with`/`async for`/`async def` wrap the compound statement in an
806 # async_stmt, so the leading ASYNC token is a sibling of compound_stmt rather
807 # than a child. Pick it up first, otherwise it is left out of the ignored
808 # nodes and reformatted onto its own line, breaking the statement.
809 if (
810 compound_stmt.parent is not None
811 and compound_stmt.parent.type == syms.async_stmt
812 and compound_stmt.prev_sibling is not None
813 and compound_stmt.prev_sibling.type == token.ASYNC
814 and id(compound_stmt.prev_sibling) not in covered
815 ):
816 header_leaves.append(compound_stmt.prev_sibling)
817 for child in compound_stmt.children:
818 if child is body_node:
819 break
820 if isinstance(child, Leaf):
821 if child.type not in (token.NEWLINE, token.INDENT) and (
822 id(child) not in covered
823 ):
824 header_leaves.append(child)
825 else:
826 header_leaves.extend(
827 leaf for leaf in child.leaves() if id(leaf) not in covered
828 )
829 return header_leaves
832def _find_closest_previous_sibling(node: LN) -> LN | None:
833 """Find the closest previous sibling by walking up the ancestor chain."""
834 current: LN | None = node
835 while current is not None:
836 prev_sibling = current.prev_sibling
837 if prev_sibling is not None:
838 return prev_sibling
839 current = current.parent
840 return None
843def _children_lead_ignored_nodes(node: LN, ignored_nodes: list[LN]) -> bool:
844 """Return True if `ignored_nodes` starts with every child of `node`."""
845 children = node.children
846 return (
847 bool(children)
848 and len(ignored_nodes) >= len(children)
849 and all(a is b for a, b in zip(ignored_nodes, children, strict=False))
850 )
853def _generate_ignored_nodes_from_fmt_skip(
854 leaf: Leaf, comment: ProtoComment, mode: Mode
855) -> Iterator[LN]:
856 """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`."""
857 prev_sibling = leaf.prev_sibling
858 parent = leaf.parent
859 ignored_nodes: list[LN] = []
860 # Need to properly format the leaf prefix to compare it to comment.value,
861 # which is also formatted
862 comments = list_comments(leaf.prefix, is_endmarker=False, mode=mode)
863 if not comments or comment.value != comments[0].value:
864 return
866 if prev_sibling is None and parent is not None:
867 prev_sibling = parent.prev_sibling
869 if prev_sibling is None and comment.type == token.COMMENT:
870 prev_sibling = _find_closest_previous_sibling(leaf)
872 if parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE:
873 # The `# fmt: skip` is on the colon line of the if/while/def/class/...
874 # statements. The ignored nodes should be previous siblings of the
875 # parent suite node. Do this before the generic "same physical line"
876 # logic so multiline headers are preserved as a whole.
877 leaf.prefix = ""
878 parent_sibling = parent.prev_sibling
879 while parent_sibling is not None and parent_sibling.type != syms.suite:
880 ignored_nodes.insert(0, parent_sibling)
881 parent_sibling = parent_sibling.prev_sibling
882 # Special case for `async_stmt` where the ASYNC token is on the
883 # grandparent node.
884 grandparent = parent.parent
885 if (
886 grandparent is not None
887 and grandparent.prev_sibling is not None
888 and grandparent.prev_sibling.type == token.ASYNC
889 ):
890 ignored_nodes.insert(0, grandparent.prev_sibling)
891 yield from iter(ignored_nodes)
892 elif prev_sibling is not None:
893 # Generates the nodes to be ignored by `fmt: skip`.
895 # Nodes to ignore are the ones on the same line as the
896 # `# fmt: skip` comment, excluding the `# fmt: skip`
897 # node itself.
899 # Traversal process (starting at the `# fmt: skip` node):
900 # 1. Move to the `prev_sibling` of the current node.
901 # 2. If `prev_sibling` has children, go to its rightmost leaf.
902 # 3. If there's no `prev_sibling`, move up to the parent
903 # node and repeat.
904 # 4. Continue until:
905 # a. You encounter an `INDENT` or `NEWLINE` node (indicates
906 # start of the line).
907 # b. You reach the root node.
909 # Include all visited LEAVES in the ignored list, except INDENT
910 # or NEWLINE leaves.
912 current_node = prev_sibling
913 if (
914 isinstance(current_node, Leaf)
915 and current_node.type in OPENING_BRACKETS
916 and current_node.parent
917 and current_node.parent.type == syms.atom
918 ):
919 current_node = current_node.parent
921 ignored_nodes = [current_node]
922 while current_node.prev_sibling is None and current_node.parent is not None:
923 current_node = current_node.parent
925 # Track seen nodes to detect cycles that can occur after tree modifications
926 seen_nodes = {id(current_node)}
928 while "\n" not in current_node.prefix and current_node.prev_sibling is not None:
929 leaf_nodes = list(current_node.prev_sibling.leaves())
930 next_node = leaf_nodes[-1] if leaf_nodes else current_node
932 # Detect infinite loop - if we've seen this node before, stop
933 # This can happen when STANDALONE_COMMENT nodes are inserted
934 # during processing
935 if id(next_node) in seen_nodes:
936 break
938 current_node = next_node
939 seen_nodes.add(id(current_node))
941 # Stop if we encounter a STANDALONE_COMMENT created by fmt processing
942 if (
943 isinstance(current_node, Leaf)
944 and current_node.type == STANDALONE_COMMENT
945 and hasattr(current_node, "fmt_pass_converted_first_leaf")
946 ):
947 break
949 if (
950 current_node.type in CLOSING_BRACKETS
951 and current_node.parent
952 and current_node.parent.type in (
953 syms.atom,
954 syms.trailer,
955 syms.import_from,
956 )
957 ):
958 current_node = current_node.parent
960 if current_node.type in (token.NEWLINE, token.INDENT):
961 if not list_comments(
962 current_node.prefix, is_endmarker=False, mode=mode
963 ):
964 current_node.prefix = ""
965 break
967 if current_node.type == token.DEDENT:
968 break
970 # Special case for with expressions
971 # Without this, we can stuck inside the asexpr_test's children's children
972 if (
973 current_node.parent
974 and current_node.parent.type == syms.asexpr_test
975 and current_node.parent.parent
976 and current_node.parent.parent.type == syms.with_stmt
977 ):
978 current_node = current_node.parent
980 ignored_nodes.insert(0, current_node)
982 while current_node.prev_sibling is None and current_node.parent is not None:
983 current_node = current_node.parent
984 # Every child of the node we are climbing out of is now ignored, so
985 # take the node itself instead. Converting only its leaves would
986 # leave the emptied node in the tree, and visitors that expect
987 # children, like the one for PEP 695 type parameters, crash on it.
988 # Children collapsed on an earlier climb are already nodes here, so
989 # compare against children rather than leaves.
990 # The collapsed node can be the first child of a parent that is now
991 # fully ignored as well (`a + b` in `a + b if c else d`), so keep
992 # climbing and collapse that parent too. Stopping one level short
993 # leaves the parent's children in the list next to a sibling from
994 # the enclosing node, and the standalone comment then lands inside
995 # the parent while that sibling is removed from around it, turning
996 # e.g. a tuple into a call. Climbing also has to go on past parents
997 # that aren't fully ignored, or the walk stops before reaching the
998 # start of the line (`with` in `with not ( # fmt: skip`).
999 if _children_lead_ignored_nodes(current_node, ignored_nodes):
1000 ignored_nodes[: len(current_node.children)] = [current_node]
1002 # Special handling for compound statements with semicolon-separated bodies
1003 if isinstance(parent, Node):
1004 body_node = _find_compound_statement_context(parent)
1005 if body_node is not None:
1006 # The walk above can already reach the header, e.g. when the body
1007 # starts with a bare name as in `if x: a; b # fmt: skip`, so pass
1008 # the nodes it found to avoid adding those header leaves twice.
1009 header_nodes = _get_compound_statement_header(
1010 body_node, parent, ignored=ignored_nodes
1011 )
1012 if header_nodes:
1013 ignored_nodes = header_nodes + ignored_nodes
1015 # If the nodes captured for the comment's physical line leave a bracket
1016 # open, the `# fmt: skip` sits inside a multi-line bracketed statement
1017 # (e.g. on the `from x import (` line). Skipping only that line would
1018 # reformat the rest of the statement, so preserve the whole enclosing
1019 # statement instead. A closing bracket on the line can belong to one
1020 # opened on an earlier line (the `)` of a multi-line parameter list in
1021 # `) -> A[B # fmt: skip`), so it must not cancel an opening one.
1022 open_brackets = 0
1023 for node in ignored_nodes:
1024 for ignored_leaf in node.leaves():
1025 if ignored_leaf.type in OPENING_BRACKETS:
1026 open_brackets += 1
1027 elif ignored_leaf.type in CLOSING_BRACKETS and open_brackets:
1028 open_brackets -= 1
1029 if open_brackets:
1030 statement: LN = leaf
1031 while statement.parent is not None and statement.parent.type not in (
1032 syms.file_input,
1033 syms.suite,
1034 ):
1035 statement = statement.parent
1036 ignored_nodes = [statement]
1038 leaf_is_ignored = any(
1039 ignored is leaf
1040 or (
1041 isinstance(ignored, Node)
1042 and any(child is leaf for child in ignored.leaves())
1043 )
1044 for ignored in ignored_nodes
1045 )
1046 if not leaf_is_ignored:
1047 leaf.prefix = leaf.prefix[comment.consumed :]
1049 yield from ignored_nodes
1052def is_fmt_on(container: LN, mode: Mode) -> bool:
1053 """Determine whether formatting is switched on within a container.
1054 Determined by whether the last `# fmt:` comment is `on` or `off`.
1055 """
1056 fmt_on = False
1057 for comment in list_comments(container.prefix, is_endmarker=False, mode=mode):
1058 if contains_fmt_directive(comment.value, FMT_ON):
1059 fmt_on = True
1060 elif contains_fmt_directive(comment.value, FMT_OFF):
1061 fmt_on = False
1062 return fmt_on
1065def children_contains_fmt_on(container: LN, mode: Mode) -> bool:
1066 """Determine if children have formatting switched on."""
1067 for child in container.children:
1068 leaf = first_leaf_of(child)
1069 if leaf is not None and is_fmt_on(leaf, mode=mode):
1070 return True
1072 return False
1075def contains_pragma_comment(comment_list: list[Leaf]) -> bool:
1076 """
1077 Returns:
1078 True iff one of the comments in @comment_list is a pragma used by one
1079 of the more common static analysis tools for python (e.g. mypy, flake8,
1080 pylint).
1081 """
1082 for comment in comment_list:
1083 if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
1084 return True
1086 return False
1089def contains_fmt_directive(
1090 comment_line: str, directives: set[str] = FMT_OFF | FMT_ON | FMT_SKIP
1091) -> bool:
1092 """
1093 Checks if the given comment contains format directives, alone or paired with
1094 other comments.
1096 Defaults to checking all directives (skip, off, on, yapf), but can be
1097 narrowed to specific ones.
1099 Matching styles:
1100 # foobar <-- single comment
1101 # foobar # foobar # foobar <-- multiple comments
1102 # foobar; foobar <-- list of comments (; separated)
1103 """
1104 semantic_comment_blocks = [
1105 comment_line,
1106 *[
1107 _COMMENT_PREFIX + comment.strip()
1108 for comment in comment_line.split(_COMMENT_PREFIX)[1:]
1109 ],
1110 *[
1111 _COMMENT_PREFIX + comment.strip()
1112 for comment in comment_line.strip(_COMMENT_PREFIX).split(
1113 _COMMENT_LIST_SEPARATOR
1114 )
1115 ],
1116 ]
1118 return any(comment in directives for comment in semantic_comment_blocks)