Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 COMPARATOR_PRIORITY,
16 DOT_PRIORITY,
17 STRING_PRIORITY,
18 get_leaves_inside_matching_brackets,
19 max_delimiter_priority_in_atom,
20)
21from black.comments import (
22 FMT_OFF,
23 FMT_ON,
24 contains_fmt_directive,
25 generate_comments,
26 list_comments,
27)
28from black.lines import (
29 Line,
30 RHSResult,
31 append_leaves,
32 can_be_split,
33 can_omit_invisible_parens,
34 is_line_short_enough,
35 line_to_string,
36)
37from black.mode import Feature, Mode, Preview
38from black.nodes import (
39 ASSIGNMENTS,
40 BRACKETS,
41 CLOSING_BRACKETS,
42 OPENING_BRACKETS,
43 STANDALONE_COMMENT,
44 STATEMENT,
45 WHITESPACE,
46 Visitor,
47 ensure_visible,
48 fstring_tstring_to_string,
49 get_annotation_type,
50 has_sibling_with_type,
51 is_arith_like,
52 is_async_stmt_or_funcdef,
53 is_atom_with_invisible_parens,
54 is_docstring,
55 is_empty_tuple,
56 is_generator,
57 is_lpar_token,
58 is_multiline_string,
59 is_name_token,
60 is_one_sequence_between,
61 is_one_tuple,
62 is_parent_function_or_class,
63 is_part_of_annotation,
64 is_rpar_token,
65 is_stub_body,
66 is_stub_suite,
67 is_tuple,
68 is_tuple_containing_star,
69 is_tuple_containing_walrus,
70 is_type_ignore_comment_string,
71 is_vararg,
72 is_walrus_assignment,
73 is_yield,
74 syms,
75 wrap_in_parentheses,
76)
77from black.numerics import normalize_numeric_literal
78from black.strings import (
79 fix_multiline_docstring,
80 get_string_prefix,
81 normalize_string_prefix,
82 normalize_string_quotes,
83 normalize_unicode_escape_sequences,
84 str_width,
85)
86from black.trans import (
87 CannotTransform,
88 StringMerger,
89 StringParenStripper,
90 StringParenWrapper,
91 StringSplitter,
92 Transformer,
93 hug_power_op,
94)
95from blib2to3.pgen2 import token
96from blib2to3.pytree import Leaf, Node
98# types
99LeafID = int
100LN = Union[Leaf, Node]
103class CannotSplit(CannotTransform):
104 """A readable split that fits the allotted line length is impossible."""
107# This isn't a dataclass because @dataclass + Generic breaks mypyc.
108# See also https://github.com/mypyc/mypyc/issues/827.
109class LineGenerator(Visitor[Line]):
110 """Generates reformatted Line objects. Empty lines are not emitted.
112 Note: destroys the tree it's visiting by mutating prefixes of its leaves
113 in ways that will no longer stringify to valid Python code on the tree.
114 """
116 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
117 self.mode = mode
118 self.features = features
119 self.current_line: Line
120 self.__post_init__()
122 def line(self, indent: int = 0) -> Iterator[Line]:
123 """Generate a line.
125 If the line is empty, only emit if it makes sense.
126 If the line is too long, split it first and then generate.
128 If any lines were generated, set up a new current_line.
129 """
130 if not self.current_line:
131 self.current_line.depth += indent
132 return # Line is empty, don't emit. Creating a new one unnecessary.
134 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
135 self.current_line.leaves[0]
136 ):
137 # Special case for async def/for/with statements. `visit_async_stmt`
138 # adds an `ASYNC` leaf then visits the child def/for/with statement
139 # nodes. Line yields from those nodes shouldn't treat the former
140 # `ASYNC` leaf as a complete line.
141 return
143 complete_line = self.current_line
144 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
145 yield complete_line
147 def visit_default(self, node: LN) -> Iterator[Line]:
148 """Default `visit_*()` implementation. Recurses to children of `node`."""
149 if isinstance(node, Leaf):
150 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
151 for comment in generate_comments(node, mode=self.mode):
152 if any_open_brackets:
153 # any comment within brackets is subject to splitting
154 self.current_line.append(comment)
155 elif comment.type == token.COMMENT:
156 # regular trailing comment
157 self.current_line.append(comment)
158 yield from self.line()
160 else:
161 # regular standalone comment
162 yield from self.line()
164 self.current_line.append(comment)
165 yield from self.line()
167 if any_open_brackets:
168 node.prefix = ""
169 if node.type not in WHITESPACE:
170 self.current_line.append(node)
171 yield from super().visit_default(node)
173 def visit_test(self, node: Node) -> Iterator[Line]:
174 """Visit an `x if y else z` test"""
176 already_parenthesized = (
177 node.prev_sibling and node.prev_sibling.type == token.LPAR
178 )
180 if not already_parenthesized:
181 # Similar to logic in wrap_in_parentheses
182 lpar = Leaf(token.LPAR, "")
183 rpar = Leaf(token.RPAR, "")
184 prefix = node.prefix
185 node.prefix = ""
186 lpar.prefix = prefix
187 node.insert_child(0, lpar)
188 node.append_child(rpar)
190 yield from self.visit_default(node)
192 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
193 """Increase indentation level, maybe yield a line."""
194 # In blib2to3 INDENT never holds comments.
195 yield from self.line(+1)
196 yield from self.visit_default(node)
198 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
199 """Decrease indentation level, maybe yield a line."""
200 # The current line might still wait for trailing comments. At DEDENT time
201 # there won't be any (they would be prefixes on the preceding NEWLINE).
202 # Emit the line then.
203 yield from self.line()
205 # While DEDENT has no value, its prefix may contain standalone comments
206 # that belong to the current indentation level. Get 'em.
207 yield from self.visit_default(node)
209 # Finally, emit the dedent.
210 yield from self.line(-1)
212 def visit_stmt(
213 self, node: Node, keywords: set[str], parens: set[str]
214 ) -> Iterator[Line]:
215 """Visit a statement.
217 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
218 `def`, `with`, `class`, `assert`, and assignments.
220 The relevant Python language `keywords` for a given statement will be
221 NAME leaves within it. This methods puts those on a separate line.
223 `parens` holds a set of string leaf values immediately after which
224 invisible parens should be put.
225 """
226 normalize_invisible_parens(
227 node, parens_after=parens, mode=self.mode, features=self.features
228 )
229 for child in node.children:
230 if is_name_token(child) and child.value in keywords:
231 yield from self.line()
233 yield from self.visit(child)
235 def visit_typeparams(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[0].prefix = ""
239 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
240 yield from self.visit_default(node)
241 node.children[1].prefix = ""
243 def visit_paramspec(self, node: Node) -> Iterator[Line]:
244 yield from self.visit_default(node)
245 node.children[1].prefix = ""
247 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
248 if Preview.wrap_long_dict_values_in_parens in self.mode:
249 for i, child in enumerate(node.children):
250 if i == 0:
251 continue
252 if node.children[i - 1].type == token.COLON:
253 if (
254 child.type == syms.atom
255 and child.children[0].type in OPENING_BRACKETS
256 and not is_walrus_assignment(child)
257 ):
258 maybe_make_parens_invisible_in_atom(
259 child,
260 parent=node,
261 mode=self.mode,
262 features=self.features,
263 remove_brackets_around_comma=False,
264 )
265 else:
266 wrap_in_parentheses(node, child, visible=False, index=i)
267 yield from self.visit_default(node)
269 def visit_funcdef(self, node: Node) -> Iterator[Line]:
270 """Visit function definition."""
271 yield from self.line()
273 # Remove redundant brackets around return type annotation.
274 is_return_annotation = False
275 for child in node.children:
276 if child.type == token.RARROW:
277 is_return_annotation = True
278 elif is_return_annotation:
279 if child.type == syms.atom and child.children[0].type == token.LPAR:
280 if maybe_make_parens_invisible_in_atom(
281 child,
282 parent=node,
283 mode=self.mode,
284 features=self.features,
285 remove_brackets_around_comma=False,
286 ):
287 wrap_in_parentheses(node, child, visible=False)
288 else:
289 wrap_in_parentheses(node, child, visible=False)
290 is_return_annotation = False
292 for child in node.children:
293 yield from self.visit(child)
295 def visit_match_case(self, node: Node) -> Iterator[Line]:
296 """Visit either a match or case statement."""
297 normalize_invisible_parens(
298 node, parens_after=set(), mode=self.mode, features=self.features
299 )
301 yield from self.line()
302 for child in node.children:
303 yield from self.visit(child)
305 def visit_suite(self, node: Node) -> Iterator[Line]:
306 """Visit a suite."""
307 if is_stub_suite(node):
308 yield from self.visit(node.children[2])
309 else:
310 yield from self.visit_default(node)
312 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
313 """Visit a statement without nested statements."""
314 prev_type: int | None = None
315 for i, child in enumerate(node.children):
316 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
317 wrap_in_parentheses(node, child, visible=False, index=i)
318 prev_type = child.type
320 if node.parent and node.parent.type in STATEMENT:
321 if is_parent_function_or_class(node) and is_stub_body(node):
322 yield from self.visit_default(node)
323 else:
324 yield from self.line(+1)
325 yield from self.visit_default(node)
326 yield from self.line(-1)
328 else:
329 if node.parent and is_stub_suite(node.parent):
330 node.prefix = ""
331 yield from self.visit_default(node)
332 return
333 yield from self.line()
334 yield from self.visit_default(node)
336 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
337 """Visit `async def`, `async for`, `async with`."""
338 yield from self.line()
340 children = iter(node.children)
341 for child in children:
342 yield from self.visit(child)
344 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
345 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
346 # line.
347 break
349 internal_stmt = next(children)
350 yield from self.visit(internal_stmt)
352 def visit_decorators(self, node: Node) -> Iterator[Line]:
353 """Visit decorators."""
354 for child in node.children:
355 yield from self.line()
356 yield from self.visit(child)
358 def visit_power(self, node: Node) -> Iterator[Line]:
359 for idx, leaf in enumerate(node.children[:-1]):
360 next_leaf = node.children[idx + 1]
362 if not isinstance(leaf, Leaf):
363 continue
365 value = leaf.value.lower()
366 if (
367 leaf.type == token.NUMBER
368 and next_leaf.type == syms.trailer
369 # Ensure that we are in an attribute trailer
370 and next_leaf.children[0].type == token.DOT
371 # It shouldn't wrap hexadecimal, binary and octal literals
372 and not value.startswith(("0x", "0b", "0o"))
373 # It shouldn't wrap complex literals
374 and "j" not in value
375 ):
376 wrap_in_parentheses(node, leaf)
378 remove_await_parens(node, mode=self.mode, features=self.features)
380 yield from self.visit_default(node)
382 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
383 """Remove a semicolon and put the other statement on a separate line."""
384 yield from self.line()
386 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
387 """End of file. Process outstanding comments and end with a newline."""
388 yield from self.visit_default(leaf)
389 yield from self.line()
391 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
392 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
393 if not any_open_brackets:
394 yield from self.line()
395 # STANDALONE_COMMENT nodes created by our special handling in
396 # normalize_fmt_off for comment-only blocks have fmt:off as the first
397 # line and fmt:on as the last line (each directive on its own line,
398 # not embedded in other text). These should be appended directly
399 # without calling visit_default, which would process their prefix and
400 # lose indentation. Normal STANDALONE_COMMENT nodes go through
401 # visit_default.
402 value = leaf.value
403 lines = value.splitlines()
404 is_fmt_off_block = (
405 len(lines) >= 2
406 and contains_fmt_directive(lines[0], FMT_OFF)
407 and contains_fmt_directive(lines[-1], FMT_ON)
408 )
409 if is_fmt_off_block:
410 is_after_invisible_lpar = (
411 leaf.fmt_pass_converted_first_leaf is None
412 and len(self.current_line.leaves) == 1
413 and self.current_line.leaves[0].type == token.LPAR
414 and not self.current_line.leaves[0].value
415 )
416 # This is a fmt:off/on block from normalize_fmt_off - we still need
417 # to process any prefix comments (like markdown comments) but append
418 # the fmt block itself directly to preserve its formatting
420 # Only process prefix comments if there actually is a prefix with comments
421 if leaf.prefix and any(
422 line.strip().startswith("#")
423 and not contains_fmt_directive(line.strip())
424 for line in leaf.prefix.split("\n")
425 ):
426 for comment in generate_comments(leaf, mode=self.mode):
427 yield from self.line()
428 self.current_line.append(comment)
429 yield from self.line()
430 # Clear the prefix since we've processed it as comments above
431 leaf.prefix = ""
433 self.current_line.append(leaf)
434 if not any_open_brackets or is_after_invisible_lpar:
435 yield from self.line()
436 else:
437 # Normal standalone comment - process through visit_default
438 yield from self.visit_default(leaf)
440 def visit_factor(self, node: Node) -> Iterator[Line]:
441 """Force parentheses between a unary op and a binary power:
443 -2 ** 8 -> -(2 ** 8)
444 """
445 _operator, operand = node.children
446 if (
447 operand.type == syms.power
448 and len(operand.children) == 3
449 and operand.children[1].type == token.DOUBLESTAR
450 ):
451 lpar = Leaf(token.LPAR, "(")
452 rpar = Leaf(token.RPAR, ")")
453 index = operand.remove() or 0
454 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
455 yield from self.visit_default(node)
457 def visit_tname(self, node: Node) -> Iterator[Line]:
458 """
459 Add potential parentheses around types in function parameter lists to be made
460 into real parentheses in case the type hint is too long to fit on a line
461 Examples:
462 def foo(a: int, b: float = 7): ...
464 ->
466 def foo(a: (int), b: (float) = 7): ...
467 """
468 if len(node.children) == 3 and maybe_make_parens_invisible_in_atom(
469 node.children[2], parent=node, mode=self.mode, features=self.features
470 ):
471 wrap_in_parentheses(node, node.children[2], visible=False)
473 yield from self.visit_default(node)
475 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
476 normalize_unicode_escape_sequences(leaf)
478 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
479 # We're ignoring docstrings with backslash newline escapes because changing
480 # indentation of those changes the AST representation of the code.
481 if self.mode.string_normalization:
482 docstring = normalize_string_prefix(leaf.value)
483 # We handle string normalization at the end of this method, but since
484 # what we do right now acts differently depending on quote style (ex.
485 # see padding logic below), there's a possibility for unstable
486 # formatting. To avoid a situation where this function formats a
487 # docstring differently on the second pass, normalize it early.
488 docstring = normalize_string_quotes(docstring)
489 else:
490 docstring = leaf.value
491 prefix = get_string_prefix(docstring)
492 docstring = docstring[len(prefix) :] # Remove the prefix
493 quote_char = docstring[0]
494 # A natural way to remove the outer quotes is to do:
495 # docstring = docstring.strip(quote_char)
496 # but that breaks on """""x""" (which is '""x').
497 # So we actually need to remove the first character and the next two
498 # characters but only if they are the same as the first.
499 quote_len = 1 if docstring[1] != quote_char else 3
500 docstring = docstring[quote_len:-quote_len]
501 docstring_started_empty = not docstring
502 indent = " " * 4 * self.current_line.depth
504 if is_multiline_string(leaf):
505 docstring = fix_multiline_docstring(docstring, indent)
506 else:
507 docstring = docstring.strip()
509 has_trailing_backslash = False
510 if docstring:
511 # Add some padding if the docstring starts / ends with a quote mark.
512 if docstring[0] == quote_char:
513 docstring = " " + docstring
514 if docstring[-1] == quote_char:
515 docstring += " "
516 if docstring[-1] == "\\":
517 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
518 if backslash_count % 2:
519 # Odd number of tailing backslashes, add some padding to
520 # avoid escaping the closing string quote.
521 docstring += " "
522 has_trailing_backslash = True
523 elif not docstring_started_empty:
524 docstring = " "
526 # We could enforce triple quotes at this point.
527 quote = quote_char * quote_len
529 # It's invalid to put closing single-character quotes on a new line.
530 if quote_len == 3:
531 # We need to find the length of the last line of the docstring
532 # to find if we can add the closing quotes to the line without
533 # exceeding the maximum line length.
534 # If docstring is one line, we don't put the closing quotes on a
535 # separate line because it looks ugly (#3320).
536 lines = docstring.splitlines()
537 last_line_length = len(lines[-1]) if docstring else 0
539 # If adding closing quotes would cause the last line to exceed
540 # the maximum line length, and the closing quote is not
541 # prefixed by a newline then put a line break before
542 # the closing quotes
543 if (
544 len(lines) > 1
545 and last_line_length + quote_len > self.mode.line_length
546 and len(indent) + quote_len <= self.mode.line_length
547 and not has_trailing_backslash
548 ):
549 if leaf.value[-1 - quote_len] == "\n":
550 leaf.value = prefix + quote + docstring + quote
551 else:
552 leaf.value = prefix + quote + docstring + "\n" + indent + quote
553 else:
554 leaf.value = prefix + quote + docstring + quote
555 else:
556 leaf.value = prefix + quote + docstring + quote
558 if self.mode.string_normalization and leaf.type == token.STRING:
559 leaf.value = normalize_string_prefix(leaf.value)
560 leaf.value = normalize_string_quotes(leaf.value)
561 yield from self.visit_default(leaf)
563 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
564 normalize_numeric_literal(leaf)
565 yield from self.visit_default(leaf)
567 def visit_atom(self, node: Node) -> Iterator[Line]:
568 """Visit any atom"""
569 if len(node.children) == 3:
570 first = node.children[0]
571 last = node.children[-1]
572 if (first.type == token.LSQB and last.type == token.RSQB) or (
573 first.type == token.LBRACE and last.type == token.RBRACE
574 ):
575 # Lists or sets of one item
576 maybe_make_parens_invisible_in_atom(
577 node.children[1],
578 parent=node,
579 mode=self.mode,
580 features=self.features,
581 )
583 yield from self.visit_default(node)
585 def visit_fstring(self, node: Node) -> Iterator[Line]:
586 # If the fstring was converted to a STANDALONE_COMMENT by
587 # normalize_fmt_off (e.g. it was inside a # fmt: off block),
588 # skip the fstring-to-string conversion and just visit normally.
589 if any(child.type == STANDALONE_COMMENT for child in node.children):
590 yield from self.visit_default(node)
591 return
592 # currently we don't want to format and split f-strings at all.
593 string_leaf = fstring_tstring_to_string(node)
594 node.replace(string_leaf)
595 if "\\" in string_leaf.value and any(
596 "\\" in str(child)
597 for child in node.children
598 if child.type == syms.fstring_replacement_field
599 ):
600 # string normalization doesn't account for nested quotes,
601 # causing breakages. skip normalization when nested quotes exist
602 yield from self.visit_default(string_leaf)
603 return
604 yield from self.visit_STRING(string_leaf)
606 def visit_tstring(self, node: Node) -> Iterator[Line]:
607 # If the tstring was converted to a STANDALONE_COMMENT by
608 # normalize_fmt_off, skip the conversion and just visit normally.
609 if any(child.type == STANDALONE_COMMENT for child in node.children):
610 yield from self.visit_default(node)
611 return
612 # currently we don't want to format and split t-strings at all.
613 string_leaf = fstring_tstring_to_string(node)
614 node.replace(string_leaf)
615 if "\\" in string_leaf.value and any(
616 "\\" in str(child)
617 for child in node.children
618 if child.type == syms.fstring_replacement_field
619 ):
620 # string normalization doesn't account for nested quotes,
621 # causing breakages. skip normalization when nested quotes exist
622 yield from self.visit_default(string_leaf)
623 return
624 yield from self.visit_STRING(string_leaf)
626 # TODO: Uncomment Implementation to format f-string children
627 # fstring_start = node.children[0]
628 # fstring_end = node.children[-1]
629 # assert isinstance(fstring_start, Leaf)
630 # assert isinstance(fstring_end, Leaf)
632 # quote_char = fstring_end.value[0]
633 # quote_idx = fstring_start.value.index(quote_char)
634 # prefix, quote = (
635 # fstring_start.value[:quote_idx],
636 # fstring_start.value[quote_idx:]
637 # )
639 # if not is_docstring(node, self.mode):
640 # prefix = normalize_string_prefix(prefix)
642 # assert quote == fstring_end.value
644 # is_raw_fstring = "r" in prefix or "R" in prefix
645 # middles = [
646 # leaf
647 # for leaf in node.leaves()
648 # if leaf.type == token.FSTRING_MIDDLE
649 # ]
651 # if self.mode.string_normalization:
652 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
654 # fstring_start.value = prefix + quote
655 # fstring_end.value = quote
657 # yield from self.visit_default(node)
659 def visit_comp_for(self, node: Node) -> Iterator[Line]:
660 if Preview.wrap_comprehension_in in self.mode:
661 normalize_invisible_parens(
662 node, parens_after={"in"}, mode=self.mode, features=self.features
663 )
664 yield from self.visit_default(node)
666 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
667 yield from self.visit_comp_for(node)
669 def __post_init__(self) -> None:
670 """You are in a twisty little maze of passages."""
671 self.current_line = Line(mode=self.mode)
673 v = self.visit_stmt
674 Ø: set[str] = set()
675 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
676 self.visit_if_stmt = partial(
677 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
678 )
679 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
680 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
681 self.visit_try_stmt = partial(
682 v, keywords={"try", "except", "else", "finally"}, parens=Ø
683 )
684 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
685 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
686 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
688 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
689 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
690 self.visit_yield_expr = partial(
691 v,
692 keywords=Ø,
693 parens=(
694 {"yield"} if Preview.parenthesize_tuple_in_yield in self.mode else Ø
695 ),
696 )
697 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
698 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
699 self.visit_async_funcdef = self.visit_async_stmt
700 self.visit_decorated = self.visit_decorators
702 # PEP 634
703 self.visit_match_stmt = self.visit_match_case
704 self.visit_case_block = self.visit_match_case
705 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
708# Remove when `simplify_power_operator_hugging` becomes stable.
709def _hugging_power_ops_line_to_string(
710 line: Line,
711 features: Collection[Feature],
712 mode: Mode,
713) -> str | None:
714 try:
715 return line_to_string(next(hug_power_op(line, features, mode)))
716 except CannotTransform:
717 return None
720def transform_line(
721 line: Line, mode: Mode, features: Collection[Feature] = ()
722) -> Iterator[Line]:
723 """Transform a `line`, potentially splitting it into many lines.
725 They should fit in the allotted `line_length` but might not be able to.
727 `features` are syntactical features that may be used in the output.
728 """
729 if line.is_comment:
730 yield line
731 return
733 line_str = line_to_string(line)
735 if Preview.simplify_power_operator_hugging in mode:
736 line_str_hugging_power_ops = line_str
737 else:
738 # We need the line string when power operators are hugging to determine if we
739 # should split the line. Default to line_str, if no power operator are present
740 # on the line.
741 line_str_hugging_power_ops = (
742 _hugging_power_ops_line_to_string(line, features, mode) or line_str
743 )
745 ll = mode.line_length
746 sn = mode.string_normalization
747 string_merge = StringMerger(ll, sn)
748 string_paren_strip = StringParenStripper(ll, sn)
749 string_split = StringSplitter(ll, sn)
750 string_paren_wrap = StringParenWrapper(ll, sn)
752 transformers: list[Transformer]
753 if (
754 not line.contains_uncollapsable_type_comments()
755 and not line.should_split_rhs
756 and not line.magic_trailing_comma
757 and (
758 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
759 or line.contains_unsplittable_type_ignore()
760 )
761 and not (line.inside_brackets and line.contains_standalone_comments())
762 and not line.contains_implicit_multiline_string_with_comments()
763 ):
764 # Only apply basic string preprocessing, since lines shouldn't be split here.
765 if Preview.string_processing in mode:
766 transformers = [string_merge, string_paren_strip]
767 else:
768 transformers = []
769 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
770 transformers = [left_hand_split]
771 else:
773 def _rhs(
774 self: object, line: Line, features: Collection[Feature], mode: Mode
775 ) -> Iterator[Line]:
776 """Wraps calls to `right_hand_split`.
778 The calls increasingly `omit` right-hand trailers (bracket pairs with
779 content), meaning the trailers get glued together to split on another
780 bracket pair instead.
781 """
782 for omit in generate_trailers_to_omit(line, mode.line_length):
783 lines = list(right_hand_split(line, mode, features, omit=omit))
784 # Note: this check is only able to figure out if the first line of the
785 # *current* transformation fits in the line length. This is true only
786 # for simple cases. All others require running more transforms via
787 # `transform_line()`. This check doesn't know if those would succeed.
788 if is_line_short_enough(lines[0], mode=mode) or (
789 omit and _over_length_only_due_to_subscript_comment(lines[0], mode)
790 ):
791 yield from lines
792 return
794 # All splits failed, best effort split with no omits.
795 # This mostly happens to multiline strings that are by definition
796 # reported as not fitting a single line, as well as lines that contain
797 # trailing commas (those have to be exploded).
798 yield from right_hand_split(line, mode, features=features)
800 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
801 # __name__ attribute which is needed in `run_transformer` further down.
802 # Unfortunately a nested class breaks mypyc too. So a class must be created
803 # via type ... https://github.com/mypyc/mypyc/issues/884
804 rhs = type("rhs", (), {"__call__": _rhs})()
806 if Preview.string_processing in mode:
807 if line.inside_brackets:
808 transformers = [
809 string_merge,
810 string_paren_strip,
811 string_split,
812 delimiter_split,
813 standalone_comment_split,
814 string_paren_wrap,
815 rhs,
816 ]
817 else:
818 transformers = [
819 string_merge,
820 string_paren_strip,
821 string_split,
822 string_paren_wrap,
823 rhs,
824 ]
825 else:
826 if line.inside_brackets:
827 transformers = [delimiter_split, standalone_comment_split, rhs]
828 else:
829 transformers = [rhs]
831 if Preview.simplify_power_operator_hugging not in mode:
832 # It's always safe to attempt hugging of power operations and pretty much every
833 # line could match.
834 transformers.append(hug_power_op)
836 for transform in transformers:
837 # We are accumulating lines in `result` because we might want to abort
838 # mission and return the original line in the end, or attempt a different
839 # split altogether.
840 try:
841 result = run_transformer(line, transform, mode, features, line_str=line_str)
842 except CannotTransform:
843 continue
844 else:
845 yield from result
846 break
848 else:
849 # A leftover standalone comment would render inline and break the
850 # second pass with a parse error (#4296). Force a split so the
851 # output is at least valid Python.
852 if line.contains_standalone_comments():
853 yield from _force_standalone_comment_split(line)
854 else:
855 yield line
858def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
859 """If a funcdef has a magic trailing comma in the return type, then we should first
860 split the line with rhs to respect the comma.
861 """
862 return_type_leaves: list[Leaf] = []
863 in_return_type = False
865 for leaf in line.leaves:
866 if leaf.type == token.COLON:
867 in_return_type = False
868 if in_return_type:
869 return_type_leaves.append(leaf)
870 if leaf.type == token.RARROW:
871 in_return_type = True
873 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
874 # couple lines from it.
875 result = Line(mode=line.mode, depth=line.depth)
876 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
877 for leaf in return_type_leaves:
878 result.append(
879 leaf,
880 preformatted=True,
881 track_bracket=id(leaf) in leaves_to_track,
882 )
884 # we could also return true if the line is too long, and the return type is longer
885 # than the param list. Or if `should_split_rhs` returns True.
886 return result.magic_trailing_comma is not None
889class _BracketSplitComponent(Enum):
890 head = auto()
891 body = auto()
892 tail = auto()
895def left_hand_split(
896 line: Line, _features: Collection[Feature], mode: Mode
897) -> Iterator[Line]:
898 """Split line into many lines, starting with the first matching bracket pair.
900 Note: this usually looks weird, only use this for function definitions.
901 Prefer RHS otherwise. This is why this function is not symmetrical with
902 :func:`right_hand_split` which also handles optional parentheses.
903 """
904 for leaf_type in [token.LPAR, token.LSQB]:
905 tail_leaves: list[Leaf] = []
906 body_leaves: list[Leaf] = []
907 head_leaves: list[Leaf] = []
908 current_leaves = head_leaves
909 matching_bracket: Leaf | None = None
910 depth = 0
911 for index, leaf in enumerate(line.leaves):
912 if index == 2 and leaf.type == token.LSQB:
913 # A [ at index 2 means this is a type param, so start
914 # tracking the depth
915 depth += 1
916 elif depth > 0:
917 if leaf.type == token.LSQB:
918 depth += 1
919 elif leaf.type == token.RSQB:
920 depth -= 1
921 if (
922 current_leaves is body_leaves
923 and leaf.type in CLOSING_BRACKETS
924 and leaf.opening_bracket is matching_bracket
925 and isinstance(matching_bracket, Leaf)
926 # If the code is still on LPAR and we are inside a type
927 # param, ignore the match since this is searching
928 # for the function arguments
929 and not (leaf_type == token.LPAR and depth > 0)
930 ):
931 ensure_visible(leaf)
932 ensure_visible(matching_bracket)
933 current_leaves = tail_leaves if body_leaves else head_leaves
934 current_leaves.append(leaf)
935 if current_leaves is head_leaves:
936 if leaf.type == leaf_type and (
937 not (leaf_type == token.LPAR and depth > 0)
938 ):
939 matching_bracket = leaf
940 current_leaves = body_leaves
941 if matching_bracket and tail_leaves:
942 break
943 if not matching_bracket or not tail_leaves:
944 raise CannotSplit("No brackets found")
946 head = bracket_split_build_line(
947 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
948 )
949 body = bracket_split_build_line(
950 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
951 )
952 tail = bracket_split_build_line(
953 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
954 )
955 bracket_split_succeeded_or_raise(head, body, tail)
956 for result in (head, body, tail):
957 if result:
958 yield result
961def right_hand_split(
962 line: Line,
963 mode: Mode,
964 features: Collection[Feature] = (),
965 omit: Collection[LeafID] = (),
966) -> Iterator[Line]:
967 """Split line into many lines, starting with the last matching bracket pair.
969 If the split was by optional parentheses, attempt splitting without them, too.
970 `omit` is a collection of closing bracket IDs that shouldn't be considered for
971 this split.
973 Note: running this function modifies `bracket_depth` on the leaves of `line`.
974 """
975 rhs_result = _first_right_hand_split(line, omit=omit)
976 yield from _maybe_split_omitting_optional_parens(
977 rhs_result, line, mode, features=features, omit=omit
978 )
981def _first_right_hand_split(
982 line: Line,
983 omit: Collection[LeafID] = (),
984) -> RHSResult:
985 """Split the line into head, body, tail starting with the last bracket pair.
987 Note: this function should not have side effects. It's relied upon by
988 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
989 splitting on the right side of an assignment statement.
990 """
991 tail_leaves: list[Leaf] = []
992 body_leaves: list[Leaf] = []
993 head_leaves: list[Leaf] = []
994 current_leaves = tail_leaves
995 opening_bracket: Leaf | None = None
996 closing_bracket: Leaf | None = None
997 for leaf in reversed(line.leaves):
998 if current_leaves is body_leaves:
999 if leaf is opening_bracket:
1000 current_leaves = head_leaves if body_leaves else tail_leaves
1001 current_leaves.append(leaf)
1002 if current_leaves is tail_leaves:
1003 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
1004 opening_bracket = leaf.opening_bracket
1005 closing_bracket = leaf
1006 current_leaves = body_leaves
1007 if not (opening_bracket and closing_bracket and head_leaves):
1008 # If there is no opening or closing_bracket that means the split failed and
1009 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
1010 # the matching `opening_bracket` wasn't available on `line` anymore.
1011 raise CannotSplit("No brackets found")
1013 tail_leaves.reverse()
1014 body_leaves.reverse()
1015 head_leaves.reverse()
1017 body: Line | None = None
1018 if (
1019 Preview.hug_parens_with_braces_and_square_brackets in line.mode
1020 and tail_leaves[0].value
1021 and tail_leaves[0].opening_bracket is head_leaves[-1]
1022 ):
1023 inner_body_leaves = list(body_leaves)
1024 hugged_opening_leaves: list[Leaf] = []
1025 hugged_closing_leaves: list[Leaf] = []
1026 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
1027 unpacking_offset: int = 1 if is_unpacking else 0
1028 while (
1029 len(inner_body_leaves) >= 2 + unpacking_offset
1030 and inner_body_leaves[-1].type in CLOSING_BRACKETS
1031 and inner_body_leaves[-1].opening_bracket
1032 is inner_body_leaves[unpacking_offset]
1033 ):
1034 if unpacking_offset:
1035 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1036 unpacking_offset = 0
1037 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1038 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
1040 if hugged_opening_leaves and inner_body_leaves:
1041 inner_body = bracket_split_build_line(
1042 inner_body_leaves,
1043 line,
1044 hugged_opening_leaves[-1],
1045 component=_BracketSplitComponent.body,
1046 )
1047 if (
1048 line.mode.magic_trailing_comma
1049 and inner_body_leaves[-1].type == token.COMMA
1050 ):
1051 should_hug = True
1052 else:
1053 line_length = line.mode.line_length - sum(
1054 len(str(leaf))
1055 for leaf in hugged_opening_leaves + hugged_closing_leaves
1056 )
1057 if is_line_short_enough(
1058 inner_body, mode=replace(line.mode, line_length=line_length)
1059 ):
1060 # Do not hug if it fits on a single line.
1061 should_hug = False
1062 else:
1063 should_hug = True
1064 if should_hug:
1065 body_leaves = inner_body_leaves
1066 head_leaves.extend(hugged_opening_leaves)
1067 tail_leaves = hugged_closing_leaves + tail_leaves
1068 body = inner_body # No need to re-calculate the body again later.
1070 head = bracket_split_build_line(
1071 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1072 )
1073 if body is None:
1074 body = bracket_split_build_line(
1075 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1076 )
1077 tail = bracket_split_build_line(
1078 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1079 )
1080 bracket_split_succeeded_or_raise(head, body, tail)
1081 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1084def _maybe_split_omitting_optional_parens(
1085 rhs: RHSResult,
1086 line: Line,
1087 mode: Mode,
1088 features: Collection[Feature] = (),
1089 omit: Collection[LeafID] = (),
1090) -> Iterator[Line]:
1091 if (
1092 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1093 # the opening bracket is an optional paren
1094 and rhs.opening_bracket.type == token.LPAR
1095 and not rhs.opening_bracket.value
1096 # the closing bracket is an optional paren
1097 and rhs.closing_bracket.type == token.RPAR
1098 and not rhs.closing_bracket.value
1099 # it's not an import (optional parens are the only thing we can split on
1100 # in this case; attempting a split without them is a waste of time)
1101 and not line.is_import
1102 # and we can actually remove the parens
1103 and can_omit_invisible_parens(rhs, mode.line_length, mode)
1104 ):
1105 omit = {id(rhs.closing_bracket), *omit}
1106 try:
1107 # The RHSResult Omitting Optional Parens.
1108 rhs_oop = _first_right_hand_split(line, omit=omit)
1109 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1110 yield from _maybe_split_omitting_optional_parens(
1111 rhs_oop, line, mode, features=features, omit=omit
1112 )
1113 return
1115 except CannotSplit as e:
1116 # For chained assignments we want to use the previous successful split
1117 if line.is_chained_assignment:
1118 pass
1120 elif (
1121 not can_be_split(rhs.body)
1122 and not is_line_short_enough(rhs.body, mode=mode)
1123 and not (
1124 Preview.wrap_long_dict_values_in_parens
1125 and rhs.opening_bracket.parent
1126 and rhs.opening_bracket.parent.parent
1127 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1128 )
1129 and not (
1130 rhs.opening_bracket.parent
1131 and rhs.opening_bracket.parent.parent
1132 and rhs.opening_bracket.parent.parent.type == syms.case_block
1133 )
1134 ):
1135 raise CannotSplit(
1136 "Splitting failed, body is still too long and can't be split."
1137 ) from e
1139 elif (
1140 rhs.head.contains_multiline_strings()
1141 or rhs.tail.contains_multiline_strings()
1142 ):
1143 raise CannotSplit(
1144 "The current optional pair of parentheses is bound to fail to"
1145 " satisfy the splitting algorithm because the head or the tail"
1146 " contains multiline strings which by definition never fit one"
1147 " line."
1148 ) from e
1150 ensure_visible(rhs.opening_bracket)
1151 ensure_visible(rhs.closing_bracket)
1152 for result in (rhs.head, rhs.body, rhs.tail):
1153 if result:
1154 yield result
1157def _prefer_split_rhs_oop_over_rhs(
1158 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1159) -> bool:
1160 """
1161 Returns whether we should prefer the result from a split omitting optional parens
1162 (rhs_oop) over the original (rhs).
1163 """
1164 # contains unsplittable type ignore
1165 if (
1166 rhs_oop.head.contains_unsplittable_type_ignore()
1167 or rhs_oop.body.contains_unsplittable_type_ignore()
1168 or rhs_oop.tail.contains_unsplittable_type_ignore()
1169 ):
1170 return True
1172 # Retain optional parens around dictionary values
1173 if (
1174 Preview.wrap_long_dict_values_in_parens
1175 and rhs.opening_bracket.parent
1176 and rhs.opening_bracket.parent.parent
1177 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1178 and rhs.body.bracket_tracker.delimiters
1179 ):
1180 # Unless the split is inside the key
1181 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1183 # the split is right after `=`
1184 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1185 return True
1187 # the left side of assignment contains brackets
1188 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1189 return True
1191 # the left side of assignment is short enough (the -1 is for the ending optional
1192 # paren)
1193 if not is_line_short_enough(
1194 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1195 ):
1196 return True
1198 # the left side of assignment won't explode further because of magic trailing comma
1199 if rhs.head.magic_trailing_comma is not None:
1200 return True
1202 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1203 # the body
1204 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1205 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1206 token.EQUAL
1207 )
1208 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1209 return False
1211 has_closing_bracket_after_assign = False
1212 for leaf in reversed(rhs_oop.head.leaves):
1213 if leaf.type == token.EQUAL:
1214 break
1215 if leaf.type in CLOSING_BRACKETS:
1216 has_closing_bracket_after_assign = True
1217 break
1218 return (
1219 # contains matching brackets after the `=` (done by checking there is a
1220 # closing bracket)
1221 has_closing_bracket_after_assign
1222 or (
1223 # the split is actually from inside the optional parens (done by checking
1224 # the first line still contains the `=`)
1225 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1226 # the first line is short enough
1227 and is_line_short_enough(rhs_oop.head, mode=mode)
1228 )
1229 )
1232def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1233 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1235 Do nothing otherwise.
1237 A left- or right-hand split is based on a pair of brackets. Content before
1238 (and including) the opening bracket is left on one line, content inside the
1239 brackets is put on a separate line, and finally content starting with and
1240 following the closing bracket is put on a separate line.
1242 Those are called `head`, `body`, and `tail`, respectively. If the split
1243 produced the same line (all content in `head`) or ended up with an empty `body`
1244 and the `tail` is just the closing bracket, then it's considered failed.
1245 """
1246 tail_len = len(str(tail).strip())
1247 if not body:
1248 if tail_len == 0:
1249 raise CannotSplit("Splitting brackets produced the same line")
1251 elif tail_len < 3:
1252 raise CannotSplit(
1253 f"Splitting brackets on an empty body to save {tail_len} characters is"
1254 " not worth it"
1255 )
1258def _ensure_trailing_comma(
1259 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1260) -> bool:
1261 if not leaves:
1262 return False
1263 # Ensure a trailing comma for imports
1264 if original.is_import:
1265 return True
1266 # ...and standalone function arguments
1267 if not original.is_def:
1268 return False
1269 if opening_bracket.value != "(":
1270 return False
1271 # Don't add commas if we already have any commas
1272 if any(
1273 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1274 ):
1275 return False
1277 # Find a leaf with a parent (comments don't have parents)
1278 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1279 if leaf_with_parent is None:
1280 return True
1281 # Don't add commas inside parenthesized return annotations
1282 if get_annotation_type(leaf_with_parent) == "return":
1283 return False
1284 # Don't add commas inside PEP 604 unions
1285 if (
1286 leaf_with_parent.parent
1287 and leaf_with_parent.parent.next_sibling
1288 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1289 ):
1290 return False
1291 return True
1294def bracket_split_build_line(
1295 leaves: list[Leaf],
1296 original: Line,
1297 opening_bracket: Leaf,
1298 *,
1299 component: _BracketSplitComponent,
1300) -> Line:
1301 """Return a new line with given `leaves` and respective comments from `original`.
1303 If it's the head component, brackets will be tracked so trailing commas are
1304 respected.
1306 If it's the body component, the result line is one-indented inside brackets and as
1307 such has its first leaf's prefix normalized and a trailing comma added when
1308 expected.
1309 """
1310 result = Line(mode=original.mode, depth=original.depth)
1311 if component is _BracketSplitComponent.body:
1312 result.inside_brackets = True
1313 result.depth += 1
1314 if _ensure_trailing_comma(leaves, original, opening_bracket):
1315 for i in range(len(leaves) - 1, -1, -1):
1316 if leaves[i].type == STANDALONE_COMMENT:
1317 continue
1319 if leaves[i].type != token.COMMA:
1320 new_comma = Leaf(token.COMMA, ",")
1321 leaves.insert(i + 1, new_comma)
1322 break
1324 leaves_to_track: set[LeafID] = set()
1325 if component is _BracketSplitComponent.head:
1326 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1327 # Populate the line
1328 for leaf in leaves:
1329 result.append(
1330 leaf,
1331 preformatted=True,
1332 track_bracket=id(leaf) in leaves_to_track,
1333 )
1334 for comment_after in original.comments_after(leaf):
1335 result.append(comment_after, preformatted=True)
1336 if component is _BracketSplitComponent.body and should_split_line(
1337 result, opening_bracket
1338 ):
1339 result.should_split_rhs = True
1340 return result
1343def dont_increase_indentation(split_func: Transformer) -> Transformer:
1344 """Normalize prefix of the first leaf in every line returned by `split_func`.
1346 This is a decorator over relevant split functions.
1347 """
1349 @wraps(split_func)
1350 def split_wrapper(
1351 line: Line, features: Collection[Feature], mode: Mode
1352 ) -> Iterator[Line]:
1353 for split_line in split_func(line, features, mode):
1354 split_line.leaves[0].prefix = ""
1355 yield split_line
1357 return split_wrapper
1360def _get_last_non_comment_leaf(line: Line) -> int | None:
1361 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1362 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1363 return leaf_idx
1364 return None
1367def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1368 if is_vararg(leaf, within={syms.typedargslist}):
1369 return Feature.TRAILING_COMMA_IN_DEF in features
1370 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1371 return Feature.TRAILING_COMMA_IN_CALL in features
1372 return True
1375def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1376 if (
1377 safe
1378 and delimiter_priority == COMMA_PRIORITY
1379 and line.leaves[-1].type != token.COMMA
1380 and line.leaves[-1].type != STANDALONE_COMMENT
1381 ):
1382 new_comma = Leaf(token.COMMA, ",")
1383 line.append(new_comma)
1384 return line
1387MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1390def _can_defer_lone_comparator_to_rhs(line: Line, mode: Mode) -> bool:
1391 """Return True if the lone comparator on `line` can defer to right_hand_split.
1393 Caller has already established exactly one delimiter at
1394 `COMPARATOR_PRIORITY`. We defer only when:
1396 - the LHS up to the comparator has no opening brackets, so the existing
1397 "break before the comparator" wouldn't produce a balanced two-sided
1398 split anyway, and
1399 - `right_hand_split` would produce a head that fits in the line length,
1400 so we don't strand `if t` on its own line just to push it back onto an
1401 overflowing single line when the RHS bracket can't be exploded
1402 usefully (e.g. an empty `decode()` paren).
1403 """
1404 past_comparator = False
1405 for leaf in line.leaves:
1406 if leaf.type in OPENING_BRACKETS and not past_comparator:
1407 return False
1408 if not past_comparator and (
1409 line.bracket_tracker.delimiters.get(id(leaf)) == COMPARATOR_PRIORITY
1410 ):
1411 past_comparator = True
1412 try:
1413 rhs = _first_right_hand_split(line)
1414 except CannotSplit:
1415 return False
1416 return is_line_short_enough(rhs.head, mode=mode)
1419@dont_increase_indentation
1420def delimiter_split(
1421 line: Line, features: Collection[Feature], mode: Mode
1422) -> Iterator[Line]:
1423 """Split according to delimiters of the highest priority.
1425 If the appropriate Features are given, the split will add trailing commas
1426 also in function signatures and calls that contain `*` and `**`.
1427 """
1428 if len(line.leaves) == 0:
1429 raise CannotSplit("Line empty") from None
1430 last_leaf = line.leaves[-1]
1432 bt = line.bracket_tracker
1433 try:
1434 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1435 except ValueError:
1436 raise CannotSplit("No delimiters found") from None
1438 if (
1439 delimiter_priority == DOT_PRIORITY
1440 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1441 ):
1442 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1444 if (
1445 Preview.hug_comparator in mode
1446 and delimiter_priority == COMPARATOR_PRIORITY
1447 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1448 and _can_defer_lone_comparator_to_rhs(line, mode)
1449 ):
1450 raise CannotSplit("Bracketed RHS will explode via right_hand_split")
1452 current_line = Line(
1453 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1454 )
1455 lowest_depth = sys.maxsize
1456 trailing_comma_safe = True
1458 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1459 """Append `leaf` to current line or to new line if appending impossible."""
1460 nonlocal current_line
1461 try:
1462 current_line.append_safe(leaf, preformatted=True)
1463 except ValueError:
1464 yield current_line
1466 current_line = Line(
1467 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1468 )
1469 current_line.append(leaf)
1471 def append_comments(leaf: Leaf) -> Iterator[Line]:
1472 for comment_after in line.comments_after(leaf):
1473 yield from append_to_line(comment_after)
1475 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1476 for leaf_idx, leaf in enumerate(line.leaves):
1477 yield from append_to_line(leaf)
1479 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1480 id(line.leaves[leaf_idx - 1])
1481 )
1482 if (
1483 previous_priority != delimiter_priority
1484 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1485 ):
1486 yield from append_comments(leaf)
1488 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1489 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1490 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1492 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1493 current_line = _safe_add_trailing_comma(
1494 trailing_comma_safe, delimiter_priority, current_line
1495 )
1497 leaf_priority = bt.delimiters.get(id(leaf))
1498 if leaf_priority == delimiter_priority:
1499 if (
1500 leaf_idx + 1 < len(line.leaves)
1501 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1502 ):
1503 yield from append_comments(line.leaves[leaf_idx + 1])
1505 yield current_line
1506 current_line = Line(
1507 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1508 )
1510 if current_line:
1511 current_line = _safe_add_trailing_comma(
1512 trailing_comma_safe, delimiter_priority, current_line
1513 )
1514 yield current_line
1517@dont_increase_indentation
1518def standalone_comment_split(
1519 line: Line, features: Collection[Feature], mode: Mode
1520) -> Iterator[Line]:
1521 """Split standalone comments from the rest of the line."""
1522 if not line.contains_standalone_comments():
1523 raise CannotSplit("Line does not have any standalone comments")
1525 current_line = Line(
1526 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1527 )
1529 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1530 """Append `leaf` to current line or to new line if appending impossible."""
1531 nonlocal current_line
1532 try:
1533 current_line.append_safe(leaf, preformatted=True)
1534 except ValueError:
1535 yield current_line
1537 current_line = Line(
1538 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1539 )
1540 current_line.append(leaf)
1542 for leaf in line.leaves:
1543 yield from append_to_line(leaf)
1545 for comment_after in line.comments_after(leaf):
1546 yield from append_to_line(comment_after)
1548 if current_line:
1549 yield current_line
1552def _force_standalone_comment_split(line: Line) -> Iterator[Line]:
1553 """Last-resort split at every standalone-comment boundary."""
1554 current_line = Line(
1555 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1556 )
1557 for leaf in line.leaves:
1558 if current_line.leaves and (
1559 leaf.type == STANDALONE_COMMENT or current_line.is_comment
1560 ):
1561 yield current_line
1562 current_line = Line(
1563 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1564 )
1565 current_line.append(leaf, preformatted=True)
1566 if current_line:
1567 yield current_line
1570def _is_parenthesized_lambda_or_ternary(node: LN) -> bool:
1571 """Whether `node` is an atom wrapping a lambda or conditional expression,
1572 looking through any redundant nested parentheses.
1574 As a comprehension's iterable, such an expression must keep at least one pair
1575 of parentheses: without them the trailing `for`/`if` clauses would be parsed
1576 as part of the lambda body (or break the ternary), producing invalid code.
1577 """
1578 while (
1579 node.type == syms.atom
1580 and len(node.children) == 3
1581 and is_lpar_token(node.children[0])
1582 and is_rpar_token(node.children[-1])
1583 ):
1584 middle = node.children[1]
1585 if middle.type in {syms.test, syms.lambdef}:
1586 return True
1587 node = middle
1588 return False
1591def normalize_invisible_parens(
1592 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1593) -> None:
1594 """Make existing optional parentheses invisible or create new ones.
1596 `parens_after` is a set of string leaf values immediately after which parens
1597 should be put.
1599 Standardizes on visible parentheses for single-element tuples, and keeps
1600 existing visible parentheses for other tuples and generator expressions.
1601 """
1602 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1603 if contains_fmt_directive(pc.value, FMT_OFF):
1604 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1605 return
1607 # The multiple context managers grammar has a different pattern, thus this is
1608 # separate from the for-loop below. This possibly wraps them in invisible parens,
1609 # and later will be removed in remove_with_parens when needed.
1610 if node.type == syms.with_stmt:
1611 _maybe_wrap_cms_in_parens(node, mode, features)
1613 check_lpar = False
1614 for index, child in enumerate(list(node.children)):
1615 # Fixes a bug where invisible parens are not properly stripped from
1616 # assignment statements that contain type annotations.
1617 if isinstance(child, Node) and child.type == syms.annassign:
1618 normalize_invisible_parens(
1619 child, parens_after=parens_after, mode=mode, features=features
1620 )
1622 # Fixes a bug where invisible parens are not properly wrapped around
1623 # case blocks.
1624 if isinstance(child, Node) and child.type == syms.case_block:
1625 normalize_invisible_parens(
1626 child, parens_after={"case"}, mode=mode, features=features
1627 )
1629 # Add parentheses around if guards in case blocks
1630 if isinstance(child, Node) and child.type == syms.guard:
1631 normalize_invisible_parens(
1632 child, parens_after={"if"}, mode=mode, features=features
1633 )
1635 # Add parentheses around long tuple unpacking in assignments.
1636 if (
1637 index == 0
1638 and isinstance(child, Node)
1639 and child.type == syms.testlist_star_expr
1640 ):
1641 check_lpar = True
1643 if (
1644 index == 0
1645 and isinstance(child, Node)
1646 and child.type == syms.atom
1647 and node.type == syms.expr_stmt
1648 and not _atom_has_magic_trailing_comma(child, mode)
1649 and not _is_atom_multiline(child)
1650 ):
1651 if maybe_make_parens_invisible_in_atom(
1652 child,
1653 parent=node,
1654 mode=mode,
1655 features=features,
1656 remove_brackets_around_comma=True,
1657 allow_star_expr=True,
1658 ):
1659 wrap_in_parentheses(node, child, visible=False)
1661 if check_lpar:
1662 if (
1663 child.type == syms.atom
1664 and node.type == syms.for_stmt
1665 and isinstance(child.prev_sibling, Leaf)
1666 and child.prev_sibling.type == token.NAME
1667 and child.prev_sibling.value == "for"
1668 ):
1669 if maybe_make_parens_invisible_in_atom(
1670 child,
1671 parent=node,
1672 mode=mode,
1673 features=features,
1674 remove_brackets_around_comma=True,
1675 ):
1676 wrap_in_parentheses(node, child, visible=False)
1677 elif isinstance(child, Node) and node.type == syms.with_stmt:
1678 remove_with_parens(child, node, mode=mode, features=features)
1679 elif (
1680 isinstance(child, Node)
1681 and node.type == syms.yield_expr
1682 and child.type == syms.yield_arg
1683 and Preview.parenthesize_tuple_in_yield in mode
1684 ):
1685 if (
1686 len(child.children) == 1
1687 and child.children[0].type != syms.atom
1688 and is_one_tuple(child.children[0])
1689 ):
1690 wrap_in_parentheses(node, child, visible=True)
1691 elif child.type == syms.atom:
1692 if "in" in parens_after and _is_parenthesized_lambda_or_ternary(child):
1693 # A lambda or conditional expression used as a comprehension's
1694 # iterable must keep at least one pair of parentheses, otherwise
1695 # the trailing `for`/`if` clauses get absorbed into it and the
1696 # code becomes invalid. Any extra nested pairs are redundant, so
1697 # collapse them while keeping exactly one visible pair.
1698 maybe_make_parens_invisible_in_atom(
1699 child, parent=node, mode=mode, features=features
1700 )
1701 opening = child.children[0]
1702 closing = child.children[-1]
1703 if is_lpar_token(opening) and is_rpar_token(closing):
1704 opening.value = "("
1705 closing.value = ")"
1706 elif maybe_make_parens_invisible_in_atom(
1707 child, parent=node, mode=mode, features=features
1708 ):
1709 wrap_in_parentheses(node, child, visible=False)
1710 elif is_one_tuple(child):
1711 wrap_in_parentheses(node, child, visible=True)
1712 elif node.type == syms.import_from:
1713 _normalize_import_from(node, child, index)
1714 break
1715 elif (
1716 index == 1
1717 and child.type == token.STAR
1718 and node.type == syms.except_clause
1719 ):
1720 # In except* (PEP 654), the star is actually part of
1721 # of the keyword. So we need to skip the insertion of
1722 # invisible parentheses to work more precisely.
1723 continue
1725 elif (
1726 isinstance(child, Leaf)
1727 and child.next_sibling is not None
1728 and child.next_sibling.type == token.COLON
1729 and child.value == "case"
1730 ):
1731 # A special patch for "case case:" scenario, the second occurrence
1732 # of case will be not parsed as a Python keyword.
1733 break
1735 elif isinstance(child, Node) and child.type == syms.guard:
1736 # Guard nodes handle their own inner wrapping. Wrapping the guard
1737 # itself can produce invalid output when the case pattern splits.
1738 pass
1740 elif not is_multiline_string(child):
1741 if (
1742 Preview.fix_if_guard_explosion_in_case_statement in mode
1743 and node.type == syms.guard
1744 ):
1745 mock_line = Line(mode=mode)
1746 for leaf in child.leaves():
1747 mock_line.append(leaf)
1748 # If it's a guard AND it's short, we DON'T wrap
1749 if not is_line_short_enough(mock_line, mode=mode):
1750 wrap_in_parentheses(node, child, visible=False)
1751 else:
1752 wrap_in_parentheses(node, child, visible=False)
1754 comma_check = child.type == token.COMMA
1756 check_lpar = isinstance(child, Leaf) and (
1757 child.value in parens_after or comma_check
1758 )
1761def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1762 # "import from" nodes store parentheses directly as part of
1763 # the statement
1764 if is_lpar_token(child):
1765 assert is_rpar_token(parent.children[-1])
1766 # make parentheses invisible
1767 child.value = ""
1768 parent.children[-1].value = ""
1769 elif child.type != token.STAR:
1770 # insert invisible parentheses
1771 parent.insert_child(index, Leaf(token.LPAR, ""))
1772 parent.append_child(Leaf(token.RPAR, ""))
1775def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1776 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1777 if (
1778 node.children[1].type == syms.atom
1779 and node.children[1].children[0].type == token.LPAR
1780 ):
1781 if maybe_make_parens_invisible_in_atom(
1782 node.children[1],
1783 parent=node,
1784 mode=mode,
1785 features=features,
1786 remove_brackets_around_comma=True,
1787 ):
1788 wrap_in_parentheses(node, node.children[1], visible=False)
1790 # Since await is an expression we shouldn't remove
1791 # brackets in cases where this would change
1792 # the AST due to operator precedence.
1793 # Therefore we only aim to remove brackets around
1794 # power nodes that aren't also await expressions themselves.
1795 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1796 # N.B. We've still removed any redundant nested brackets though :)
1797 opening_bracket = cast(Leaf, node.children[1].children[0])
1798 closing_bracket = cast(Leaf, node.children[1].children[-1])
1799 bracket_contents = node.children[1].children[1]
1800 if isinstance(bracket_contents, Node) and (
1801 bracket_contents.type != syms.power
1802 or bracket_contents.children[0].type == token.AWAIT
1803 or any(
1804 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1805 for child in bracket_contents.children
1806 )
1807 ):
1808 ensure_visible(opening_bracket)
1809 ensure_visible(closing_bracket)
1812def _maybe_wrap_cms_in_parens(
1813 node: Node, mode: Mode, features: Collection[Feature]
1814) -> None:
1815 """When enabled and safe, wrap the multiple context managers in invisible parens.
1817 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1818 """
1819 if (
1820 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1821 or len(node.children) <= 2
1822 # If it's an atom, it's already wrapped in parens.
1823 or node.children[1].type == syms.atom
1824 ):
1825 return
1826 colon_index: int | None = None
1827 for i in range(2, len(node.children)):
1828 if node.children[i].type == token.COLON:
1829 colon_index = i
1830 break
1831 if colon_index is not None:
1832 lpar = Leaf(token.LPAR, "")
1833 rpar = Leaf(token.RPAR, "")
1834 context_managers = node.children[1:colon_index]
1835 for child in context_managers:
1836 child.remove()
1837 # After wrapping, the with_stmt will look like this:
1838 # with_stmt
1839 # NAME 'with'
1840 # atom
1841 # LPAR ''
1842 # testlist_gexp
1843 # ... <-- context_managers
1844 # /testlist_gexp
1845 # RPAR ''
1846 # /atom
1847 # COLON ':'
1848 new_child = Node(
1849 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1850 )
1851 node.insert_child(1, new_child)
1854def remove_with_parens(
1855 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1856) -> None:
1857 """Recursively hide optional parens in `with` statements."""
1858 # Removing all unnecessary parentheses in with statements in one pass is a tad
1859 # complex as different variations of bracketed statements result in pretty
1860 # different parse trees:
1861 #
1862 # with (open("file")) as f: # this is an asexpr_test
1863 # ...
1864 #
1865 # with (open("file") as f): # this is an atom containing an
1866 # ... # asexpr_test
1867 #
1868 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1869 # ... # asexpr_test
1870 #
1871 # with (open("file") as f, open("file") as f): # an atom containing a
1872 # ... # testlist_gexp which then
1873 # # contains multiple asexpr_test(s)
1874 if node.type == syms.atom:
1875 if maybe_make_parens_invisible_in_atom(
1876 node,
1877 parent=parent,
1878 mode=mode,
1879 features=features,
1880 remove_brackets_around_comma=True,
1881 ):
1882 wrap_in_parentheses(parent, node, visible=False)
1883 if isinstance(node.children[1], Node):
1884 remove_with_parens(node.children[1], node, mode=mode, features=features)
1885 elif node.type == syms.testlist_gexp:
1886 for child in node.children:
1887 if isinstance(child, Node):
1888 remove_with_parens(child, node, mode=mode, features=features)
1889 elif node.type == syms.asexpr_test and not any(
1890 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1891 ):
1892 if maybe_make_parens_invisible_in_atom(
1893 node.children[0],
1894 parent=node,
1895 mode=mode,
1896 features=features,
1897 remove_brackets_around_comma=True,
1898 ):
1899 wrap_in_parentheses(node, node.children[0], visible=False)
1902def _atom_has_magic_trailing_comma(node: LN, mode: Mode) -> bool:
1903 """Check if an atom node has a magic trailing comma.
1905 Returns True for single-element tuples with trailing commas like (a,),
1906 which should be preserved to maintain their tuple type.
1907 """
1908 if not mode.magic_trailing_comma:
1909 return False
1911 return is_one_tuple(node)
1914def _is_atom_multiline(node: LN) -> bool:
1915 """Check if an atom node is multiline (indicating intentional formatting)."""
1916 if not isinstance(node, Node) or len(node.children) < 3:
1917 return False
1919 # Check the middle child (between LPAR and RPAR) for newlines in its subtree
1920 # The first child's prefix contains blank lines/comments before the opening paren
1921 middle = node.children[1]
1922 for child in middle.pre_order():
1923 if isinstance(child, Leaf) and "\n" in child.prefix:
1924 return True
1926 return False
1929def maybe_make_parens_invisible_in_atom(
1930 node: LN,
1931 parent: LN,
1932 mode: Mode,
1933 features: Collection[Feature],
1934 remove_brackets_around_comma: bool = False,
1935 allow_star_expr: bool = False,
1936) -> bool:
1937 """If it's safe, make the parens in the atom `node` invisible, recursively.
1938 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1939 as they are redundant.
1941 Returns whether the node should itself be wrapped in invisible parentheses.
1942 """
1943 if (
1944 node.type not in (syms.atom, syms.expr)
1945 or is_empty_tuple(node)
1946 or is_one_tuple(node)
1947 or (is_tuple(node) and parent.type == syms.asexpr_test)
1948 or (
1949 is_tuple(node)
1950 and parent.type == syms.with_stmt
1951 and has_sibling_with_type(node, token.COMMA)
1952 )
1953 or (is_yield(node) and parent.type != syms.expr_stmt)
1954 or (
1955 # This condition tries to prevent removing non-optional brackets
1956 # around a tuple, however, can be a bit overzealous so we provide
1957 # and option to skip this check for `for` and `with` statements.
1958 not remove_brackets_around_comma
1959 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1960 # Remove parentheses around multiple exception types in except and
1961 # except* without as. See PEP 758 for details.
1962 and not (
1963 Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1964 # is a tuple
1965 and is_tuple(node)
1966 # has a parent node
1967 and node.parent is not None
1968 # parent is an except clause
1969 and node.parent.type == syms.except_clause
1970 # is not immediately followed by as clause
1971 and not (
1972 node.next_sibling is not None
1973 and is_name_token(node.next_sibling)
1974 and node.next_sibling.value == "as"
1975 )
1976 )
1977 )
1978 or is_tuple_containing_walrus(node)
1979 or (not allow_star_expr and is_tuple_containing_star(node))
1980 or is_generator(node)
1981 ):
1982 return False
1984 if is_walrus_assignment(node):
1985 if parent.type in [
1986 syms.annassign,
1987 syms.expr_stmt,
1988 syms.assert_stmt,
1989 syms.return_stmt,
1990 syms.yield_arg,
1991 syms.yield_expr,
1992 syms.except_clause,
1993 syms.funcdef,
1994 syms.with_stmt,
1995 syms.testlist_gexp,
1996 syms.tname,
1997 # these ones aren't useful to end users, but they do please fuzzers
1998 syms.for_stmt,
1999 syms.del_stmt,
2000 syms.for_stmt,
2001 ]:
2002 return False
2004 first = node.children[0]
2005 last = node.children[-1]
2006 if is_lpar_token(first) and is_rpar_token(last):
2007 middle = node.children[1]
2008 # make parentheses invisible
2009 if (
2010 # If the prefix of `middle` includes a type comment with
2011 # ignore annotation, then we do not remove the parentheses
2012 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
2013 ):
2014 first.value = ""
2015 last.value = ""
2016 maybe_make_parens_invisible_in_atom(
2017 middle,
2018 parent=parent,
2019 mode=mode,
2020 features=features,
2021 remove_brackets_around_comma=remove_brackets_around_comma,
2022 )
2024 if is_atom_with_invisible_parens(middle):
2025 # Strip the invisible parens from `middle` by replacing
2026 # it with the child in-between the invisible parens
2027 middle.replace(middle.children[1])
2029 if middle.children[0].prefix.strip():
2030 # Preserve comments before first paren
2031 middle.children[1].prefix = (
2032 middle.children[0].prefix + middle.children[1].prefix
2033 )
2035 if middle.children[-1].prefix.strip():
2036 # Preserve comments before last paren
2037 last.prefix = middle.children[-1].prefix + last.prefix
2039 return False
2041 return True
2044def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
2045 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
2047 if not (opening_bracket.parent and opening_bracket.value in "[{("):
2048 return False
2050 # We're essentially checking if the body is delimited by commas and there's more
2051 # than one of them (we're excluding the trailing comma and if the delimiter priority
2052 # is still commas, that means there's more).
2053 exclude = set()
2054 trailing_comma = False
2055 try:
2056 last_leaf = line.leaves[-1]
2057 if last_leaf.type == token.COMMA:
2058 trailing_comma = True
2059 exclude.add(id(last_leaf))
2060 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
2061 except (IndexError, ValueError):
2062 return False
2064 return max_priority == COMMA_PRIORITY and (
2065 (line.mode.magic_trailing_comma and trailing_comma)
2066 # always explode imports
2067 or opening_bracket.parent.type in {syms.atom, syms.import_from}
2068 )
2071def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
2072 """Generate sets of closing bracket IDs that should be omitted in a RHS.
2074 Brackets can be omitted if the entire trailer up to and including
2075 a preceding closing bracket fits in one line.
2077 Yielded sets are cumulative (contain results of previous yields, too). First
2078 set is empty, unless the line should explode, in which case bracket pairs until
2079 the one that needs to explode are omitted.
2080 """
2082 omit: set[LeafID] = set()
2083 if not line.magic_trailing_comma:
2084 yield omit
2086 length = 4 * line.depth
2087 opening_bracket: Leaf | None = None
2088 closing_bracket: Leaf | None = None
2089 inner_brackets: set[LeafID] = set()
2090 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
2091 length += leaf_length
2092 if length > line_length:
2093 break
2095 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
2096 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
2097 break
2099 if opening_bracket:
2100 if leaf is opening_bracket:
2101 opening_bracket = None
2102 elif leaf.type in CLOSING_BRACKETS:
2103 prev = line.leaves[index - 1] if index > 0 else None
2104 if (
2105 prev
2106 and prev.type == token.COMMA
2107 and leaf.opening_bracket is not None
2108 and not is_one_sequence_between(
2109 leaf.opening_bracket, leaf, line.leaves
2110 )
2111 ):
2112 # Never omit bracket pairs with trailing commas.
2113 # We need to explode on those.
2114 break
2116 inner_brackets.add(id(leaf))
2117 elif leaf.type in CLOSING_BRACKETS:
2118 prev = line.leaves[index - 1] if index > 0 else None
2119 if prev and prev.type in OPENING_BRACKETS:
2120 # Empty brackets would fail a split so treat them as "inner"
2121 # brackets (e.g. only add them to the `omit` set if another
2122 # pair of brackets was good enough.
2123 inner_brackets.add(id(leaf))
2124 continue
2126 if closing_bracket:
2127 omit.add(id(closing_bracket))
2128 omit.update(inner_brackets)
2129 inner_brackets.clear()
2130 yield omit
2132 if (
2133 prev
2134 and prev.type == token.COMMA
2135 and leaf.opening_bracket is not None
2136 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
2137 ):
2138 # Never omit bracket pairs with trailing commas.
2139 # We need to explode on those.
2140 break
2142 if leaf.value:
2143 opening_bracket = leaf.opening_bracket
2144 closing_bracket = leaf
2147def _over_length_only_due_to_subscript_comment(line: Line, mode: Mode) -> bool:
2148 """Return True if `line` only exceeds `mode.line_length` because of an inline
2149 comment attached to a subscript opening bracket (`[`).
2151 This is the shape produced by the original of the issue #4733 reproducer:
2152 a comment inside the annotation's subscript brackets renders at the end of
2153 the head line after Black splits the statement, pushing it past the limit.
2154 Taking the FORCE_OPTIONAL_PARENTHESES "second opinion" in that case wraps
2155 the annotation in extra parens and migrates the comment outside the
2156 subscript, which then oscillates on the next formatter pass.
2157 """
2158 if not line.leaves:
2159 return False
2160 # The over-length must be caused entirely by a trailing comment.
2161 indent = " " * line.depth
2162 leaves_iter = iter(line.leaves)
2163 first = next(leaves_iter)
2164 text_without_comments = f"{first.prefix}{indent}{first.value}"
2165 text_without_comments += "".join(str(leaf) for leaf in leaves_iter)
2166 if str_width(text_without_comments) > mode.line_length:
2167 return False
2168 # And the comment must be attached to a subscript opening bracket.
2169 for leaf_id, comments in line.comments.items():
2170 if not comments:
2171 continue
2172 leaf = next((lf for lf in line.leaves if id(lf) == leaf_id), None)
2173 if leaf is None or leaf.type != token.LSQB:
2174 return False
2175 return True
2178def run_transformer(
2179 line: Line,
2180 transform: Transformer,
2181 mode: Mode,
2182 features: Collection[Feature],
2183 *,
2184 line_str: str = "",
2185) -> list[Line]:
2186 if not line_str:
2187 line_str = line_to_string(line)
2188 result: list[Line] = []
2189 for transformed_line in transform(line, features, mode):
2190 if str(transformed_line).strip("\n") == line_str:
2191 raise CannotTransform("Line transformer returned an unchanged result")
2193 result.extend(transform_line(transformed_line, mode=mode, features=features))
2195 features_set = set(features)
2196 if (
2197 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
2198 or transform.__class__.__name__ != "rhs"
2199 or not line.bracket_tracker.invisible
2200 or any(bracket.value for bracket in line.bracket_tracker.invisible)
2201 or line.contains_multiline_strings()
2202 or result[0].contains_uncollapsable_type_comments()
2203 or result[0].contains_unsplittable_type_ignore()
2204 or is_line_short_enough(result[0], mode=mode)
2205 # result[0] only exceeds the length because of a comment attached to a
2206 # subscript opening bracket. Taking the FORCE_OPTIONAL_PARENTHESES
2207 # "second opinion" wraps the annotation in extra invisible parens and
2208 # migrates the comment outside the subscript, which then oscillates with
2209 # a deeper-bracket split on the next formatter pass (issue #4733).
2210 or _over_length_only_due_to_subscript_comment(result[0], mode)
2211 # If any leaves have no parents (which _can_ occur since
2212 # `transform(line)` potentially destroys the line's underlying node
2213 # structure), then we can't proceed. Doing so would cause the below
2214 # call to `append_leaves()` to fail.
2215 or any(leaf.parent is None for leaf in line.leaves)
2216 ):
2217 return result
2219 line_copy = line.clone()
2220 append_leaves(line_copy, line, line.leaves)
2221 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
2222 second_opinion = run_transformer(
2223 line_copy, transform, mode, features_fop, line_str=line_str
2224 )
2225 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
2226 result = second_opinion
2227 return result