Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 COMPARATOR_PRIORITY,
16 DOT_PRIORITY,
17 STRING_PRIORITY,
18 get_leaves_inside_matching_brackets,
19 max_delimiter_priority_in_atom,
20)
21from black.comments import (
22 FMT_OFF,
23 FMT_ON,
24 contains_fmt_directive,
25 generate_comments,
26 list_comments,
27)
28from black.lines import (
29 Line,
30 RHSResult,
31 append_leaves,
32 can_be_split,
33 can_omit_invisible_parens,
34 is_line_short_enough,
35 line_to_string,
36)
37from black.mode import Feature, Mode, Preview
38from black.nodes import (
39 ASSIGNMENTS,
40 BRACKETS,
41 CLOSING_BRACKETS,
42 OPENING_BRACKETS,
43 STANDALONE_COMMENT,
44 STATEMENT,
45 WHITESPACE,
46 Visitor,
47 ensure_visible,
48 fstring_tstring_to_string,
49 get_annotation_type,
50 has_sibling_with_type,
51 is_arith_like,
52 is_async_stmt_or_funcdef,
53 is_atom_with_invisible_parens,
54 is_docstring,
55 is_empty_tuple,
56 is_generator,
57 is_lpar_token,
58 is_multiline_string,
59 is_name_token,
60 is_one_sequence_between,
61 is_one_tuple,
62 is_parent_function_or_class,
63 is_part_of_annotation,
64 is_rpar_token,
65 is_stub_body,
66 is_stub_suite,
67 is_tuple,
68 is_tuple_containing_star,
69 is_tuple_containing_walrus,
70 is_type_ignore_comment_string,
71 is_vararg,
72 is_walrus_assignment,
73 is_yield,
74 syms,
75 wrap_in_parentheses,
76)
77from black.numerics import normalize_numeric_literal
78from black.strings import (
79 fix_multiline_docstring,
80 get_string_prefix,
81 normalize_string_prefix,
82 normalize_string_quotes,
83 normalize_unicode_escape_sequences,
84 str_width,
85)
86from black.trans import (
87 CannotTransform,
88 StringMerger,
89 StringParenStripper,
90 StringParenWrapper,
91 StringSplitter,
92 Transformer,
93 hug_power_op,
94)
95from blib2to3.pgen2 import token
96from blib2to3.pytree import Leaf, Node
98# types
99LeafID = int
100LN = Union[Leaf, Node]
103class CannotSplit(CannotTransform):
104 """A readable split that fits the allotted line length is impossible."""
107# This isn't a dataclass because @dataclass + Generic breaks mypyc.
108# See also https://github.com/mypyc/mypyc/issues/827.
109class LineGenerator(Visitor[Line]):
110 """Generates reformatted Line objects. Empty lines are not emitted.
112 Note: destroys the tree it's visiting by mutating prefixes of its leaves
113 in ways that will no longer stringify to valid Python code on the tree.
114 """
116 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
117 self.mode = mode
118 self.features = features
119 self.current_line: Line
120 self.__post_init__()
122 def line(self, indent: int = 0) -> Iterator[Line]:
123 """Generate a line.
125 If the line is empty, only emit if it makes sense.
126 If the line is too long, split it first and then generate.
128 If any lines were generated, set up a new current_line.
129 """
130 if not self.current_line:
131 self.current_line.depth += indent
132 return # Line is empty, don't emit. Creating a new one unnecessary.
134 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
135 self.current_line.leaves[0]
136 ):
137 # Special case for async def/for/with statements. `visit_async_stmt`
138 # adds an `ASYNC` leaf then visits the child def/for/with statement
139 # nodes. Line yields from those nodes shouldn't treat the former
140 # `ASYNC` leaf as a complete line.
141 return
143 complete_line = self.current_line
144 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
145 yield complete_line
147 def visit_default(self, node: LN) -> Iterator[Line]:
148 """Default `visit_*()` implementation. Recurses to children of `node`."""
149 if isinstance(node, Leaf):
150 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
151 for comment in generate_comments(node, mode=self.mode):
152 if any_open_brackets:
153 # any comment within brackets is subject to splitting
154 self.current_line.append(comment)
155 elif comment.type == token.COMMENT:
156 # regular trailing comment
157 self.current_line.append(comment)
158 yield from self.line()
160 else:
161 # regular standalone comment
162 yield from self.line()
164 self.current_line.append(comment)
165 yield from self.line()
167 if any_open_brackets:
168 node.prefix = ""
169 if node.type not in WHITESPACE:
170 self.current_line.append(node)
171 yield from super().visit_default(node)
173 def visit_test(self, node: Node) -> Iterator[Line]:
174 """Visit an `x if y else z` test"""
176 already_parenthesized = (
177 node.prev_sibling and node.prev_sibling.type == token.LPAR
178 )
180 if not already_parenthesized:
181 # Similar to logic in wrap_in_parentheses
182 lpar = Leaf(token.LPAR, "")
183 rpar = Leaf(token.RPAR, "")
184 prefix = node.prefix
185 node.prefix = ""
186 lpar.prefix = prefix
187 node.insert_child(0, lpar)
188 node.append_child(rpar)
190 yield from self.visit_default(node)
192 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
193 """Increase indentation level, maybe yield a line."""
194 # In blib2to3 INDENT never holds comments.
195 yield from self.line(+1)
196 yield from self.visit_default(node)
198 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
199 """Decrease indentation level, maybe yield a line."""
200 # The current line might still wait for trailing comments. At DEDENT time
201 # there won't be any (they would be prefixes on the preceding NEWLINE).
202 # Emit the line then.
203 yield from self.line()
205 # While DEDENT has no value, its prefix may contain standalone comments
206 # that belong to the current indentation level. Get 'em.
207 yield from self.visit_default(node)
209 # Finally, emit the dedent.
210 yield from self.line(-1)
212 def visit_stmt(
213 self, node: Node, keywords: set[str], parens: set[str]
214 ) -> Iterator[Line]:
215 """Visit a statement.
217 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
218 `def`, `with`, `class`, `assert`, and assignments.
220 The relevant Python language `keywords` for a given statement will be
221 NAME leaves within it. This methods puts those on a separate line.
223 `parens` holds a set of string leaf values immediately after which
224 invisible parens should be put.
225 """
226 normalize_invisible_parens(
227 node, parens_after=parens, mode=self.mode, features=self.features
228 )
229 for child in node.children:
230 if is_name_token(child) and child.value in keywords:
231 yield from self.line()
233 yield from self.visit(child)
235 def visit_typeparams(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[0].prefix = ""
239 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
240 yield from self.visit_default(node)
241 node.children[1].prefix = ""
243 def visit_paramspec(self, node: Node) -> Iterator[Line]:
244 yield from self.visit_default(node)
245 node.children[1].prefix = ""
247 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
248 if Preview.wrap_long_dict_values_in_parens in self.mode:
249 for i, child in enumerate(node.children):
250 if i == 0:
251 continue
252 if node.children[i - 1].type == token.COLON:
253 if (
254 child.type == syms.atom
255 and child.children[0].type in OPENING_BRACKETS
256 and not is_walrus_assignment(child)
257 ):
258 maybe_make_parens_invisible_in_atom(
259 child,
260 parent=node,
261 mode=self.mode,
262 features=self.features,
263 remove_brackets_around_comma=False,
264 )
265 else:
266 wrap_in_parentheses(node, child, visible=False)
267 yield from self.visit_default(node)
269 def visit_funcdef(self, node: Node) -> Iterator[Line]:
270 """Visit function definition."""
271 yield from self.line()
273 # Remove redundant brackets around return type annotation.
274 is_return_annotation = False
275 for child in node.children:
276 if child.type == token.RARROW:
277 is_return_annotation = True
278 elif is_return_annotation:
279 if child.type == syms.atom and child.children[0].type == token.LPAR:
280 if maybe_make_parens_invisible_in_atom(
281 child,
282 parent=node,
283 mode=self.mode,
284 features=self.features,
285 remove_brackets_around_comma=False,
286 ):
287 wrap_in_parentheses(node, child, visible=False)
288 else:
289 wrap_in_parentheses(node, child, visible=False)
290 is_return_annotation = False
292 for child in node.children:
293 yield from self.visit(child)
295 def visit_match_case(self, node: Node) -> Iterator[Line]:
296 """Visit either a match or case statement."""
297 normalize_invisible_parens(
298 node, parens_after=set(), mode=self.mode, features=self.features
299 )
301 yield from self.line()
302 for child in node.children:
303 yield from self.visit(child)
305 def visit_suite(self, node: Node) -> Iterator[Line]:
306 """Visit a suite."""
307 if is_stub_suite(node):
308 yield from self.visit(node.children[2])
309 else:
310 yield from self.visit_default(node)
312 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
313 """Visit a statement without nested statements."""
314 prev_type: int | None = None
315 for child in node.children:
316 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
317 wrap_in_parentheses(node, child, visible=False)
318 prev_type = child.type
320 if node.parent and node.parent.type in STATEMENT:
321 if is_parent_function_or_class(node) and is_stub_body(node):
322 yield from self.visit_default(node)
323 else:
324 yield from self.line(+1)
325 yield from self.visit_default(node)
326 yield from self.line(-1)
328 else:
329 if node.parent and is_stub_suite(node.parent):
330 node.prefix = ""
331 yield from self.visit_default(node)
332 return
333 yield from self.line()
334 yield from self.visit_default(node)
336 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
337 """Visit `async def`, `async for`, `async with`."""
338 yield from self.line()
340 children = iter(node.children)
341 for child in children:
342 yield from self.visit(child)
344 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
345 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
346 # line.
347 break
349 internal_stmt = next(children)
350 yield from self.visit(internal_stmt)
352 def visit_decorators(self, node: Node) -> Iterator[Line]:
353 """Visit decorators."""
354 for child in node.children:
355 yield from self.line()
356 yield from self.visit(child)
358 def visit_power(self, node: Node) -> Iterator[Line]:
359 for idx, leaf in enumerate(node.children[:-1]):
360 next_leaf = node.children[idx + 1]
362 if not isinstance(leaf, Leaf):
363 continue
365 value = leaf.value.lower()
366 if (
367 leaf.type == token.NUMBER
368 and next_leaf.type == syms.trailer
369 # Ensure that we are in an attribute trailer
370 and next_leaf.children[0].type == token.DOT
371 # It shouldn't wrap hexadecimal, binary and octal literals
372 and not value.startswith(("0x", "0b", "0o"))
373 # It shouldn't wrap complex literals
374 and "j" not in value
375 ):
376 wrap_in_parentheses(node, leaf)
378 remove_await_parens(node, mode=self.mode, features=self.features)
380 yield from self.visit_default(node)
382 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
383 """Remove a semicolon and put the other statement on a separate line."""
384 yield from self.line()
386 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
387 """End of file. Process outstanding comments and end with a newline."""
388 yield from self.visit_default(leaf)
389 yield from self.line()
391 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
392 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
393 if not any_open_brackets:
394 yield from self.line()
395 # STANDALONE_COMMENT nodes created by our special handling in
396 # normalize_fmt_off for comment-only blocks have fmt:off as the first
397 # line and fmt:on as the last line (each directive on its own line,
398 # not embedded in other text). These should be appended directly
399 # without calling visit_default, which would process their prefix and
400 # lose indentation. Normal STANDALONE_COMMENT nodes go through
401 # visit_default.
402 value = leaf.value
403 lines = value.splitlines()
404 is_fmt_off_block = (
405 len(lines) >= 2
406 and contains_fmt_directive(lines[0], FMT_OFF)
407 and contains_fmt_directive(lines[-1], FMT_ON)
408 )
409 if is_fmt_off_block:
410 # This is a fmt:off/on block from normalize_fmt_off - we still need
411 # to process any prefix comments (like markdown comments) but append
412 # the fmt block itself directly to preserve its formatting
414 # Only process prefix comments if there actually is a prefix with comments
415 if leaf.prefix and any(
416 line.strip().startswith("#")
417 and not contains_fmt_directive(line.strip())
418 for line in leaf.prefix.split("\n")
419 ):
420 for comment in generate_comments(leaf, mode=self.mode):
421 yield from self.line()
422 self.current_line.append(comment)
423 yield from self.line()
424 # Clear the prefix since we've processed it as comments above
425 leaf.prefix = ""
427 self.current_line.append(leaf)
428 if not any_open_brackets:
429 yield from self.line()
430 else:
431 # Normal standalone comment - process through visit_default
432 yield from self.visit_default(leaf)
434 def visit_factor(self, node: Node) -> Iterator[Line]:
435 """Force parentheses between a unary op and a binary power:
437 -2 ** 8 -> -(2 ** 8)
438 """
439 _operator, operand = node.children
440 if (
441 operand.type == syms.power
442 and len(operand.children) == 3
443 and operand.children[1].type == token.DOUBLESTAR
444 ):
445 lpar = Leaf(token.LPAR, "(")
446 rpar = Leaf(token.RPAR, ")")
447 index = operand.remove() or 0
448 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
449 yield from self.visit_default(node)
451 def visit_tname(self, node: Node) -> Iterator[Line]:
452 """
453 Add potential parentheses around types in function parameter lists to be made
454 into real parentheses in case the type hint is too long to fit on a line
455 Examples:
456 def foo(a: int, b: float = 7): ...
458 ->
460 def foo(a: (int), b: (float) = 7): ...
461 """
462 if len(node.children) == 3 and maybe_make_parens_invisible_in_atom(
463 node.children[2], parent=node, mode=self.mode, features=self.features
464 ):
465 wrap_in_parentheses(node, node.children[2], visible=False)
467 yield from self.visit_default(node)
469 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
470 normalize_unicode_escape_sequences(leaf)
472 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
473 # We're ignoring docstrings with backslash newline escapes because changing
474 # indentation of those changes the AST representation of the code.
475 if self.mode.string_normalization:
476 docstring = normalize_string_prefix(leaf.value)
477 # We handle string normalization at the end of this method, but since
478 # what we do right now acts differently depending on quote style (ex.
479 # see padding logic below), there's a possibility for unstable
480 # formatting. To avoid a situation where this function formats a
481 # docstring differently on the second pass, normalize it early.
482 docstring = normalize_string_quotes(docstring)
483 else:
484 docstring = leaf.value
485 prefix = get_string_prefix(docstring)
486 docstring = docstring[len(prefix) :] # Remove the prefix
487 quote_char = docstring[0]
488 # A natural way to remove the outer quotes is to do:
489 # docstring = docstring.strip(quote_char)
490 # but that breaks on """""x""" (which is '""x').
491 # So we actually need to remove the first character and the next two
492 # characters but only if they are the same as the first.
493 quote_len = 1 if docstring[1] != quote_char else 3
494 docstring = docstring[quote_len:-quote_len]
495 docstring_started_empty = not docstring
496 indent = " " * 4 * self.current_line.depth
498 if is_multiline_string(leaf):
499 docstring = fix_multiline_docstring(docstring, indent)
500 else:
501 docstring = docstring.strip()
503 has_trailing_backslash = False
504 if docstring:
505 # Add some padding if the docstring starts / ends with a quote mark.
506 if docstring[0] == quote_char:
507 docstring = " " + docstring
508 if docstring[-1] == quote_char:
509 docstring += " "
510 if docstring[-1] == "\\":
511 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
512 if backslash_count % 2:
513 # Odd number of tailing backslashes, add some padding to
514 # avoid escaping the closing string quote.
515 docstring += " "
516 has_trailing_backslash = True
517 elif not docstring_started_empty:
518 docstring = " "
520 # We could enforce triple quotes at this point.
521 quote = quote_char * quote_len
523 # It's invalid to put closing single-character quotes on a new line.
524 if quote_len == 3:
525 # We need to find the length of the last line of the docstring
526 # to find if we can add the closing quotes to the line without
527 # exceeding the maximum line length.
528 # If docstring is one line, we don't put the closing quotes on a
529 # separate line because it looks ugly (#3320).
530 lines = docstring.splitlines()
531 last_line_length = len(lines[-1]) if docstring else 0
533 # If adding closing quotes would cause the last line to exceed
534 # the maximum line length, and the closing quote is not
535 # prefixed by a newline then put a line break before
536 # the closing quotes
537 if (
538 len(lines) > 1
539 and last_line_length + quote_len > self.mode.line_length
540 and len(indent) + quote_len <= self.mode.line_length
541 and not has_trailing_backslash
542 ):
543 if leaf.value[-1 - quote_len] == "\n":
544 leaf.value = prefix + quote + docstring + quote
545 else:
546 leaf.value = prefix + quote + docstring + "\n" + indent + quote
547 else:
548 leaf.value = prefix + quote + docstring + quote
549 else:
550 leaf.value = prefix + quote + docstring + quote
552 if self.mode.string_normalization and leaf.type == token.STRING:
553 leaf.value = normalize_string_prefix(leaf.value)
554 leaf.value = normalize_string_quotes(leaf.value)
555 yield from self.visit_default(leaf)
557 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
558 normalize_numeric_literal(leaf)
559 yield from self.visit_default(leaf)
561 def visit_atom(self, node: Node) -> Iterator[Line]:
562 """Visit any atom"""
563 if len(node.children) == 3:
564 first = node.children[0]
565 last = node.children[-1]
566 if (first.type == token.LSQB and last.type == token.RSQB) or (
567 first.type == token.LBRACE and last.type == token.RBRACE
568 ):
569 # Lists or sets of one item
570 maybe_make_parens_invisible_in_atom(
571 node.children[1],
572 parent=node,
573 mode=self.mode,
574 features=self.features,
575 )
577 yield from self.visit_default(node)
579 def visit_fstring(self, node: Node) -> Iterator[Line]:
580 # If the fstring was converted to a STANDALONE_COMMENT by
581 # normalize_fmt_off (e.g. it was inside a # fmt: off block),
582 # skip the fstring-to-string conversion and just visit normally.
583 if any(child.type == STANDALONE_COMMENT for child in node.children):
584 yield from self.visit_default(node)
585 return
586 # currently we don't want to format and split f-strings at all.
587 string_leaf = fstring_tstring_to_string(node)
588 node.replace(string_leaf)
589 if "\\" in string_leaf.value and any(
590 "\\" in str(child)
591 for child in node.children
592 if child.type == syms.fstring_replacement_field
593 ):
594 # string normalization doesn't account for nested quotes,
595 # causing breakages. skip normalization when nested quotes exist
596 yield from self.visit_default(string_leaf)
597 return
598 yield from self.visit_STRING(string_leaf)
600 def visit_tstring(self, node: Node) -> Iterator[Line]:
601 # If the tstring was converted to a STANDALONE_COMMENT by
602 # normalize_fmt_off, skip the conversion and just visit normally.
603 if any(child.type == STANDALONE_COMMENT for child in node.children):
604 yield from self.visit_default(node)
605 return
606 # currently we don't want to format and split t-strings at all.
607 string_leaf = fstring_tstring_to_string(node)
608 node.replace(string_leaf)
609 if "\\" in string_leaf.value and any(
610 "\\" in str(child)
611 for child in node.children
612 if child.type == syms.fstring_replacement_field
613 ):
614 # string normalization doesn't account for nested quotes,
615 # causing breakages. skip normalization when nested quotes exist
616 yield from self.visit_default(string_leaf)
617 return
618 yield from self.visit_STRING(string_leaf)
620 # TODO: Uncomment Implementation to format f-string children
621 # fstring_start = node.children[0]
622 # fstring_end = node.children[-1]
623 # assert isinstance(fstring_start, Leaf)
624 # assert isinstance(fstring_end, Leaf)
626 # quote_char = fstring_end.value[0]
627 # quote_idx = fstring_start.value.index(quote_char)
628 # prefix, quote = (
629 # fstring_start.value[:quote_idx],
630 # fstring_start.value[quote_idx:]
631 # )
633 # if not is_docstring(node, self.mode):
634 # prefix = normalize_string_prefix(prefix)
636 # assert quote == fstring_end.value
638 # is_raw_fstring = "r" in prefix or "R" in prefix
639 # middles = [
640 # leaf
641 # for leaf in node.leaves()
642 # if leaf.type == token.FSTRING_MIDDLE
643 # ]
645 # if self.mode.string_normalization:
646 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
648 # fstring_start.value = prefix + quote
649 # fstring_end.value = quote
651 # yield from self.visit_default(node)
653 def visit_comp_for(self, node: Node) -> Iterator[Line]:
654 if Preview.wrap_comprehension_in in self.mode:
655 normalize_invisible_parens(
656 node, parens_after={"in"}, mode=self.mode, features=self.features
657 )
658 yield from self.visit_default(node)
660 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
661 yield from self.visit_comp_for(node)
663 def __post_init__(self) -> None:
664 """You are in a twisty little maze of passages."""
665 self.current_line = Line(mode=self.mode)
667 v = self.visit_stmt
668 Ø: set[str] = set()
669 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
670 self.visit_if_stmt = partial(
671 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
672 )
673 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
674 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
675 self.visit_try_stmt = partial(
676 v, keywords={"try", "except", "else", "finally"}, parens=Ø
677 )
678 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
679 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
680 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
682 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
683 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
684 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
685 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
686 self.visit_async_funcdef = self.visit_async_stmt
687 self.visit_decorated = self.visit_decorators
689 # PEP 634
690 self.visit_match_stmt = self.visit_match_case
691 self.visit_case_block = self.visit_match_case
692 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
695# Remove when `simplify_power_operator_hugging` becomes stable.
696def _hugging_power_ops_line_to_string(
697 line: Line,
698 features: Collection[Feature],
699 mode: Mode,
700) -> str | None:
701 try:
702 return line_to_string(next(hug_power_op(line, features, mode)))
703 except CannotTransform:
704 return None
707def transform_line(
708 line: Line, mode: Mode, features: Collection[Feature] = ()
709) -> Iterator[Line]:
710 """Transform a `line`, potentially splitting it into many lines.
712 They should fit in the allotted `line_length` but might not be able to.
714 `features` are syntactical features that may be used in the output.
715 """
716 if line.is_comment:
717 yield line
718 return
720 line_str = line_to_string(line)
722 if Preview.simplify_power_operator_hugging in mode:
723 line_str_hugging_power_ops = line_str
724 else:
725 # We need the line string when power operators are hugging to determine if we
726 # should split the line. Default to line_str, if no power operator are present
727 # on the line.
728 line_str_hugging_power_ops = (
729 _hugging_power_ops_line_to_string(line, features, mode) or line_str
730 )
732 ll = mode.line_length
733 sn = mode.string_normalization
734 string_merge = StringMerger(ll, sn)
735 string_paren_strip = StringParenStripper(ll, sn)
736 string_split = StringSplitter(ll, sn)
737 string_paren_wrap = StringParenWrapper(ll, sn)
739 transformers: list[Transformer]
740 if (
741 not line.contains_uncollapsable_type_comments()
742 and not line.should_split_rhs
743 and not line.magic_trailing_comma
744 and (
745 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
746 or line.contains_unsplittable_type_ignore()
747 )
748 and not (line.inside_brackets and line.contains_standalone_comments())
749 and not line.contains_implicit_multiline_string_with_comments()
750 ):
751 # Only apply basic string preprocessing, since lines shouldn't be split here.
752 if Preview.string_processing in mode:
753 transformers = [string_merge, string_paren_strip]
754 else:
755 transformers = []
756 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
757 transformers = [left_hand_split]
758 else:
760 def _rhs(
761 self: object, line: Line, features: Collection[Feature], mode: Mode
762 ) -> Iterator[Line]:
763 """Wraps calls to `right_hand_split`.
765 The calls increasingly `omit` right-hand trailers (bracket pairs with
766 content), meaning the trailers get glued together to split on another
767 bracket pair instead.
768 """
769 for omit in generate_trailers_to_omit(line, mode.line_length):
770 lines = list(right_hand_split(line, mode, features, omit=omit))
771 # Note: this check is only able to figure out if the first line of the
772 # *current* transformation fits in the line length. This is true only
773 # for simple cases. All others require running more transforms via
774 # `transform_line()`. This check doesn't know if those would succeed.
775 if is_line_short_enough(lines[0], mode=mode) or (
776 omit and _over_length_only_due_to_subscript_comment(lines[0], mode)
777 ):
778 yield from lines
779 return
781 # All splits failed, best effort split with no omits.
782 # This mostly happens to multiline strings that are by definition
783 # reported as not fitting a single line, as well as lines that contain
784 # trailing commas (those have to be exploded).
785 yield from right_hand_split(line, mode, features=features)
787 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
788 # __name__ attribute which is needed in `run_transformer` further down.
789 # Unfortunately a nested class breaks mypyc too. So a class must be created
790 # via type ... https://github.com/mypyc/mypyc/issues/884
791 rhs = type("rhs", (), {"__call__": _rhs})()
793 if Preview.string_processing in mode:
794 if line.inside_brackets:
795 transformers = [
796 string_merge,
797 string_paren_strip,
798 string_split,
799 delimiter_split,
800 standalone_comment_split,
801 string_paren_wrap,
802 rhs,
803 ]
804 else:
805 transformers = [
806 string_merge,
807 string_paren_strip,
808 string_split,
809 string_paren_wrap,
810 rhs,
811 ]
812 else:
813 if line.inside_brackets:
814 transformers = [delimiter_split, standalone_comment_split, rhs]
815 else:
816 transformers = [rhs]
818 if Preview.simplify_power_operator_hugging not in mode:
819 # It's always safe to attempt hugging of power operations and pretty much every
820 # line could match.
821 transformers.append(hug_power_op)
823 for transform in transformers:
824 # We are accumulating lines in `result` because we might want to abort
825 # mission and return the original line in the end, or attempt a different
826 # split altogether.
827 try:
828 result = run_transformer(line, transform, mode, features, line_str=line_str)
829 except CannotTransform:
830 continue
831 else:
832 yield from result
833 break
835 else:
836 yield line
839def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
840 """If a funcdef has a magic trailing comma in the return type, then we should first
841 split the line with rhs to respect the comma.
842 """
843 return_type_leaves: list[Leaf] = []
844 in_return_type = False
846 for leaf in line.leaves:
847 if leaf.type == token.COLON:
848 in_return_type = False
849 if in_return_type:
850 return_type_leaves.append(leaf)
851 if leaf.type == token.RARROW:
852 in_return_type = True
854 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
855 # couple lines from it.
856 result = Line(mode=line.mode, depth=line.depth)
857 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
858 for leaf in return_type_leaves:
859 result.append(
860 leaf,
861 preformatted=True,
862 track_bracket=id(leaf) in leaves_to_track,
863 )
865 # we could also return true if the line is too long, and the return type is longer
866 # than the param list. Or if `should_split_rhs` returns True.
867 return result.magic_trailing_comma is not None
870class _BracketSplitComponent(Enum):
871 head = auto()
872 body = auto()
873 tail = auto()
876def left_hand_split(
877 line: Line, _features: Collection[Feature], mode: Mode
878) -> Iterator[Line]:
879 """Split line into many lines, starting with the first matching bracket pair.
881 Note: this usually looks weird, only use this for function definitions.
882 Prefer RHS otherwise. This is why this function is not symmetrical with
883 :func:`right_hand_split` which also handles optional parentheses.
884 """
885 for leaf_type in [token.LPAR, token.LSQB]:
886 tail_leaves: list[Leaf] = []
887 body_leaves: list[Leaf] = []
888 head_leaves: list[Leaf] = []
889 current_leaves = head_leaves
890 matching_bracket: Leaf | None = None
891 depth = 0
892 for index, leaf in enumerate(line.leaves):
893 if index == 2 and leaf.type == token.LSQB:
894 # A [ at index 2 means this is a type param, so start
895 # tracking the depth
896 depth += 1
897 elif depth > 0:
898 if leaf.type == token.LSQB:
899 depth += 1
900 elif leaf.type == token.RSQB:
901 depth -= 1
902 if (
903 current_leaves is body_leaves
904 and leaf.type in CLOSING_BRACKETS
905 and leaf.opening_bracket is matching_bracket
906 and isinstance(matching_bracket, Leaf)
907 # If the code is still on LPAR and we are inside a type
908 # param, ignore the match since this is searching
909 # for the function arguments
910 and not (leaf_type == token.LPAR and depth > 0)
911 ):
912 ensure_visible(leaf)
913 ensure_visible(matching_bracket)
914 current_leaves = tail_leaves if body_leaves else head_leaves
915 current_leaves.append(leaf)
916 if current_leaves is head_leaves:
917 if leaf.type == leaf_type and (
918 not (leaf_type == token.LPAR and depth > 0)
919 ):
920 matching_bracket = leaf
921 current_leaves = body_leaves
922 if matching_bracket and tail_leaves:
923 break
924 if not matching_bracket or not tail_leaves:
925 raise CannotSplit("No brackets found")
927 head = bracket_split_build_line(
928 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
929 )
930 body = bracket_split_build_line(
931 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
932 )
933 tail = bracket_split_build_line(
934 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
935 )
936 bracket_split_succeeded_or_raise(head, body, tail)
937 for result in (head, body, tail):
938 if result:
939 yield result
942def right_hand_split(
943 line: Line,
944 mode: Mode,
945 features: Collection[Feature] = (),
946 omit: Collection[LeafID] = (),
947) -> Iterator[Line]:
948 """Split line into many lines, starting with the last matching bracket pair.
950 If the split was by optional parentheses, attempt splitting without them, too.
951 `omit` is a collection of closing bracket IDs that shouldn't be considered for
952 this split.
954 Note: running this function modifies `bracket_depth` on the leaves of `line`.
955 """
956 rhs_result = _first_right_hand_split(line, omit=omit)
957 yield from _maybe_split_omitting_optional_parens(
958 rhs_result, line, mode, features=features, omit=omit
959 )
962def _first_right_hand_split(
963 line: Line,
964 omit: Collection[LeafID] = (),
965) -> RHSResult:
966 """Split the line into head, body, tail starting with the last bracket pair.
968 Note: this function should not have side effects. It's relied upon by
969 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
970 splitting on the right side of an assignment statement.
971 """
972 tail_leaves: list[Leaf] = []
973 body_leaves: list[Leaf] = []
974 head_leaves: list[Leaf] = []
975 current_leaves = tail_leaves
976 opening_bracket: Leaf | None = None
977 closing_bracket: Leaf | None = None
978 for leaf in reversed(line.leaves):
979 if current_leaves is body_leaves:
980 if leaf is opening_bracket:
981 current_leaves = head_leaves if body_leaves else tail_leaves
982 current_leaves.append(leaf)
983 if current_leaves is tail_leaves:
984 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
985 opening_bracket = leaf.opening_bracket
986 closing_bracket = leaf
987 current_leaves = body_leaves
988 if not (opening_bracket and closing_bracket and head_leaves):
989 # If there is no opening or closing_bracket that means the split failed and
990 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
991 # the matching `opening_bracket` wasn't available on `line` anymore.
992 raise CannotSplit("No brackets found")
994 tail_leaves.reverse()
995 body_leaves.reverse()
996 head_leaves.reverse()
998 body: Line | None = None
999 if (
1000 Preview.hug_parens_with_braces_and_square_brackets in line.mode
1001 and tail_leaves[0].value
1002 and tail_leaves[0].opening_bracket is head_leaves[-1]
1003 ):
1004 inner_body_leaves = list(body_leaves)
1005 hugged_opening_leaves: list[Leaf] = []
1006 hugged_closing_leaves: list[Leaf] = []
1007 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
1008 unpacking_offset: int = 1 if is_unpacking else 0
1009 while (
1010 len(inner_body_leaves) >= 2 + unpacking_offset
1011 and inner_body_leaves[-1].type in CLOSING_BRACKETS
1012 and inner_body_leaves[-1].opening_bracket
1013 is inner_body_leaves[unpacking_offset]
1014 ):
1015 if unpacking_offset:
1016 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1017 unpacking_offset = 0
1018 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1019 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
1021 if hugged_opening_leaves and inner_body_leaves:
1022 inner_body = bracket_split_build_line(
1023 inner_body_leaves,
1024 line,
1025 hugged_opening_leaves[-1],
1026 component=_BracketSplitComponent.body,
1027 )
1028 if (
1029 line.mode.magic_trailing_comma
1030 and inner_body_leaves[-1].type == token.COMMA
1031 ):
1032 should_hug = True
1033 else:
1034 line_length = line.mode.line_length - sum(
1035 len(str(leaf))
1036 for leaf in hugged_opening_leaves + hugged_closing_leaves
1037 )
1038 if is_line_short_enough(
1039 inner_body, mode=replace(line.mode, line_length=line_length)
1040 ):
1041 # Do not hug if it fits on a single line.
1042 should_hug = False
1043 else:
1044 should_hug = True
1045 if should_hug:
1046 body_leaves = inner_body_leaves
1047 head_leaves.extend(hugged_opening_leaves)
1048 tail_leaves = hugged_closing_leaves + tail_leaves
1049 body = inner_body # No need to re-calculate the body again later.
1051 head = bracket_split_build_line(
1052 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1053 )
1054 if body is None:
1055 body = bracket_split_build_line(
1056 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1057 )
1058 tail = bracket_split_build_line(
1059 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1060 )
1061 bracket_split_succeeded_or_raise(head, body, tail)
1062 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1065def _maybe_split_omitting_optional_parens(
1066 rhs: RHSResult,
1067 line: Line,
1068 mode: Mode,
1069 features: Collection[Feature] = (),
1070 omit: Collection[LeafID] = (),
1071) -> Iterator[Line]:
1072 if (
1073 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1074 # the opening bracket is an optional paren
1075 and rhs.opening_bracket.type == token.LPAR
1076 and not rhs.opening_bracket.value
1077 # the closing bracket is an optional paren
1078 and rhs.closing_bracket.type == token.RPAR
1079 and not rhs.closing_bracket.value
1080 # it's not an import (optional parens are the only thing we can split on
1081 # in this case; attempting a split without them is a waste of time)
1082 and not line.is_import
1083 # and we can actually remove the parens
1084 and can_omit_invisible_parens(rhs, mode.line_length)
1085 ):
1086 omit = {id(rhs.closing_bracket), *omit}
1087 try:
1088 # The RHSResult Omitting Optional Parens.
1089 rhs_oop = _first_right_hand_split(line, omit=omit)
1090 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1091 yield from _maybe_split_omitting_optional_parens(
1092 rhs_oop, line, mode, features=features, omit=omit
1093 )
1094 return
1096 except CannotSplit as e:
1097 # For chained assignments we want to use the previous successful split
1098 if line.is_chained_assignment:
1099 pass
1101 elif (
1102 not can_be_split(rhs.body)
1103 and not is_line_short_enough(rhs.body, mode=mode)
1104 and not (
1105 Preview.wrap_long_dict_values_in_parens
1106 and rhs.opening_bracket.parent
1107 and rhs.opening_bracket.parent.parent
1108 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1109 )
1110 and not (
1111 rhs.opening_bracket.parent
1112 and rhs.opening_bracket.parent.parent
1113 and rhs.opening_bracket.parent.parent.type == syms.case_block
1114 )
1115 ):
1116 raise CannotSplit(
1117 "Splitting failed, body is still too long and can't be split."
1118 ) from e
1120 elif (
1121 rhs.head.contains_multiline_strings()
1122 or rhs.tail.contains_multiline_strings()
1123 ):
1124 raise CannotSplit(
1125 "The current optional pair of parentheses is bound to fail to"
1126 " satisfy the splitting algorithm because the head or the tail"
1127 " contains multiline strings which by definition never fit one"
1128 " line."
1129 ) from e
1131 ensure_visible(rhs.opening_bracket)
1132 ensure_visible(rhs.closing_bracket)
1133 for result in (rhs.head, rhs.body, rhs.tail):
1134 if result:
1135 yield result
1138def _prefer_split_rhs_oop_over_rhs(
1139 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1140) -> bool:
1141 """
1142 Returns whether we should prefer the result from a split omitting optional parens
1143 (rhs_oop) over the original (rhs).
1144 """
1145 # contains unsplittable type ignore
1146 if (
1147 rhs_oop.head.contains_unsplittable_type_ignore()
1148 or rhs_oop.body.contains_unsplittable_type_ignore()
1149 or rhs_oop.tail.contains_unsplittable_type_ignore()
1150 ):
1151 return True
1153 # Retain optional parens around dictionary values
1154 if (
1155 Preview.wrap_long_dict_values_in_parens
1156 and rhs.opening_bracket.parent
1157 and rhs.opening_bracket.parent.parent
1158 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1159 and rhs.body.bracket_tracker.delimiters
1160 ):
1161 # Unless the split is inside the key
1162 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1164 # the split is right after `=`
1165 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1166 return True
1168 # the left side of assignment contains brackets
1169 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1170 return True
1172 # the left side of assignment is short enough (the -1 is for the ending optional
1173 # paren)
1174 if not is_line_short_enough(
1175 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1176 ):
1177 return True
1179 # the left side of assignment won't explode further because of magic trailing comma
1180 if rhs.head.magic_trailing_comma is not None:
1181 return True
1183 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1184 # the body
1185 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1186 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1187 token.EQUAL
1188 )
1189 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1190 return False
1192 has_closing_bracket_after_assign = False
1193 for leaf in reversed(rhs_oop.head.leaves):
1194 if leaf.type == token.EQUAL:
1195 break
1196 if leaf.type in CLOSING_BRACKETS:
1197 has_closing_bracket_after_assign = True
1198 break
1199 return (
1200 # contains matching brackets after the `=` (done by checking there is a
1201 # closing bracket)
1202 has_closing_bracket_after_assign
1203 or (
1204 # the split is actually from inside the optional parens (done by checking
1205 # the first line still contains the `=`)
1206 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1207 # the first line is short enough
1208 and is_line_short_enough(rhs_oop.head, mode=mode)
1209 )
1210 )
1213def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1214 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1216 Do nothing otherwise.
1218 A left- or right-hand split is based on a pair of brackets. Content before
1219 (and including) the opening bracket is left on one line, content inside the
1220 brackets is put on a separate line, and finally content starting with and
1221 following the closing bracket is put on a separate line.
1223 Those are called `head`, `body`, and `tail`, respectively. If the split
1224 produced the same line (all content in `head`) or ended up with an empty `body`
1225 and the `tail` is just the closing bracket, then it's considered failed.
1226 """
1227 tail_len = len(str(tail).strip())
1228 if not body:
1229 if tail_len == 0:
1230 raise CannotSplit("Splitting brackets produced the same line")
1232 elif tail_len < 3:
1233 raise CannotSplit(
1234 f"Splitting brackets on an empty body to save {tail_len} characters is"
1235 " not worth it"
1236 )
1239def _ensure_trailing_comma(
1240 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1241) -> bool:
1242 if not leaves:
1243 return False
1244 # Ensure a trailing comma for imports
1245 if original.is_import:
1246 return True
1247 # ...and standalone function arguments
1248 if not original.is_def:
1249 return False
1250 if opening_bracket.value != "(":
1251 return False
1252 # Don't add commas if we already have any commas
1253 if any(
1254 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1255 ):
1256 return False
1258 # Find a leaf with a parent (comments don't have parents)
1259 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1260 if leaf_with_parent is None:
1261 return True
1262 # Don't add commas inside parenthesized return annotations
1263 if get_annotation_type(leaf_with_parent) == "return":
1264 return False
1265 # Don't add commas inside PEP 604 unions
1266 if (
1267 leaf_with_parent.parent
1268 and leaf_with_parent.parent.next_sibling
1269 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1270 ):
1271 return False
1272 return True
1275def bracket_split_build_line(
1276 leaves: list[Leaf],
1277 original: Line,
1278 opening_bracket: Leaf,
1279 *,
1280 component: _BracketSplitComponent,
1281) -> Line:
1282 """Return a new line with given `leaves` and respective comments from `original`.
1284 If it's the head component, brackets will be tracked so trailing commas are
1285 respected.
1287 If it's the body component, the result line is one-indented inside brackets and as
1288 such has its first leaf's prefix normalized and a trailing comma added when
1289 expected.
1290 """
1291 result = Line(mode=original.mode, depth=original.depth)
1292 if component is _BracketSplitComponent.body:
1293 result.inside_brackets = True
1294 result.depth += 1
1295 if _ensure_trailing_comma(leaves, original, opening_bracket):
1296 for i in range(len(leaves) - 1, -1, -1):
1297 if leaves[i].type == STANDALONE_COMMENT:
1298 continue
1300 if leaves[i].type != token.COMMA:
1301 new_comma = Leaf(token.COMMA, ",")
1302 leaves.insert(i + 1, new_comma)
1303 break
1305 leaves_to_track: set[LeafID] = set()
1306 if component is _BracketSplitComponent.head:
1307 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1308 # Populate the line
1309 for leaf in leaves:
1310 result.append(
1311 leaf,
1312 preformatted=True,
1313 track_bracket=id(leaf) in leaves_to_track,
1314 )
1315 for comment_after in original.comments_after(leaf):
1316 result.append(comment_after, preformatted=True)
1317 if component is _BracketSplitComponent.body and should_split_line(
1318 result, opening_bracket
1319 ):
1320 result.should_split_rhs = True
1321 return result
1324def dont_increase_indentation(split_func: Transformer) -> Transformer:
1325 """Normalize prefix of the first leaf in every line returned by `split_func`.
1327 This is a decorator over relevant split functions.
1328 """
1330 @wraps(split_func)
1331 def split_wrapper(
1332 line: Line, features: Collection[Feature], mode: Mode
1333 ) -> Iterator[Line]:
1334 for split_line in split_func(line, features, mode):
1335 split_line.leaves[0].prefix = ""
1336 yield split_line
1338 return split_wrapper
1341def _get_last_non_comment_leaf(line: Line) -> int | None:
1342 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1343 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1344 return leaf_idx
1345 return None
1348def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1349 if is_vararg(leaf, within={syms.typedargslist}):
1350 return Feature.TRAILING_COMMA_IN_DEF in features
1351 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1352 return Feature.TRAILING_COMMA_IN_CALL in features
1353 return True
1356def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1357 if (
1358 safe
1359 and delimiter_priority == COMMA_PRIORITY
1360 and line.leaves[-1].type != token.COMMA
1361 and line.leaves[-1].type != STANDALONE_COMMENT
1362 ):
1363 new_comma = Leaf(token.COMMA, ",")
1364 line.append(new_comma)
1365 return line
1368MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1371def _can_defer_lone_comparator_to_rhs(line: Line, mode: Mode) -> bool:
1372 """Return True if the lone comparator on `line` can defer to right_hand_split.
1374 Caller has already established exactly one delimiter at
1375 `COMPARATOR_PRIORITY`. We defer only when:
1377 - the LHS up to the comparator has no opening brackets, so the existing
1378 "break before the comparator" wouldn't produce a balanced two-sided
1379 split anyway, and
1380 - `right_hand_split` would produce a head that fits in the line length,
1381 so we don't strand `if t` on its own line just to push it back onto an
1382 overflowing single line when the RHS bracket can't be exploded
1383 usefully (e.g. an empty `decode()` paren).
1384 """
1385 past_comparator = False
1386 for leaf in line.leaves:
1387 if leaf.type in OPENING_BRACKETS and not past_comparator:
1388 return False
1389 if not past_comparator and (
1390 line.bracket_tracker.delimiters.get(id(leaf)) == COMPARATOR_PRIORITY
1391 ):
1392 past_comparator = True
1393 try:
1394 rhs = _first_right_hand_split(line)
1395 except CannotSplit:
1396 return False
1397 return is_line_short_enough(rhs.head, mode=mode)
1400@dont_increase_indentation
1401def delimiter_split(
1402 line: Line, features: Collection[Feature], mode: Mode
1403) -> Iterator[Line]:
1404 """Split according to delimiters of the highest priority.
1406 If the appropriate Features are given, the split will add trailing commas
1407 also in function signatures and calls that contain `*` and `**`.
1408 """
1409 if len(line.leaves) == 0:
1410 raise CannotSplit("Line empty") from None
1411 last_leaf = line.leaves[-1]
1413 bt = line.bracket_tracker
1414 try:
1415 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1416 except ValueError:
1417 raise CannotSplit("No delimiters found") from None
1419 if (
1420 delimiter_priority == DOT_PRIORITY
1421 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1422 ):
1423 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1425 if (
1426 Preview.hug_comparator in mode
1427 and delimiter_priority == COMPARATOR_PRIORITY
1428 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1429 and _can_defer_lone_comparator_to_rhs(line, mode)
1430 ):
1431 raise CannotSplit("Bracketed RHS will explode via right_hand_split")
1433 current_line = Line(
1434 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1435 )
1436 lowest_depth = sys.maxsize
1437 trailing_comma_safe = True
1439 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1440 """Append `leaf` to current line or to new line if appending impossible."""
1441 nonlocal current_line
1442 try:
1443 current_line.append_safe(leaf, preformatted=True)
1444 except ValueError:
1445 yield current_line
1447 current_line = Line(
1448 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1449 )
1450 current_line.append(leaf)
1452 def append_comments(leaf: Leaf) -> Iterator[Line]:
1453 for comment_after in line.comments_after(leaf):
1454 yield from append_to_line(comment_after)
1456 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1457 for leaf_idx, leaf in enumerate(line.leaves):
1458 yield from append_to_line(leaf)
1460 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1461 id(line.leaves[leaf_idx - 1])
1462 )
1463 if (
1464 previous_priority != delimiter_priority
1465 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1466 ):
1467 yield from append_comments(leaf)
1469 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1470 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1471 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1473 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1474 current_line = _safe_add_trailing_comma(
1475 trailing_comma_safe, delimiter_priority, current_line
1476 )
1478 leaf_priority = bt.delimiters.get(id(leaf))
1479 if leaf_priority == delimiter_priority:
1480 if (
1481 leaf_idx + 1 < len(line.leaves)
1482 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1483 ):
1484 yield from append_comments(line.leaves[leaf_idx + 1])
1486 yield current_line
1487 current_line = Line(
1488 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1489 )
1491 if current_line:
1492 current_line = _safe_add_trailing_comma(
1493 trailing_comma_safe, delimiter_priority, current_line
1494 )
1495 yield current_line
1498@dont_increase_indentation
1499def standalone_comment_split(
1500 line: Line, features: Collection[Feature], mode: Mode
1501) -> Iterator[Line]:
1502 """Split standalone comments from the rest of the line."""
1503 if not line.contains_standalone_comments():
1504 raise CannotSplit("Line does not have any standalone comments")
1506 current_line = Line(
1507 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1508 )
1510 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1511 """Append `leaf` to current line or to new line if appending impossible."""
1512 nonlocal current_line
1513 try:
1514 current_line.append_safe(leaf, preformatted=True)
1515 except ValueError:
1516 yield current_line
1518 current_line = Line(
1519 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1520 )
1521 current_line.append(leaf)
1523 for leaf in line.leaves:
1524 yield from append_to_line(leaf)
1526 for comment_after in line.comments_after(leaf):
1527 yield from append_to_line(comment_after)
1529 if current_line:
1530 yield current_line
1533def normalize_invisible_parens(
1534 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1535) -> None:
1536 """Make existing optional parentheses invisible or create new ones.
1538 `parens_after` is a set of string leaf values immediately after which parens
1539 should be put.
1541 Standardizes on visible parentheses for single-element tuples, and keeps
1542 existing visible parentheses for other tuples and generator expressions.
1543 """
1544 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1545 if contains_fmt_directive(pc.value, FMT_OFF):
1546 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1547 return
1549 # The multiple context managers grammar has a different pattern, thus this is
1550 # separate from the for-loop below. This possibly wraps them in invisible parens,
1551 # and later will be removed in remove_with_parens when needed.
1552 if node.type == syms.with_stmt:
1553 _maybe_wrap_cms_in_parens(node, mode, features)
1555 check_lpar = False
1556 for index, child in enumerate(list(node.children)):
1557 # Fixes a bug where invisible parens are not properly stripped from
1558 # assignment statements that contain type annotations.
1559 if isinstance(child, Node) and child.type == syms.annassign:
1560 normalize_invisible_parens(
1561 child, parens_after=parens_after, mode=mode, features=features
1562 )
1564 # Fixes a bug where invisible parens are not properly wrapped around
1565 # case blocks.
1566 if isinstance(child, Node) and child.type == syms.case_block:
1567 normalize_invisible_parens(
1568 child, parens_after={"case"}, mode=mode, features=features
1569 )
1571 # Add parentheses around if guards in case blocks
1572 if isinstance(child, Node) and child.type == syms.guard:
1573 normalize_invisible_parens(
1574 child, parens_after={"if"}, mode=mode, features=features
1575 )
1577 # Add parentheses around long tuple unpacking in assignments.
1578 if (
1579 index == 0
1580 and isinstance(child, Node)
1581 and child.type == syms.testlist_star_expr
1582 ):
1583 check_lpar = True
1585 if (
1586 index == 0
1587 and isinstance(child, Node)
1588 and child.type == syms.atom
1589 and node.type == syms.expr_stmt
1590 and not _atom_has_magic_trailing_comma(child, mode)
1591 and not _is_atom_multiline(child)
1592 ):
1593 if maybe_make_parens_invisible_in_atom(
1594 child,
1595 parent=node,
1596 mode=mode,
1597 features=features,
1598 remove_brackets_around_comma=True,
1599 allow_star_expr=True,
1600 ):
1601 wrap_in_parentheses(node, child, visible=False)
1603 if check_lpar:
1604 if (
1605 child.type == syms.atom
1606 and node.type == syms.for_stmt
1607 and isinstance(child.prev_sibling, Leaf)
1608 and child.prev_sibling.type == token.NAME
1609 and child.prev_sibling.value == "for"
1610 ):
1611 if maybe_make_parens_invisible_in_atom(
1612 child,
1613 parent=node,
1614 mode=mode,
1615 features=features,
1616 remove_brackets_around_comma=True,
1617 ):
1618 wrap_in_parentheses(node, child, visible=False)
1619 elif isinstance(child, Node) and node.type == syms.with_stmt:
1620 remove_with_parens(child, node, mode=mode, features=features)
1621 elif child.type == syms.atom and not (
1622 "in" in parens_after
1623 and len(child.children) == 3
1624 and is_lpar_token(child.children[0])
1625 and is_rpar_token(child.children[-1])
1626 and child.children[1].type == syms.test
1627 ):
1628 if maybe_make_parens_invisible_in_atom(
1629 child, parent=node, mode=mode, features=features
1630 ):
1631 wrap_in_parentheses(node, child, visible=False)
1632 elif is_one_tuple(child):
1633 wrap_in_parentheses(node, child, visible=True)
1634 elif node.type == syms.import_from:
1635 _normalize_import_from(node, child, index)
1636 break
1637 elif (
1638 index == 1
1639 and child.type == token.STAR
1640 and node.type == syms.except_clause
1641 ):
1642 # In except* (PEP 654), the star is actually part of
1643 # of the keyword. So we need to skip the insertion of
1644 # invisible parentheses to work more precisely.
1645 continue
1647 elif (
1648 isinstance(child, Leaf)
1649 and child.next_sibling is not None
1650 and child.next_sibling.type == token.COLON
1651 and child.value == "case"
1652 ):
1653 # A special patch for "case case:" scenario, the second occurrence
1654 # of case will be not parsed as a Python keyword.
1655 break
1657 elif isinstance(child, Node) and child.type == syms.guard:
1658 # Guard nodes handle their own inner wrapping. Wrapping the guard
1659 # itself can produce invalid output when the case pattern splits.
1660 pass
1662 elif not is_multiline_string(child):
1663 if (
1664 Preview.fix_if_guard_explosion_in_case_statement in mode
1665 and node.type == syms.guard
1666 ):
1667 mock_line = Line(mode=mode)
1668 for leaf in child.leaves():
1669 mock_line.append(leaf)
1670 # If it's a guard AND it's short, we DON'T wrap
1671 if not is_line_short_enough(mock_line, mode=mode):
1672 wrap_in_parentheses(node, child, visible=False)
1673 else:
1674 wrap_in_parentheses(node, child, visible=False)
1676 comma_check = child.type == token.COMMA
1678 check_lpar = isinstance(child, Leaf) and (
1679 child.value in parens_after or comma_check
1680 )
1683def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1684 # "import from" nodes store parentheses directly as part of
1685 # the statement
1686 if is_lpar_token(child):
1687 assert is_rpar_token(parent.children[-1])
1688 # make parentheses invisible
1689 child.value = ""
1690 parent.children[-1].value = ""
1691 elif child.type != token.STAR:
1692 # insert invisible parentheses
1693 parent.insert_child(index, Leaf(token.LPAR, ""))
1694 parent.append_child(Leaf(token.RPAR, ""))
1697def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1698 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1699 if (
1700 node.children[1].type == syms.atom
1701 and node.children[1].children[0].type == token.LPAR
1702 ):
1703 if maybe_make_parens_invisible_in_atom(
1704 node.children[1],
1705 parent=node,
1706 mode=mode,
1707 features=features,
1708 remove_brackets_around_comma=True,
1709 ):
1710 wrap_in_parentheses(node, node.children[1], visible=False)
1712 # Since await is an expression we shouldn't remove
1713 # brackets in cases where this would change
1714 # the AST due to operator precedence.
1715 # Therefore we only aim to remove brackets around
1716 # power nodes that aren't also await expressions themselves.
1717 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1718 # N.B. We've still removed any redundant nested brackets though :)
1719 opening_bracket = cast(Leaf, node.children[1].children[0])
1720 closing_bracket = cast(Leaf, node.children[1].children[-1])
1721 bracket_contents = node.children[1].children[1]
1722 if isinstance(bracket_contents, Node) and (
1723 bracket_contents.type != syms.power
1724 or bracket_contents.children[0].type == token.AWAIT
1725 or any(
1726 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1727 for child in bracket_contents.children
1728 )
1729 ):
1730 ensure_visible(opening_bracket)
1731 ensure_visible(closing_bracket)
1734def _maybe_wrap_cms_in_parens(
1735 node: Node, mode: Mode, features: Collection[Feature]
1736) -> None:
1737 """When enabled and safe, wrap the multiple context managers in invisible parens.
1739 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1740 """
1741 if (
1742 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1743 or len(node.children) <= 2
1744 # If it's an atom, it's already wrapped in parens.
1745 or node.children[1].type == syms.atom
1746 ):
1747 return
1748 colon_index: int | None = None
1749 for i in range(2, len(node.children)):
1750 if node.children[i].type == token.COLON:
1751 colon_index = i
1752 break
1753 if colon_index is not None:
1754 lpar = Leaf(token.LPAR, "")
1755 rpar = Leaf(token.RPAR, "")
1756 context_managers = node.children[1:colon_index]
1757 for child in context_managers:
1758 child.remove()
1759 # After wrapping, the with_stmt will look like this:
1760 # with_stmt
1761 # NAME 'with'
1762 # atom
1763 # LPAR ''
1764 # testlist_gexp
1765 # ... <-- context_managers
1766 # /testlist_gexp
1767 # RPAR ''
1768 # /atom
1769 # COLON ':'
1770 new_child = Node(
1771 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1772 )
1773 node.insert_child(1, new_child)
1776def remove_with_parens(
1777 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1778) -> None:
1779 """Recursively hide optional parens in `with` statements."""
1780 # Removing all unnecessary parentheses in with statements in one pass is a tad
1781 # complex as different variations of bracketed statements result in pretty
1782 # different parse trees:
1783 #
1784 # with (open("file")) as f: # this is an asexpr_test
1785 # ...
1786 #
1787 # with (open("file") as f): # this is an atom containing an
1788 # ... # asexpr_test
1789 #
1790 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1791 # ... # asexpr_test
1792 #
1793 # with (open("file") as f, open("file") as f): # an atom containing a
1794 # ... # testlist_gexp which then
1795 # # contains multiple asexpr_test(s)
1796 if node.type == syms.atom:
1797 if maybe_make_parens_invisible_in_atom(
1798 node,
1799 parent=parent,
1800 mode=mode,
1801 features=features,
1802 remove_brackets_around_comma=True,
1803 ):
1804 wrap_in_parentheses(parent, node, visible=False)
1805 if isinstance(node.children[1], Node):
1806 remove_with_parens(node.children[1], node, mode=mode, features=features)
1807 elif node.type == syms.testlist_gexp:
1808 for child in node.children:
1809 if isinstance(child, Node):
1810 remove_with_parens(child, node, mode=mode, features=features)
1811 elif node.type == syms.asexpr_test and not any(
1812 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1813 ):
1814 if maybe_make_parens_invisible_in_atom(
1815 node.children[0],
1816 parent=node,
1817 mode=mode,
1818 features=features,
1819 remove_brackets_around_comma=True,
1820 ):
1821 wrap_in_parentheses(node, node.children[0], visible=False)
1824def _atom_has_magic_trailing_comma(node: LN, mode: Mode) -> bool:
1825 """Check if an atom node has a magic trailing comma.
1827 Returns True for single-element tuples with trailing commas like (a,),
1828 which should be preserved to maintain their tuple type.
1829 """
1830 if not mode.magic_trailing_comma:
1831 return False
1833 return is_one_tuple(node)
1836def _is_atom_multiline(node: LN) -> bool:
1837 """Check if an atom node is multiline (indicating intentional formatting)."""
1838 if not isinstance(node, Node) or len(node.children) < 3:
1839 return False
1841 # Check the middle child (between LPAR and RPAR) for newlines in its subtree
1842 # The first child's prefix contains blank lines/comments before the opening paren
1843 middle = node.children[1]
1844 for child in middle.pre_order():
1845 if isinstance(child, Leaf) and "\n" in child.prefix:
1846 return True
1848 return False
1851def maybe_make_parens_invisible_in_atom(
1852 node: LN,
1853 parent: LN,
1854 mode: Mode,
1855 features: Collection[Feature],
1856 remove_brackets_around_comma: bool = False,
1857 allow_star_expr: bool = False,
1858) -> bool:
1859 """If it's safe, make the parens in the atom `node` invisible, recursively.
1860 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1861 as they are redundant.
1863 Returns whether the node should itself be wrapped in invisible parentheses.
1864 """
1865 if (
1866 node.type not in (syms.atom, syms.expr)
1867 or is_empty_tuple(node)
1868 or is_one_tuple(node)
1869 or (is_tuple(node) and parent.type == syms.asexpr_test)
1870 or (
1871 is_tuple(node)
1872 and parent.type == syms.with_stmt
1873 and has_sibling_with_type(node, token.COMMA)
1874 )
1875 or (is_yield(node) and parent.type != syms.expr_stmt)
1876 or (
1877 # This condition tries to prevent removing non-optional brackets
1878 # around a tuple, however, can be a bit overzealous so we provide
1879 # and option to skip this check for `for` and `with` statements.
1880 not remove_brackets_around_comma
1881 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1882 # Remove parentheses around multiple exception types in except and
1883 # except* without as. See PEP 758 for details.
1884 and not (
1885 Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1886 # is a tuple
1887 and is_tuple(node)
1888 # has a parent node
1889 and node.parent is not None
1890 # parent is an except clause
1891 and node.parent.type == syms.except_clause
1892 # is not immediately followed by as clause
1893 and not (
1894 node.next_sibling is not None
1895 and is_name_token(node.next_sibling)
1896 and node.next_sibling.value == "as"
1897 )
1898 )
1899 )
1900 or is_tuple_containing_walrus(node)
1901 or (not allow_star_expr and is_tuple_containing_star(node))
1902 or is_generator(node)
1903 ):
1904 return False
1906 if is_walrus_assignment(node):
1907 if parent.type in [
1908 syms.annassign,
1909 syms.expr_stmt,
1910 syms.assert_stmt,
1911 syms.return_stmt,
1912 syms.except_clause,
1913 syms.funcdef,
1914 syms.with_stmt,
1915 syms.testlist_gexp,
1916 syms.tname,
1917 # these ones aren't useful to end users, but they do please fuzzers
1918 syms.for_stmt,
1919 syms.del_stmt,
1920 syms.for_stmt,
1921 ]:
1922 return False
1924 first = node.children[0]
1925 last = node.children[-1]
1926 if is_lpar_token(first) and is_rpar_token(last):
1927 middle = node.children[1]
1928 # make parentheses invisible
1929 if (
1930 # If the prefix of `middle` includes a type comment with
1931 # ignore annotation, then we do not remove the parentheses
1932 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
1933 ):
1934 first.value = ""
1935 last.value = ""
1936 maybe_make_parens_invisible_in_atom(
1937 middle,
1938 parent=parent,
1939 mode=mode,
1940 features=features,
1941 remove_brackets_around_comma=remove_brackets_around_comma,
1942 )
1944 if is_atom_with_invisible_parens(middle):
1945 # Strip the invisible parens from `middle` by replacing
1946 # it with the child in-between the invisible parens
1947 middle.replace(middle.children[1])
1949 if middle.children[0].prefix.strip():
1950 # Preserve comments before first paren
1951 middle.children[1].prefix = (
1952 middle.children[0].prefix + middle.children[1].prefix
1953 )
1955 if middle.children[-1].prefix.strip():
1956 # Preserve comments before last paren
1957 last.prefix = middle.children[-1].prefix + last.prefix
1959 return False
1961 return True
1964def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1965 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1967 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1968 return False
1970 # We're essentially checking if the body is delimited by commas and there's more
1971 # than one of them (we're excluding the trailing comma and if the delimiter priority
1972 # is still commas, that means there's more).
1973 exclude = set()
1974 trailing_comma = False
1975 try:
1976 last_leaf = line.leaves[-1]
1977 if last_leaf.type == token.COMMA:
1978 trailing_comma = True
1979 exclude.add(id(last_leaf))
1980 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1981 except (IndexError, ValueError):
1982 return False
1984 return max_priority == COMMA_PRIORITY and (
1985 (line.mode.magic_trailing_comma and trailing_comma)
1986 # always explode imports
1987 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1988 )
1991def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1992 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1994 Brackets can be omitted if the entire trailer up to and including
1995 a preceding closing bracket fits in one line.
1997 Yielded sets are cumulative (contain results of previous yields, too). First
1998 set is empty, unless the line should explode, in which case bracket pairs until
1999 the one that needs to explode are omitted.
2000 """
2002 omit: set[LeafID] = set()
2003 if not line.magic_trailing_comma:
2004 yield omit
2006 length = 4 * line.depth
2007 opening_bracket: Leaf | None = None
2008 closing_bracket: Leaf | None = None
2009 inner_brackets: set[LeafID] = set()
2010 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
2011 length += leaf_length
2012 if length > line_length:
2013 break
2015 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
2016 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
2017 break
2019 if opening_bracket:
2020 if leaf is opening_bracket:
2021 opening_bracket = None
2022 elif leaf.type in CLOSING_BRACKETS:
2023 prev = line.leaves[index - 1] if index > 0 else None
2024 if (
2025 prev
2026 and prev.type == token.COMMA
2027 and leaf.opening_bracket is not None
2028 and not is_one_sequence_between(
2029 leaf.opening_bracket, leaf, line.leaves
2030 )
2031 ):
2032 # Never omit bracket pairs with trailing commas.
2033 # We need to explode on those.
2034 break
2036 inner_brackets.add(id(leaf))
2037 elif leaf.type in CLOSING_BRACKETS:
2038 prev = line.leaves[index - 1] if index > 0 else None
2039 if prev and prev.type in OPENING_BRACKETS:
2040 # Empty brackets would fail a split so treat them as "inner"
2041 # brackets (e.g. only add them to the `omit` set if another
2042 # pair of brackets was good enough.
2043 inner_brackets.add(id(leaf))
2044 continue
2046 if closing_bracket:
2047 omit.add(id(closing_bracket))
2048 omit.update(inner_brackets)
2049 inner_brackets.clear()
2050 yield omit
2052 if (
2053 prev
2054 and prev.type == token.COMMA
2055 and leaf.opening_bracket is not None
2056 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
2057 ):
2058 # Never omit bracket pairs with trailing commas.
2059 # We need to explode on those.
2060 break
2062 if leaf.value:
2063 opening_bracket = leaf.opening_bracket
2064 closing_bracket = leaf
2067def _over_length_only_due_to_subscript_comment(line: Line, mode: Mode) -> bool:
2068 """Return True if `line` only exceeds `mode.line_length` because of an inline
2069 comment attached to a subscript opening bracket (`[`).
2071 This is the shape produced by the original of the issue #4733 reproducer:
2072 a comment inside the annotation's subscript brackets renders at the end of
2073 the head line after Black splits the statement, pushing it past the limit.
2074 Taking the FORCE_OPTIONAL_PARENTHESES "second opinion" in that case wraps
2075 the annotation in extra parens and migrates the comment outside the
2076 subscript, which then oscillates on the next formatter pass.
2077 """
2078 if not line.leaves:
2079 return False
2080 # The over-length must be caused entirely by a trailing comment.
2081 indent = " " * line.depth
2082 leaves_iter = iter(line.leaves)
2083 first = next(leaves_iter)
2084 text_without_comments = f"{first.prefix}{indent}{first.value}"
2085 text_without_comments += "".join(str(leaf) for leaf in leaves_iter)
2086 if str_width(text_without_comments) > mode.line_length:
2087 return False
2088 # And the comment must be attached to a subscript opening bracket.
2089 for leaf_id, comments in line.comments.items():
2090 if not comments:
2091 continue
2092 leaf = next((lf for lf in line.leaves if id(lf) == leaf_id), None)
2093 if leaf is None or leaf.type != token.LSQB:
2094 return False
2095 return True
2098def run_transformer(
2099 line: Line,
2100 transform: Transformer,
2101 mode: Mode,
2102 features: Collection[Feature],
2103 *,
2104 line_str: str = "",
2105) -> list[Line]:
2106 if not line_str:
2107 line_str = line_to_string(line)
2108 result: list[Line] = []
2109 for transformed_line in transform(line, features, mode):
2110 if str(transformed_line).strip("\n") == line_str:
2111 raise CannotTransform("Line transformer returned an unchanged result")
2113 result.extend(transform_line(transformed_line, mode=mode, features=features))
2115 features_set = set(features)
2116 if (
2117 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
2118 or transform.__class__.__name__ != "rhs"
2119 or not line.bracket_tracker.invisible
2120 or any(bracket.value for bracket in line.bracket_tracker.invisible)
2121 or line.contains_multiline_strings()
2122 or result[0].contains_uncollapsable_type_comments()
2123 or result[0].contains_unsplittable_type_ignore()
2124 or is_line_short_enough(result[0], mode=mode)
2125 # result[0] only exceeds the length because of a comment attached to a
2126 # subscript opening bracket. Taking the FORCE_OPTIONAL_PARENTHESES
2127 # "second opinion" wraps the annotation in extra invisible parens and
2128 # migrates the comment outside the subscript, which then oscillates with
2129 # a deeper-bracket split on the next formatter pass (issue #4733).
2130 or _over_length_only_due_to_subscript_comment(result[0], mode)
2131 # If any leaves have no parents (which _can_ occur since
2132 # `transform(line)` potentially destroys the line's underlying node
2133 # structure), then we can't proceed. Doing so would cause the below
2134 # call to `append_leaves()` to fail.
2135 or any(leaf.parent is None for leaf in line.leaves)
2136 ):
2137 return result
2139 line_copy = line.clone()
2140 append_leaves(line_copy, line, line.leaves)
2141 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
2142 second_opinion = run_transformer(
2143 line_copy, transform, mode, features_fop, line_str=line_str
2144 )
2145 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
2146 result = second_opinion
2147 return result