Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 10%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterable, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 COMPARATOR_PRIORITY,
16 DOT_PRIORITY,
17 STRING_PRIORITY,
18 get_leaves_inside_matching_brackets,
19 max_delimiter_priority_in_atom,
20)
21from black.comments import (
22 FMT_OFF,
23 FMT_ON,
24 contains_fmt_directive,
25 generate_comments,
26 list_comments,
27)
28from black.lines import (
29 Line,
30 RHSResult,
31 append_leaves,
32 can_be_split,
33 can_omit_invisible_parens,
34 is_line_short_enough,
35 line_to_string,
36)
37from black.mode import Feature, Mode, Preview
38from black.nodes import (
39 ASSIGNMENTS,
40 BRACKETS,
41 CLOSING_BRACKETS,
42 OPENING_BRACKETS,
43 STANDALONE_COMMENT,
44 STATEMENT,
45 WHITESPACE,
46 Visitor,
47 ensure_visible,
48 fstring_tstring_to_string,
49 get_annotation_type,
50 has_sibling_with_type,
51 is_arith_like,
52 is_async_stmt_or_funcdef,
53 is_atom_with_invisible_parens,
54 is_docstring,
55 is_empty_tuple,
56 is_generator,
57 is_lpar_token,
58 is_multiline_string,
59 is_name_token,
60 is_one_sequence_between,
61 is_one_tuple,
62 is_parent_function_or_class,
63 is_part_of_annotation,
64 is_rpar_token,
65 is_stub_body,
66 is_stub_suite,
67 is_tuple,
68 is_tuple_containing_star,
69 is_tuple_containing_walrus,
70 is_type_ignore_comment,
71 is_type_ignore_comment_string,
72 is_vararg,
73 is_walrus_assignment,
74 is_yield,
75 syms,
76 wrap_in_parentheses,
77)
78from black.numerics import normalize_numeric_literal
79from black.strings import (
80 fix_multiline_docstring,
81 get_string_prefix,
82 normalize_string_prefix,
83 normalize_string_quotes,
84 normalize_unicode_escape_sequences,
85 str_width,
86)
87from black.trans import (
88 CannotTransform,
89 StringMerger,
90 StringParenStripper,
91 StringParenWrapper,
92 StringSplitter,
93 Transformer,
94 hug_power_op,
95)
96from blib2to3.pgen2 import token
97from blib2to3.pytree import Leaf, Node
99# types
100LeafID = int
101LN = Union[Leaf, Node]
104class CannotSplit(CannotTransform):
105 """A readable split that fits the allotted line length is impossible."""
108# This isn't a dataclass because @dataclass + Generic breaks mypyc.
109# See also https://github.com/mypyc/mypyc/issues/827.
110class LineGenerator(Visitor[Line]):
111 """Generates reformatted Line objects. Empty lines are not emitted.
113 Note: destroys the tree it's visiting by mutating prefixes of its leaves
114 in ways that will no longer stringify to valid Python code on the tree.
115 """
117 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
118 self.mode = mode
119 self.features = features
120 self.current_line: Line
121 self.__post_init__()
123 def line(self, indent: int = 0) -> Iterator[Line]:
124 """Generate a line.
126 If the line is empty, only emit if it makes sense.
127 If the line is too long, split it first and then generate.
129 If any lines were generated, set up a new current_line.
130 """
131 if not self.current_line:
132 self.current_line.depth += indent
133 return # Line is empty, don't emit. Creating a new one unnecessary.
135 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
136 self.current_line.leaves[0]
137 ):
138 # Special case for async def/for/with statements. `visit_async_stmt`
139 # adds an `ASYNC` leaf then visits the child def/for/with statement
140 # nodes. Line yields from those nodes shouldn't treat the former
141 # `ASYNC` leaf as a complete line.
142 return
144 complete_line = self.current_line
145 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
146 yield complete_line
148 def visit_default(self, node: LN) -> Iterator[Line]:
149 """Default `visit_*()` implementation. Recurses to children of `node`."""
150 if isinstance(node, Leaf):
151 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
152 for comment in generate_comments(node, mode=self.mode):
153 if any_open_brackets:
154 # any comment within brackets is subject to splitting
155 self.current_line.append(comment)
156 elif comment.type == token.COMMENT:
157 # regular trailing comment
158 self.current_line.append(comment)
159 yield from self.line()
161 else:
162 # regular standalone comment
163 yield from self.line()
165 self.current_line.append(comment)
166 yield from self.line()
168 if any_open_brackets:
169 node.prefix = ""
170 if node.type not in WHITESPACE:
171 self.current_line.append(node)
172 yield from super().visit_default(node)
174 def visit_test(self, node: Node) -> Iterator[Line]:
175 """Visit an `x if y else z` test"""
177 already_parenthesized = (
178 node.prev_sibling and node.prev_sibling.type == token.LPAR
179 )
181 if not already_parenthesized:
182 # Similar to logic in wrap_in_parentheses
183 lpar = Leaf(token.LPAR, "")
184 rpar = Leaf(token.RPAR, "")
185 prefix = node.prefix
186 node.prefix = ""
187 lpar.prefix = prefix
188 node.insert_child(0, lpar)
189 node.append_child(rpar)
191 yield from self.visit_default(node)
193 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
194 """Increase indentation level, maybe yield a line."""
195 # In blib2to3 INDENT never holds comments.
196 yield from self.line(+1)
197 yield from self.visit_default(node)
199 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
200 """Decrease indentation level, maybe yield a line."""
201 # The current line might still wait for trailing comments. At DEDENT time
202 # there won't be any (they would be prefixes on the preceding NEWLINE).
203 # Emit the line then.
204 yield from self.line()
206 # While DEDENT has no value, its prefix may contain standalone comments
207 # that belong to the current indentation level. Get 'em.
208 yield from self.visit_default(node)
210 # Finally, emit the dedent.
211 yield from self.line(-1)
213 def visit_stmt(
214 self, node: Node, keywords: set[str], parens: set[str]
215 ) -> Iterator[Line]:
216 """Visit a statement.
218 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
219 `def`, `with`, `class`, `assert`, and assignments.
221 The relevant Python language `keywords` for a given statement
222 appear as NAME leaves within it. This method puts those on a
223 separate line.
225 `parens` holds a set of string leaf values immediately after which
226 invisible parens should be put.
227 """
228 normalize_invisible_parens(
229 node, parens_after=parens, mode=self.mode, features=self.features
230 )
231 for child in node.children:
232 if is_name_token(child) and child.value in keywords:
233 yield from self.line()
235 yield from self.visit(child)
237 def visit_typeparams(self, node: Node) -> Iterator[Line]:
238 yield from self.visit_default(node)
239 node.children[0].prefix = ""
241 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
242 yield from self.visit_default(node)
243 node.children[1].prefix = ""
245 def visit_paramspec(self, node: Node) -> Iterator[Line]:
246 yield from self.visit_default(node)
247 node.children[1].prefix = ""
249 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
250 if Preview.wrap_long_dict_values_in_parens in self.mode:
251 for i, child in enumerate(node.children):
252 if i == 0:
253 continue
254 if node.children[i - 1].type == token.COLON:
255 if (
256 child.type == syms.atom
257 and child.children[0].type in OPENING_BRACKETS
258 and not is_walrus_assignment(child)
259 ):
260 maybe_make_parens_invisible_in_atom(
261 child,
262 parent=node,
263 mode=self.mode,
264 features=self.features,
265 remove_brackets_around_comma=False,
266 )
267 else:
268 wrap_in_parentheses(node, child, visible=False, index=i)
269 yield from self.visit_default(node)
271 def visit_funcdef(self, node: Node) -> Iterator[Line]:
272 """Visit function definition."""
273 yield from self.line()
275 # Remove redundant brackets around return type annotation.
276 is_return_annotation = False
277 for child in node.children:
278 if child.type == token.RARROW:
279 is_return_annotation = True
280 elif is_return_annotation:
281 if child.type == syms.atom and child.children[0].type == token.LPAR:
282 if maybe_make_parens_invisible_in_atom(
283 child,
284 parent=node,
285 mode=self.mode,
286 features=self.features,
287 remove_brackets_around_comma=False,
288 ):
289 wrap_in_parentheses(node, child, visible=False)
290 else:
291 wrap_in_parentheses(node, child, visible=False)
292 is_return_annotation = False
294 for child in node.children:
295 yield from self.visit(child)
297 def visit_match_case(self, node: Node) -> Iterator[Line]:
298 """Visit either a match or case statement."""
299 normalize_invisible_parens(
300 node, parens_after=set(), mode=self.mode, features=self.features
301 )
303 yield from self.line()
304 for child in node.children:
305 yield from self.visit(child)
307 def visit_suite(self, node: Node) -> Iterator[Line]:
308 """Visit a suite."""
309 if is_stub_suite(node):
310 yield from self.visit(node.children[2])
311 else:
312 yield from self.visit_default(node)
314 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
315 """Visit a statement without nested statements."""
316 prev_type: int | None = None
317 for i, child in enumerate(node.children):
318 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
319 wrap_in_parentheses(node, child, visible=False, index=i)
320 prev_type = child.type
322 if node.parent and node.parent.type in STATEMENT:
323 if is_parent_function_or_class(node) and is_stub_body(node):
324 yield from self.visit_default(node)
325 else:
326 yield from self.line(+1)
327 yield from self.visit_default(node)
328 yield from self.line(-1)
330 else:
331 if node.parent and is_stub_suite(node.parent):
332 node.prefix = ""
333 yield from self.visit_default(node)
334 return
335 yield from self.line()
336 yield from self.visit_default(node)
338 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
339 """Visit `async def`, `async for`, `async with`."""
340 yield from self.line()
342 children = iter(node.children)
343 for child in children:
344 yield from self.visit(child)
346 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
347 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
348 # line.
349 break
351 internal_stmt = next(children)
352 yield from self.visit(internal_stmt)
354 def visit_decorators(self, node: Node) -> Iterator[Line]:
355 """Visit decorators."""
356 for child in node.children:
357 yield from self.line()
358 yield from self.visit(child)
360 def visit_power(self, node: Node) -> Iterator[Line]:
361 for idx, leaf in enumerate(node.children[:-1]):
362 next_leaf = node.children[idx + 1]
364 if not isinstance(leaf, Leaf):
365 continue
367 value = leaf.value.lower()
368 if (
369 leaf.type == token.NUMBER
370 and next_leaf.type == syms.trailer
371 # Ensure that we are in an attribute trailer
372 and next_leaf.children[0].type == token.DOT
373 # It shouldn't wrap hexadecimal, binary and octal literals
374 and not value.startswith(("0x", "0b", "0o"))
375 # It shouldn't wrap complex literals
376 and "j" not in value
377 ):
378 wrap_in_parentheses(node, leaf)
380 if Preview.remove_redundant_generator_parentheses in self.mode:
381 for child in node.children:
382 if (
383 child.type == syms.trailer
384 and len(child.children) == 3
385 and is_lpar_token(child.children[0])
386 and is_generator(child.children[1])
387 and is_rpar_token(child.children[2])
388 ):
389 maybe_make_parens_invisible_in_atom(
390 child.children[1],
391 parent=child,
392 mode=self.mode,
393 features=self.features,
394 remove_generator_parens=True,
395 )
397 remove_await_parens(node, mode=self.mode, features=self.features)
399 yield from self.visit_default(node)
401 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
402 """Remove a semicolon and put the other statement on a separate line."""
403 yield from self.line()
405 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
406 """End of file. Process outstanding comments and end with a newline."""
407 yield from self.visit_default(leaf)
408 yield from self.line()
410 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
411 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
412 if not any_open_brackets:
413 yield from self.line()
414 # STANDALONE_COMMENT nodes created by our special handling in
415 # normalize_fmt_off for comment-only blocks have fmt:off as the first
416 # line and fmt:on as the last line (each directive on its own line,
417 # not embedded in other text). These should be appended directly
418 # without calling visit_default, which would process their prefix and
419 # lose indentation. Normal STANDALONE_COMMENT nodes go through
420 # visit_default.
421 value = leaf.value
422 lines = value.splitlines()
423 is_fmt_off_block = (
424 len(lines) >= 2
425 and contains_fmt_directive(lines[0], FMT_OFF)
426 and contains_fmt_directive(lines[-1], FMT_ON)
427 )
428 if is_fmt_off_block:
429 is_after_invisible_lpar = (
430 leaf.fmt_pass_converted_first_leaf is None
431 and len(self.current_line.leaves) == 1
432 and self.current_line.leaves[0].type == token.LPAR
433 and not self.current_line.leaves[0].value
434 )
435 # This is a fmt:off/on block from normalize_fmt_off - we still need
436 # to process any prefix comments (like markdown comments) but append
437 # the fmt block itself directly to preserve its formatting
439 # Only process prefix comments if there actually is a prefix with comments
440 if leaf.prefix and any(
441 line.strip().startswith("#")
442 and not contains_fmt_directive(line.strip())
443 for line in leaf.prefix.split("\n")
444 ):
445 for comment in generate_comments(leaf, mode=self.mode):
446 yield from self.line()
447 self.current_line.append(comment)
448 yield from self.line()
449 # Clear the prefix since we've processed it as comments above
450 leaf.prefix = ""
452 self.current_line.append(leaf)
453 if not any_open_brackets or is_after_invisible_lpar:
454 yield from self.line()
455 else:
456 # Normal standalone comment - process through visit_default
457 yield from self.visit_default(leaf)
459 def visit_factor(self, node: Node) -> Iterator[Line]:
460 """Force parentheses between a unary op and a binary power:
462 -2 ** 8 -> -(2 ** 8)
463 """
464 _operator, operand = node.children
465 if (
466 operand.type == syms.power
467 and len(operand.children) == 3
468 and operand.children[1].type == token.DOUBLESTAR
469 ):
470 lpar = Leaf(token.LPAR, "(")
471 rpar = Leaf(token.RPAR, ")")
472 index = operand.remove() or 0
473 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
474 yield from self.visit_default(node)
476 def visit_tname(self, node: Node) -> Iterator[Line]:
477 """
478 Add potential parentheses around types in function parameter lists to be made
479 into real parentheses in case the type hint is too long to fit on a line
480 Examples:
481 def foo(a: int, b: float = 7): ...
483 ->
485 def foo(a: (int), b: (float) = 7): ...
486 """
487 if len(node.children) == 3 and maybe_make_parens_invisible_in_atom(
488 node.children[2], parent=node, mode=self.mode, features=self.features
489 ):
490 wrap_in_parentheses(node, node.children[2], visible=False)
492 yield from self.visit_default(node)
494 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
495 normalize_unicode_escape_sequences(leaf)
497 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
498 # We're ignoring docstrings with backslash newline escapes because changing
499 # indentation of those changes the AST representation of the code.
500 if self.mode.string_normalization:
501 docstring = normalize_string_prefix(leaf.value)
502 # We handle string normalization at the end of this method, but since
503 # what we do right now acts differently depending on quote style (ex.
504 # see padding logic below), there's a possibility for unstable
505 # formatting. To avoid a situation where this function formats a
506 # docstring differently on the second pass, normalize it early.
507 docstring = normalize_string_quotes(docstring)
508 else:
509 docstring = leaf.value
510 prefix = get_string_prefix(docstring)
511 docstring = docstring[len(prefix) :] # Remove the prefix
512 quote_char = docstring[0]
513 # A natural way to remove the outer quotes is to do:
514 # docstring = docstring.strip(quote_char)
515 # but that breaks on """""x""" (which is '""x').
516 # So we actually need to remove the first character and the next two
517 # characters but only if they are the same as the first.
518 quote_len = 1 if docstring[1] != quote_char else 3
519 docstring = docstring[quote_len:-quote_len]
520 docstring_started_empty = not docstring
521 indent = " " * 4 * self.current_line.depth
523 if is_multiline_string(leaf):
524 docstring = fix_multiline_docstring(docstring, indent)
525 else:
526 docstring = docstring.strip()
528 has_trailing_backslash = False
529 if docstring:
530 # Add some padding if the docstring starts / ends with a quote mark.
531 if docstring[0] == quote_char:
532 docstring = " " + docstring
533 if docstring[-1] == quote_char:
534 docstring += " "
535 if docstring[-1] == "\\":
536 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
537 if backslash_count % 2:
538 # Odd number of tailing backslashes, add some padding to
539 # avoid escaping the closing string quote.
540 docstring += " "
541 has_trailing_backslash = True
542 elif not docstring_started_empty:
543 docstring = " "
545 # We could enforce triple quotes at this point.
546 quote = quote_char * quote_len
548 # It's invalid to put closing single-character quotes on a new line.
549 if quote_len == 3:
550 # We need to find the length of the last line of the docstring
551 # to find if we can add the closing quotes to the line without
552 # exceeding the maximum line length.
553 # If docstring is one line, we don't put the closing quotes on a
554 # separate line because it looks ugly (#3320).
555 lines = docstring.splitlines()
556 last_line_length = len(lines[-1]) if docstring else 0
558 # If adding closing quotes would cause the last line to exceed
559 # the maximum line length, and the closing quote is not
560 # prefixed by a newline then put a line break before
561 # the closing quotes
562 if (
563 len(lines) > 1
564 and last_line_length + quote_len > self.mode.line_length
565 and len(indent) + quote_len <= self.mode.line_length
566 and not has_trailing_backslash
567 ):
568 if leaf.value[-1 - quote_len] == "\n":
569 leaf.value = prefix + quote + docstring + quote
570 else:
571 leaf.value = prefix + quote + docstring + "\n" + indent + quote
572 else:
573 leaf.value = prefix + quote + docstring + quote
574 else:
575 leaf.value = prefix + quote + docstring + quote
577 if self.mode.string_normalization and leaf.type == token.STRING:
578 leaf.value = normalize_string_prefix(leaf.value)
579 leaf.value = normalize_string_quotes(leaf.value)
580 yield from self.visit_default(leaf)
582 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
583 normalize_numeric_literal(leaf)
584 yield from self.visit_default(leaf)
586 def visit_atom(self, node: Node) -> Iterator[Line]:
587 """Visit any atom"""
588 if (
589 Preview.remove_redundant_generator_parentheses in self.mode
590 and _has_redundant_generator_parentheses(node)
591 ):
592 maybe_make_parens_invisible_in_atom(
593 node,
594 parent=node.parent or node,
595 mode=self.mode,
596 features=self.features,
597 )
599 if len(node.children) == 3:
600 first = node.children[0]
601 last = node.children[-1]
602 if (first.type == token.LSQB and last.type == token.RSQB) or (
603 first.type == token.LBRACE and last.type == token.RBRACE
604 ):
605 # Lists or sets of one item
606 maybe_make_parens_invisible_in_atom(
607 node.children[1],
608 parent=node,
609 mode=self.mode,
610 features=self.features,
611 )
613 yield from self.visit_default(node)
615 def visit_fstring(self, node: Node) -> Iterator[Line]:
616 # If the fstring was converted to a STANDALONE_COMMENT by
617 # normalize_fmt_off (e.g. it was inside a # fmt: off block),
618 # skip the fstring-to-string conversion and just visit normally.
619 if any(child.type == STANDALONE_COMMENT for child in node.children):
620 yield from self.visit_default(node)
621 return
622 # currently we don't want to format and split f-strings at all.
623 string_leaf = fstring_tstring_to_string(node)
624 node.replace(string_leaf)
625 if "\\" in string_leaf.value and any(
626 "\\" in str(child)
627 for child in node.children
628 if child.type == syms.fstring_replacement_field
629 ):
630 # string normalization doesn't account for nested quotes,
631 # causing breakages. skip normalization when nested quotes exist
632 yield from self.visit_default(string_leaf)
633 return
634 yield from self.visit_STRING(string_leaf)
636 def visit_tstring(self, node: Node) -> Iterator[Line]:
637 # If the tstring was converted to a STANDALONE_COMMENT by
638 # normalize_fmt_off, skip the conversion and just visit normally.
639 if any(child.type == STANDALONE_COMMENT for child in node.children):
640 yield from self.visit_default(node)
641 return
642 # currently we don't want to format and split t-strings at all.
643 string_leaf = fstring_tstring_to_string(node)
644 node.replace(string_leaf)
645 if "\\" in string_leaf.value and any(
646 "\\" in str(child)
647 for child in node.children
648 if child.type == syms.tstring_replacement_field
649 ):
650 # string normalization doesn't account for nested quotes,
651 # causing breakages. skip normalization when nested quotes exist
652 yield from self.visit_default(string_leaf)
653 return
654 yield from self.visit_STRING(string_leaf)
656 # TODO: Uncomment Implementation to format f-string children
657 # fstring_start = node.children[0]
658 # fstring_end = node.children[-1]
659 # assert isinstance(fstring_start, Leaf)
660 # assert isinstance(fstring_end, Leaf)
662 # quote_char = fstring_end.value[0]
663 # quote_idx = fstring_start.value.index(quote_char)
664 # prefix, quote = (
665 # fstring_start.value[:quote_idx],
666 # fstring_start.value[quote_idx:]
667 # )
669 # if not is_docstring(node, self.mode):
670 # prefix = normalize_string_prefix(prefix)
672 # assert quote == fstring_end.value
674 # is_raw_fstring = "r" in prefix or "R" in prefix
675 # middles = [
676 # leaf
677 # for leaf in node.leaves()
678 # if leaf.type == token.FSTRING_MIDDLE
679 # ]
681 # if self.mode.string_normalization:
682 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
684 # fstring_start.value = prefix + quote
685 # fstring_end.value = quote
687 # yield from self.visit_default(node)
689 def visit_comp_for(self, node: Node) -> Iterator[Line]:
690 if Preview.wrap_comprehension_in in self.mode:
691 normalize_invisible_parens(
692 node, parens_after={"in"}, mode=self.mode, features=self.features
693 )
694 yield from self.visit_default(node)
696 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
697 yield from self.visit_comp_for(node)
699 def __post_init__(self) -> None:
700 """You are in a twisty little maze of passages."""
701 self.current_line = Line(mode=self.mode)
703 v = self.visit_stmt
704 Ø: set[str] = set()
705 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
706 self.visit_if_stmt = partial(
707 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
708 )
709 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
710 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
711 self.visit_try_stmt = partial(
712 v, keywords={"try", "except", "else", "finally"}, parens=Ø
713 )
714 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
715 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
716 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
718 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
719 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
720 self.visit_yield_expr = partial(
721 v,
722 keywords=Ø,
723 parens=(
724 {"yield"} if Preview.parenthesize_tuple_in_yield in self.mode else Ø
725 ),
726 )
727 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
728 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
729 self.visit_async_funcdef = self.visit_async_stmt
730 self.visit_decorated = self.visit_decorators
732 # PEP 634
733 self.visit_match_stmt = self.visit_match_case
734 self.visit_case_block = self.visit_match_case
735 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
738# Remove when `simplify_power_operator_hugging` becomes stable.
739def _hugging_power_ops_line_to_string(
740 line: Line,
741 features: Collection[Feature],
742 mode: Mode,
743) -> str | None:
744 try:
745 return line_to_string(next(hug_power_op(line, features, mode)))
746 except CannotTransform:
747 return None
750def transform_line(
751 line: Line, mode: Mode, features: Collection[Feature] = ()
752) -> Iterator[Line]:
753 """Transform a `line`, potentially splitting it into many lines.
755 They should fit in the allotted `line_length` but might not be able to.
757 `features` are syntactical features that may be used in the output.
758 """
759 if line.is_comment:
760 yield line
761 return
763 line_str = line_to_string(line)
765 if Preview.simplify_power_operator_hugging in mode:
766 line_str_hugging_power_ops = line_str
767 else:
768 # We need the line string when power operators are hugging to determine if we
769 # should split the line. Default to line_str, if no power operator are present
770 # on the line.
771 line_str_hugging_power_ops = (
772 _hugging_power_ops_line_to_string(line, features, mode) or line_str
773 )
775 ll = mode.line_length
776 sn = mode.string_normalization
777 string_merge = StringMerger(ll, sn)
778 string_paren_strip = StringParenStripper(ll, sn)
779 string_split = StringSplitter(ll, sn)
780 string_paren_wrap = StringParenWrapper(ll, sn)
782 transformers: list[Transformer]
783 if (
784 not line.contains_uncollapsable_type_comments()
785 and not line.should_split_rhs
786 and not line.magic_trailing_comma
787 and (
788 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
789 or line.contains_unsplittable_type_ignore()
790 )
791 and not (line.inside_brackets and line.contains_standalone_comments())
792 and not line.contains_implicit_multiline_string_with_comments()
793 ):
794 # Only apply basic string preprocessing, since lines shouldn't be split here.
795 if Preview.string_processing in mode:
796 transformers = [string_merge, string_paren_strip]
797 else:
798 transformers = []
799 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
800 transformers = [left_hand_split]
801 else:
803 def _rhs(
804 self: object, line: Line, features: Collection[Feature], mode: Mode
805 ) -> Iterator[Line]:
806 """Wraps calls to `right_hand_split`.
808 The calls increasingly `omit` right-hand trailers (bracket pairs with
809 content), meaning the trailers get glued together to split on another
810 bracket pair instead.
811 """
812 for omit in generate_trailers_to_omit(line, mode.line_length):
813 lines = list(right_hand_split(line, mode, features, omit=omit))
814 # Note: this check is only able to figure out if the first line of the
815 # *current* transformation fits in the line length. This is true only
816 # for simple cases. All others require running more transforms via
817 # `transform_line()`. This check doesn't know if those would succeed.
818 if is_line_short_enough(lines[0], mode=mode) or (
819 omit and _over_length_only_due_to_subscript_comment(lines[0], mode)
820 ):
821 yield from lines
822 return
824 # All splits failed, best effort split with no omits.
825 # This mostly happens to multiline strings that are by definition
826 # reported as not fitting a single line, as well as lines that contain
827 # trailing commas (those have to be exploded).
828 yield from right_hand_split(line, mode, features=features)
830 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
831 # __name__ attribute which is needed in `run_transformer` further down.
832 # Unfortunately a nested class breaks mypyc too. So a class must be created
833 # via type ... https://github.com/mypyc/mypyc/issues/884
834 rhs = type("rhs", (), {"__call__": _rhs})()
836 if Preview.string_processing in mode:
837 if line.inside_brackets:
838 transformers = [
839 string_merge,
840 string_paren_strip,
841 string_split,
842 delimiter_split,
843 standalone_comment_split,
844 string_paren_wrap,
845 rhs,
846 ]
847 else:
848 transformers = [
849 string_merge,
850 string_paren_strip,
851 string_split,
852 string_paren_wrap,
853 rhs,
854 ]
855 else:
856 if line.inside_brackets:
857 transformers = [delimiter_split, standalone_comment_split, rhs]
858 else:
859 transformers = [rhs]
861 if Preview.simplify_power_operator_hugging not in mode:
862 # It's always safe to attempt hugging of power operations and pretty much every
863 # line could match.
864 transformers.append(hug_power_op)
866 for transform in transformers:
867 # We are accumulating lines in `result` because we might want to abort
868 # mission and return the original line in the end, or attempt a different
869 # split altogether.
870 try:
871 result = run_transformer(line, transform, mode, features, line_str=line_str)
872 except CannotTransform:
873 continue
874 else:
875 yield from result
876 break
878 else:
879 # A leftover standalone comment would render inline and break the
880 # second pass with a parse error (#4296). Force a split so the
881 # output is at least valid Python.
882 if line.contains_standalone_comments():
883 yield from _force_standalone_comment_split(line)
884 else:
885 yield line
888def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
889 """If a funcdef has a magic trailing comma in the return type, then we should first
890 split the line with rhs to respect the comma.
891 """
892 return_type_leaves: list[Leaf] = []
893 in_return_type = False
895 for leaf in line.leaves:
896 if leaf.type == token.COLON:
897 in_return_type = False
898 if in_return_type:
899 return_type_leaves.append(leaf)
900 if leaf.type == token.RARROW:
901 in_return_type = True
903 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
904 # couple lines from it.
905 result = Line(mode=line.mode, depth=line.depth)
906 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
907 for leaf in return_type_leaves:
908 result.append(
909 leaf,
910 preformatted=True,
911 track_bracket=id(leaf) in leaves_to_track,
912 )
914 # we could also return true if the line is too long, and the return type is longer
915 # than the param list. Or if `should_split_rhs` returns True.
916 return result.magic_trailing_comma is not None
919class _BracketSplitComponent(Enum):
920 head = auto()
921 body = auto()
922 tail = auto()
925def left_hand_split(
926 line: Line, _features: Collection[Feature], mode: Mode
927) -> Iterator[Line]:
928 """Split line into many lines, starting with the first matching bracket pair.
930 Note: this usually looks weird, only use this for function definitions.
931 Prefer RHS otherwise. This is why this function is not symmetrical with
932 :func:`right_hand_split` which also handles optional parentheses.
933 """
934 for leaf_type in [token.LPAR, token.LSQB]:
935 tail_leaves: list[Leaf] = []
936 body_leaves: list[Leaf] = []
937 head_leaves: list[Leaf] = []
938 current_leaves = head_leaves
939 matching_bracket: Leaf | None = None
940 depth = 0
941 for index, leaf in enumerate(line.leaves):
942 if index == 2 and leaf.type == token.LSQB:
943 # A [ at index 2 means this is a type param, so start
944 # tracking the depth
945 depth += 1
946 elif depth > 0:
947 if leaf.type == token.LSQB:
948 depth += 1
949 elif leaf.type == token.RSQB:
950 depth -= 1
951 if (
952 current_leaves is body_leaves
953 and leaf.type in CLOSING_BRACKETS
954 and leaf.opening_bracket is matching_bracket
955 and isinstance(matching_bracket, Leaf)
956 # If the code is still on LPAR and we are inside a type
957 # param, ignore the match since this is searching
958 # for the function arguments
959 and not (leaf_type == token.LPAR and depth > 0)
960 ):
961 ensure_visible(leaf)
962 ensure_visible(matching_bracket)
963 current_leaves = tail_leaves if body_leaves else head_leaves
964 current_leaves.append(leaf)
965 if current_leaves is head_leaves:
966 if leaf.type == leaf_type and (
967 not (leaf_type == token.LPAR and depth > 0)
968 ):
969 matching_bracket = leaf
970 current_leaves = body_leaves
971 if matching_bracket and tail_leaves:
972 break
973 if not matching_bracket or not tail_leaves:
974 raise CannotSplit("No brackets found")
976 head = bracket_split_build_line(
977 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
978 )
979 body = bracket_split_build_line(
980 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
981 )
982 tail = bracket_split_build_line(
983 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
984 )
985 bracket_split_succeeded_or_raise(head, body, tail)
986 for result in (head, body, tail):
987 if result:
988 yield result
991def right_hand_split(
992 line: Line,
993 mode: Mode,
994 features: Collection[Feature] = (),
995 omit: Collection[LeafID] = (),
996) -> Iterator[Line]:
997 """Split line into many lines, starting with the last matching bracket pair.
999 If the split was by optional parentheses, attempt splitting without them, too.
1000 `omit` is a collection of closing bracket IDs that shouldn't be considered for
1001 this split.
1003 Note: running this function modifies `bracket_depth` on the leaves of `line`.
1004 """
1005 rhs_result = _first_right_hand_split(line, omit=omit)
1006 yield from _maybe_split_omitting_optional_parens(
1007 rhs_result, line, mode, features=features, omit=omit
1008 )
1011def _first_right_hand_split(
1012 line: Line,
1013 omit: Collection[LeafID] = (),
1014) -> RHSResult:
1015 """Split the line into head, body, tail starting with the last bracket pair.
1017 Note: this function should not have side effects. It's relied upon by
1018 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
1019 splitting on the right side of an assignment statement.
1020 """
1021 tail_leaves: list[Leaf] = []
1022 body_leaves: list[Leaf] = []
1023 head_leaves: list[Leaf] = []
1024 current_leaves = tail_leaves
1025 opening_bracket: Leaf | None = None
1026 closing_bracket: Leaf | None = None
1027 for leaf in reversed(line.leaves):
1028 if current_leaves is body_leaves:
1029 if leaf is opening_bracket:
1030 current_leaves = head_leaves if body_leaves else tail_leaves
1031 current_leaves.append(leaf)
1032 if current_leaves is tail_leaves:
1033 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
1034 opening_bracket = leaf.opening_bracket
1035 closing_bracket = leaf
1036 current_leaves = body_leaves
1037 if not (opening_bracket and closing_bracket and head_leaves):
1038 # If there is no opening or closing_bracket that means the split failed and
1039 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
1040 # the matching `opening_bracket` wasn't available on `line` anymore.
1041 raise CannotSplit("No brackets found")
1043 tail_leaves.reverse()
1044 body_leaves.reverse()
1045 head_leaves.reverse()
1047 body: Line | None = None
1048 if (
1049 Preview.hug_parens_with_braces_and_square_brackets in line.mode
1050 and tail_leaves[0].value
1051 and tail_leaves[0].opening_bracket is head_leaves[-1]
1052 ):
1053 inner_body_leaves = list(body_leaves)
1054 hugged_opening_leaves: list[Leaf] = []
1055 hugged_closing_leaves: list[Leaf] = []
1056 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
1057 unpacking_offset: int = 1 if is_unpacking else 0
1058 while (
1059 len(inner_body_leaves) >= 2 + unpacking_offset
1060 and inner_body_leaves[-1].type in CLOSING_BRACKETS
1061 and inner_body_leaves[-1].opening_bracket
1062 is inner_body_leaves[unpacking_offset]
1063 ):
1064 if unpacking_offset:
1065 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1066 unpacking_offset = 0
1067 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1068 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
1070 if hugged_opening_leaves and inner_body_leaves:
1071 inner_body = bracket_split_build_line(
1072 inner_body_leaves,
1073 line,
1074 hugged_opening_leaves[-1],
1075 component=_BracketSplitComponent.body,
1076 )
1077 if (
1078 line.mode.magic_trailing_comma
1079 and inner_body_leaves[-1].type == token.COMMA
1080 ):
1081 should_hug = True
1082 else:
1083 line_length = line.mode.line_length - sum(
1084 len(str(leaf))
1085 for leaf in hugged_opening_leaves + hugged_closing_leaves
1086 )
1087 if is_line_short_enough(
1088 inner_body, mode=replace(line.mode, line_length=line_length)
1089 ):
1090 # Do not hug if it fits on a single line.
1091 should_hug = False
1092 else:
1093 should_hug = True
1094 if should_hug and (
1095 _hugging_merges_type_ignores(line, head_leaves, hugged_opening_leaves)
1096 or _hugging_merges_type_ignores(
1097 line, hugged_closing_leaves, tail_leaves
1098 )
1099 ):
1100 # Hugging joins these leaves onto one physical line, and their
1101 # trailing comments come along. `type: ignore` is recorded per
1102 # line by the AST, so two of them landing on the same line would
1103 # drop one and make the output non-equivalent.
1104 should_hug = False
1105 if should_hug:
1106 body_leaves = inner_body_leaves
1107 head_leaves.extend(hugged_opening_leaves)
1108 tail_leaves = hugged_closing_leaves + tail_leaves
1109 body = inner_body # No need to re-calculate the body again later.
1111 head = bracket_split_build_line(
1112 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1113 )
1114 if body is None:
1115 body = bracket_split_build_line(
1116 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1117 )
1118 tail = bracket_split_build_line(
1119 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1120 )
1121 bracket_split_succeeded_or_raise(head, body, tail)
1122 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1125def _maybe_split_omitting_optional_parens(
1126 rhs: RHSResult,
1127 line: Line,
1128 mode: Mode,
1129 features: Collection[Feature] = (),
1130 omit: Collection[LeafID] = (),
1131) -> Iterator[Line]:
1132 if (
1133 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1134 # the opening bracket is an optional paren
1135 and rhs.opening_bracket.type == token.LPAR
1136 and not rhs.opening_bracket.value
1137 # the closing bracket is an optional paren
1138 and rhs.closing_bracket.type == token.RPAR
1139 and not rhs.closing_bracket.value
1140 # it's not an import (optional parens are the only thing we can split on
1141 # in this case; attempting a split without them is a waste of time)
1142 and not line.is_import
1143 # and we can actually remove the parens
1144 and can_omit_invisible_parens(rhs, mode.line_length, mode)
1145 ):
1146 omit = {id(rhs.closing_bracket), *omit}
1147 try:
1148 # The RHSResult Omitting Optional Parens.
1149 rhs_oop = _first_right_hand_split(line, omit=omit)
1150 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1151 yield from _maybe_split_omitting_optional_parens(
1152 rhs_oop, line, mode, features=features, omit=omit
1153 )
1154 return
1156 except CannotSplit as e:
1157 # For chained assignments we want to use the previous successful split
1158 if line.is_chained_assignment:
1159 pass
1161 elif (
1162 not can_be_split(rhs.body)
1163 and not is_line_short_enough(rhs.body, mode=mode)
1164 and not (
1165 Preview.wrap_long_dict_values_in_parens in mode
1166 and rhs.opening_bracket.parent
1167 and rhs.opening_bracket.parent.parent
1168 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1169 )
1170 and not (
1171 rhs.opening_bracket.parent
1172 and rhs.opening_bracket.parent.parent
1173 and rhs.opening_bracket.parent.parent.type == syms.case_block
1174 )
1175 ):
1176 raise CannotSplit(
1177 "Splitting failed, body is still too long and can't be split."
1178 ) from e
1180 elif (
1181 rhs.head.contains_multiline_strings()
1182 or rhs.tail.contains_multiline_strings()
1183 ):
1184 raise CannotSplit(
1185 "The current optional pair of parentheses is bound to fail to"
1186 " satisfy the splitting algorithm because the head or the tail"
1187 " contains multiline strings which by definition never fit one"
1188 " line."
1189 ) from e
1191 ensure_visible(rhs.opening_bracket)
1192 ensure_visible(rhs.closing_bracket)
1193 for result in (rhs.head, rhs.body, rhs.tail):
1194 if result:
1195 yield result
1198def _prefer_split_rhs_oop_over_rhs(
1199 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1200) -> bool:
1201 """
1202 Returns whether we should prefer the result from a split omitting optional parens
1203 (rhs_oop) over the original (rhs).
1204 """
1205 # contains unsplittable type ignore
1206 if (
1207 rhs_oop.head.contains_unsplittable_type_ignore()
1208 or rhs_oop.body.contains_unsplittable_type_ignore()
1209 or rhs_oop.tail.contains_unsplittable_type_ignore()
1210 ):
1211 return True
1213 # Retain optional parens around dictionary values
1214 if (
1215 Preview.wrap_long_dict_values_in_parens in mode
1216 and rhs.opening_bracket.parent
1217 and rhs.opening_bracket.parent.parent
1218 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1219 and rhs.body.bracket_tracker.delimiters
1220 ):
1221 # Unless the split is inside the key
1222 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1224 # the split is right after `=`
1225 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1226 return True
1228 # the left side of assignment contains brackets
1229 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1230 return True
1232 # the left side of assignment is short enough (the -1 is for the ending optional
1233 # paren)
1234 if not is_line_short_enough(
1235 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1236 ):
1237 return True
1239 # the left side of assignment won't explode further because of magic trailing comma
1240 if rhs.head.magic_trailing_comma is not None:
1241 return True
1243 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1244 # the body
1245 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1246 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1247 token.EQUAL
1248 )
1249 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1250 return False
1252 has_closing_bracket_after_assign = False
1253 for leaf in reversed(rhs_oop.head.leaves):
1254 if leaf.type == token.EQUAL:
1255 break
1256 if leaf.type in CLOSING_BRACKETS:
1257 has_closing_bracket_after_assign = True
1258 break
1259 return (
1260 # contains matching brackets after the `=` (done by checking there is a
1261 # closing bracket)
1262 has_closing_bracket_after_assign
1263 or (
1264 # the split is actually from inside the optional parens (done by checking
1265 # the first line still contains the `=`)
1266 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1267 # the first line is short enough
1268 and is_line_short_enough(rhs_oop.head, mode=mode)
1269 )
1270 )
1273def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1274 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1276 Do nothing otherwise.
1278 A left- or right-hand split is based on a pair of brackets. Content before
1279 (and including) the opening bracket is left on one line, content inside the
1280 brackets is put on a separate line, and finally content starting with and
1281 following the closing bracket is put on a separate line.
1283 Those are called `head`, `body`, and `tail`, respectively. If the split
1284 produced the same line (all content in `head`) or ended up with an empty `body`
1285 and the `tail` is just the closing bracket, then it's considered failed.
1286 """
1287 tail_len = len(str(tail).strip())
1288 if not body:
1289 if tail_len == 0:
1290 raise CannotSplit("Splitting brackets produced the same line")
1292 elif tail_len < 3:
1293 raise CannotSplit(
1294 f"Splitting brackets on an empty body to save {tail_len} characters is"
1295 " not worth it"
1296 )
1299def _ensure_trailing_comma(
1300 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1301) -> bool:
1302 if not leaves:
1303 return False
1304 # Ensure a trailing comma for imports
1305 if original.is_import:
1306 return True
1307 # ...and standalone function arguments
1308 if not original.is_def:
1309 return False
1310 if opening_bracket.value != "(":
1311 return False
1312 # Don't add commas if we already have any commas
1313 if any(
1314 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1315 ):
1316 return False
1318 # Find a leaf with a parent (comments don't have parents)
1319 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1320 if leaf_with_parent is None:
1321 return True
1322 # Don't add commas inside parenthesized return annotations
1323 if get_annotation_type(leaf_with_parent) == "return":
1324 return False
1325 # Don't add commas inside PEP 604 unions
1326 if (
1327 leaf_with_parent.parent
1328 and leaf_with_parent.parent.next_sibling
1329 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1330 ):
1331 return False
1332 return True
1335def _hugging_merges_type_ignores(line: Line, *leaf_groups: Iterable[Leaf]) -> bool:
1336 """Return True if hugging these groups would put two `type: ignore`s on a line."""
1337 seen = 0
1338 for leaves in leaf_groups:
1339 for leaf in leaves:
1340 for comment in line.comments_after(leaf):
1341 if is_type_ignore_comment(comment, mode=line.mode):
1342 seen += 1
1343 if seen > 1:
1344 return True
1345 return False
1348def bracket_split_build_line(
1349 leaves: list[Leaf],
1350 original: Line,
1351 opening_bracket: Leaf,
1352 *,
1353 component: _BracketSplitComponent,
1354) -> Line:
1355 """Return a new line with given `leaves` and respective comments from `original`.
1357 If it's the head component, brackets will be tracked so trailing commas are
1358 respected.
1360 If it's the body component, the result line is one-indented inside brackets and as
1361 such has its first leaf's prefix normalized and a trailing comma added when
1362 expected.
1363 """
1364 result = Line(mode=original.mode, depth=original.depth)
1365 if component is _BracketSplitComponent.body:
1366 result.inside_brackets = True
1367 result.depth += 1
1368 if _ensure_trailing_comma(leaves, original, opening_bracket):
1369 for i in range(len(leaves) - 1, -1, -1):
1370 if leaves[i].type == STANDALONE_COMMENT:
1371 continue
1373 if leaves[i].type != token.COMMA:
1374 new_comma = Leaf(token.COMMA, ",")
1375 leaves.insert(i + 1, new_comma)
1376 break
1378 leaves_to_track: set[LeafID] = set()
1379 if component is _BracketSplitComponent.head:
1380 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1381 # Populate the line
1382 for leaf in leaves:
1383 result.append(
1384 leaf,
1385 preformatted=True,
1386 track_bracket=id(leaf) in leaves_to_track,
1387 )
1388 for comment_after in original.comments_after(leaf):
1389 result.append(comment_after, preformatted=True)
1390 if component is _BracketSplitComponent.body and should_split_line(
1391 result, opening_bracket
1392 ):
1393 result.should_split_rhs = True
1394 return result
1397def dont_increase_indentation(split_func: Transformer) -> Transformer:
1398 """Normalize prefix of the first leaf in every line returned by `split_func`.
1400 This is a decorator over relevant split functions.
1401 """
1403 @wraps(split_func)
1404 def split_wrapper(
1405 line: Line, features: Collection[Feature], mode: Mode
1406 ) -> Iterator[Line]:
1407 for split_line in split_func(line, features, mode):
1408 split_line.leaves[0].prefix = ""
1409 yield split_line
1411 return split_wrapper
1414def _get_last_non_comment_leaf(line: Line) -> int | None:
1415 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1416 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1417 return leaf_idx
1418 return None
1421def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1422 if is_vararg(leaf, within={syms.typedargslist}):
1423 return Feature.TRAILING_COMMA_IN_DEF in features
1424 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1425 return Feature.TRAILING_COMMA_IN_CALL in features
1426 return True
1429def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1430 if (
1431 safe
1432 and delimiter_priority == COMMA_PRIORITY
1433 and line.leaves[-1].type != token.COMMA
1434 and line.leaves[-1].type != STANDALONE_COMMENT
1435 ):
1436 new_comma = Leaf(token.COMMA, ",")
1437 line.append(new_comma)
1438 return line
1441MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1444def _can_defer_lone_comparator_to_rhs(line: Line, mode: Mode) -> bool:
1445 """Return True if the lone comparator on `line` can defer to right_hand_split.
1447 Caller has already established exactly one delimiter at
1448 `COMPARATOR_PRIORITY`. We defer only when:
1450 - the LHS up to the comparator has no opening brackets, so the existing
1451 "break before the comparator" wouldn't produce a balanced two-sided
1452 split anyway, and
1453 - `right_hand_split` would produce a head that fits in the line length,
1454 so we don't strand `if t` on its own line just to push it back onto an
1455 overflowing single line when the RHS bracket can't be exploded
1456 usefully (e.g. an empty `decode()` paren).
1457 """
1458 past_comparator = False
1459 for leaf in line.leaves:
1460 if leaf.type in OPENING_BRACKETS and not past_comparator:
1461 return False
1462 if not past_comparator and (
1463 line.bracket_tracker.delimiters.get(id(leaf)) == COMPARATOR_PRIORITY
1464 ):
1465 past_comparator = True
1466 try:
1467 rhs = _first_right_hand_split(line)
1468 except CannotSplit:
1469 return False
1470 return is_line_short_enough(rhs.head, mode=mode)
1473@dont_increase_indentation
1474def delimiter_split(
1475 line: Line, features: Collection[Feature], mode: Mode
1476) -> Iterator[Line]:
1477 """Split according to delimiters of the highest priority.
1479 If the appropriate Features are given, the split will add trailing commas
1480 also in function signatures and calls that contain `*` and `**`.
1481 """
1482 if len(line.leaves) == 0:
1483 raise CannotSplit("Line empty") from None
1484 last_leaf = line.leaves[-1]
1486 bt = line.bracket_tracker
1487 try:
1488 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1489 except ValueError:
1490 raise CannotSplit("No delimiters found") from None
1492 if (
1493 delimiter_priority == DOT_PRIORITY
1494 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1495 ):
1496 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1498 if (
1499 Preview.hug_comparator in mode
1500 and delimiter_priority == COMPARATOR_PRIORITY
1501 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1502 and _can_defer_lone_comparator_to_rhs(line, mode)
1503 ):
1504 raise CannotSplit("Bracketed RHS will explode via right_hand_split")
1506 current_line = Line(
1507 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1508 )
1509 lowest_depth = sys.maxsize
1510 trailing_comma_safe = True
1512 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1513 """Append `leaf` to current line or to new line if appending impossible."""
1514 nonlocal current_line
1515 try:
1516 current_line.append_safe(leaf, preformatted=True)
1517 except ValueError:
1518 yield current_line
1520 current_line = Line(
1521 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1522 )
1523 current_line.append(leaf)
1525 def append_comments(leaf: Leaf) -> Iterator[Line]:
1526 for comment_after in line.comments_after(leaf):
1527 yield from append_to_line(comment_after)
1529 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1530 for leaf_idx, leaf in enumerate(line.leaves):
1531 yield from append_to_line(leaf)
1533 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1534 id(line.leaves[leaf_idx - 1])
1535 )
1536 if (
1537 previous_priority != delimiter_priority
1538 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1539 ):
1540 yield from append_comments(leaf)
1542 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1543 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1544 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1546 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1547 current_line = _safe_add_trailing_comma(
1548 trailing_comma_safe, delimiter_priority, current_line
1549 )
1551 leaf_priority = bt.delimiters.get(id(leaf))
1552 if leaf_priority == delimiter_priority:
1553 if (
1554 leaf_idx + 1 < len(line.leaves)
1555 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1556 ):
1557 yield from append_comments(line.leaves[leaf_idx + 1])
1559 yield current_line
1560 current_line = Line(
1561 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1562 )
1564 if current_line:
1565 current_line = _safe_add_trailing_comma(
1566 trailing_comma_safe, delimiter_priority, current_line
1567 )
1568 yield current_line
1571@dont_increase_indentation
1572def standalone_comment_split(
1573 line: Line, features: Collection[Feature], mode: Mode
1574) -> Iterator[Line]:
1575 """Split standalone comments from the rest of the line."""
1576 if not line.contains_standalone_comments():
1577 raise CannotSplit("Line does not have any standalone comments")
1579 current_line = Line(
1580 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1581 )
1583 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1584 """Append `leaf` to current line or to new line if appending impossible."""
1585 nonlocal current_line
1586 try:
1587 current_line.append_safe(leaf, preformatted=True)
1588 except ValueError:
1589 yield current_line
1591 current_line = Line(
1592 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1593 )
1594 current_line.append(leaf)
1596 for leaf in line.leaves:
1597 yield from append_to_line(leaf)
1599 for comment_after in line.comments_after(leaf):
1600 yield from append_to_line(comment_after)
1602 if current_line:
1603 yield current_line
1606def _force_standalone_comment_split(line: Line) -> Iterator[Line]:
1607 """Last-resort split at every standalone-comment boundary."""
1608 current_line = Line(
1609 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1610 )
1611 for leaf in line.leaves:
1612 if current_line.leaves and (
1613 leaf.type == STANDALONE_COMMENT or current_line.is_comment
1614 ):
1615 yield current_line
1616 current_line = Line(
1617 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1618 )
1619 current_line.append(leaf, preformatted=True)
1620 for comment_after in line.comments_after(leaf):
1621 current_line.append(comment_after, preformatted=True)
1622 if current_line:
1623 yield current_line
1626def _is_parenthesized_lambda_or_ternary(node: LN) -> bool:
1627 """Whether `node` is an atom wrapping a lambda or conditional expression,
1628 looking through any redundant nested parentheses.
1630 As a comprehension's iterable, such an expression must keep at least one pair
1631 of parentheses: without them the trailing `for`/`if` clauses would be parsed
1632 as part of the lambda body (or break the ternary), producing invalid code.
1633 """
1634 while (
1635 node.type == syms.atom
1636 and len(node.children) == 3
1637 and is_lpar_token(node.children[0])
1638 and is_rpar_token(node.children[-1])
1639 ):
1640 middle = node.children[1]
1641 if middle.type in {syms.test, syms.lambdef}:
1642 return True
1643 node = middle
1644 return False
1647def _has_redundant_generator_parentheses(node: LN) -> bool:
1648 """Whether `node` adds parentheses around a parenthesized generator."""
1649 if (
1650 node.type != syms.atom
1651 or len(node.children) != 3
1652 or not is_lpar_token(node.children[0])
1653 or not is_rpar_token(node.children[-1])
1654 ):
1655 return False
1657 middle = node.children[1]
1658 return is_generator(middle) or _has_redundant_generator_parentheses(middle)
1661def normalize_invisible_parens(
1662 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1663) -> None:
1664 """Make existing optional parentheses invisible or create new ones.
1666 `parens_after` is a set of string leaf values immediately after which parens
1667 should be put.
1669 Standardizes on visible parentheses for single-element tuples, and keeps
1670 existing visible parentheses for other tuples and generator expressions.
1671 """
1672 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1673 if contains_fmt_directive(pc.value, FMT_OFF):
1674 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1675 return
1677 # The multiple context managers grammar has a different pattern, thus this is
1678 # separate from the for-loop below. This possibly wraps them in invisible parens,
1679 # and later will be removed in remove_with_parens when needed.
1680 if node.type == syms.with_stmt:
1681 _maybe_wrap_cms_in_parens(node, mode, features)
1683 check_lpar = False
1684 for index, child in enumerate(list(node.children)):
1685 # Fixes a bug where invisible parens are not properly stripped from
1686 # assignment statements that contain type annotations.
1687 if isinstance(child, Node) and child.type == syms.annassign:
1688 normalize_invisible_parens(
1689 child, parens_after=parens_after, mode=mode, features=features
1690 )
1692 # Fixes a bug where invisible parens are not properly wrapped around
1693 # case blocks.
1694 if isinstance(child, Node) and child.type == syms.case_block:
1695 normalize_invisible_parens(
1696 child, parens_after={"case"}, mode=mode, features=features
1697 )
1699 # Add parentheses around if guards in case blocks
1700 if isinstance(child, Node) and child.type == syms.guard:
1701 normalize_invisible_parens(
1702 child, parens_after={"if"}, mode=mode, features=features
1703 )
1705 # Add parentheses around long tuple unpacking in assignments.
1706 if (
1707 index == 0
1708 and isinstance(child, Node)
1709 and child.type == syms.testlist_star_expr
1710 ):
1711 check_lpar = True
1713 if (
1714 index == 0
1715 and isinstance(child, Node)
1716 and child.type == syms.atom
1717 and node.type == syms.expr_stmt
1718 and not _atom_has_magic_trailing_comma(child, mode)
1719 and not _is_atom_multiline(child)
1720 ):
1721 if _is_parenthesized_annotation_target(node, child):
1722 # One pair is what makes the target non-simple, so any nesting
1723 # inside it is redundant and goes.
1724 inner = child.children[1]
1725 if isinstance(inner, Node) and inner.type == syms.atom:
1726 maybe_make_parens_invisible_in_atom(
1727 inner,
1728 parent=child,
1729 mode=mode,
1730 features=features,
1731 )
1732 elif maybe_make_parens_invisible_in_atom(
1733 child,
1734 parent=node,
1735 mode=mode,
1736 features=features,
1737 remove_brackets_around_comma=True,
1738 allow_star_expr=True,
1739 ):
1740 wrap_in_parentheses(node, child, visible=False, index=index)
1742 if check_lpar:
1743 if (
1744 child.type == syms.atom
1745 and node.type == syms.for_stmt
1746 and isinstance(child.prev_sibling, Leaf)
1747 and child.prev_sibling.type == token.NAME
1748 and child.prev_sibling.value == "for"
1749 ):
1750 if maybe_make_parens_invisible_in_atom(
1751 child,
1752 parent=node,
1753 mode=mode,
1754 features=features,
1755 remove_brackets_around_comma=True,
1756 ):
1757 wrap_in_parentheses(node, child, visible=False, index=index)
1758 elif isinstance(child, Node) and node.type == syms.with_stmt:
1759 remove_with_parens(child, node, mode=mode, features=features)
1760 elif (
1761 isinstance(child, Node)
1762 and node.type == syms.yield_expr
1763 and child.type == syms.yield_arg
1764 and Preview.parenthesize_tuple_in_yield in mode
1765 ):
1766 if (
1767 len(child.children) == 1
1768 and child.children[0].type != syms.atom
1769 and is_one_tuple(child.children[0])
1770 ):
1771 wrap_in_parentheses(node, child, visible=True, index=index)
1772 elif child.type == syms.atom:
1773 if "in" in parens_after and _is_parenthesized_lambda_or_ternary(child):
1774 # A lambda or conditional expression used as a comprehension's
1775 # iterable must keep at least one pair of parentheses, otherwise
1776 # the trailing `for`/`if` clauses get absorbed into it and the
1777 # code becomes invalid. Any extra nested pairs are redundant, so
1778 # collapse them while keeping exactly one visible pair.
1779 maybe_make_parens_invisible_in_atom(
1780 child, parent=node, mode=mode, features=features
1781 )
1782 opening = child.children[0]
1783 closing = child.children[-1]
1784 if is_lpar_token(opening) and is_rpar_token(closing):
1785 opening.value = "("
1786 closing.value = ")"
1787 elif maybe_make_parens_invisible_in_atom(
1788 child, parent=node, mode=mode, features=features
1789 ):
1790 wrap_in_parentheses(node, child, visible=False, index=index)
1791 elif is_one_tuple(child):
1792 wrap_in_parentheses(node, child, visible=True, index=index)
1793 elif node.type == syms.import_from:
1794 _normalize_import_from(node, child, index)
1795 break
1796 elif (
1797 index == 1
1798 and child.type == token.STAR
1799 and node.type == syms.except_clause
1800 ):
1801 # In except* (PEP 654), the star is actually part of
1802 # of the keyword. So we need to skip the insertion of
1803 # invisible parentheses to work more precisely.
1804 continue
1806 elif (
1807 isinstance(child, Leaf)
1808 and child.next_sibling is not None
1809 and child.next_sibling.type == token.COLON
1810 and child.value == "case"
1811 ):
1812 # A special patch for "case case:" scenario, the second occurrence
1813 # of case will be not parsed as a Python keyword.
1814 break
1816 elif isinstance(child, Node) and child.type == syms.guard:
1817 # Guard nodes handle their own inner wrapping. Wrapping the guard
1818 # itself can produce invalid output when the case pattern splits.
1819 pass
1821 elif not is_multiline_string(child):
1822 if (
1823 Preview.fix_if_guard_explosion_in_case_statement in mode
1824 and node.type == syms.guard
1825 ):
1826 mock_line = Line(mode=mode)
1827 for leaf in child.leaves():
1828 mock_line.append(leaf)
1829 # If it's a guard AND it's short, we DON'T wrap
1830 if not is_line_short_enough(mock_line, mode=mode):
1831 wrap_in_parentheses(node, child, visible=False, index=index)
1832 else:
1833 wrap_in_parentheses(node, child, visible=False, index=index)
1835 comma_check = child.type == token.COMMA
1837 check_lpar = isinstance(child, Leaf) and (
1838 child.value in parens_after or comma_check
1839 )
1842def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1843 # "import from" nodes store parentheses directly as part of
1844 # the statement
1845 if is_lpar_token(child):
1846 assert is_rpar_token(parent.children[-1])
1847 # make parentheses invisible
1848 child.value = ""
1849 parent.children[-1].value = ""
1850 elif child.type != token.STAR:
1851 # insert invisible parentheses
1852 parent.insert_child(index, Leaf(token.LPAR, ""))
1853 parent.append_child(Leaf(token.RPAR, ""))
1856def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1857 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1858 if (
1859 node.children[1].type == syms.atom
1860 and node.children[1].children[0].type == token.LPAR
1861 ):
1862 if maybe_make_parens_invisible_in_atom(
1863 node.children[1],
1864 parent=node,
1865 mode=mode,
1866 features=features,
1867 remove_brackets_around_comma=True,
1868 ):
1869 wrap_in_parentheses(node, node.children[1], visible=False)
1871 # Since await is an expression we shouldn't remove
1872 # brackets in cases where this would change
1873 # the AST due to operator precedence.
1874 # Therefore we only aim to remove brackets around
1875 # power nodes that aren't also await expressions themselves.
1876 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1877 # N.B. We've still removed any redundant nested brackets though :)
1878 opening_bracket = cast(Leaf, node.children[1].children[0])
1879 closing_bracket = cast(Leaf, node.children[1].children[-1])
1880 bracket_contents = node.children[1].children[1]
1881 if isinstance(bracket_contents, Node) and (
1882 bracket_contents.type != syms.power
1883 or bracket_contents.children[0].type == token.AWAIT
1884 or any(
1885 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1886 for child in bracket_contents.children
1887 )
1888 ):
1889 ensure_visible(opening_bracket)
1890 ensure_visible(closing_bracket)
1893def _maybe_wrap_cms_in_parens(
1894 node: Node, mode: Mode, features: Collection[Feature]
1895) -> None:
1896 """When enabled and safe, wrap the multiple context managers in invisible parens.
1898 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1899 """
1900 if (
1901 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1902 or len(node.children) <= 2
1903 # If it's an atom, it's already wrapped in parens.
1904 or node.children[1].type == syms.atom
1905 ):
1906 return
1907 colon_index: int | None = None
1908 for i in range(2, len(node.children)):
1909 if node.children[i].type == token.COLON:
1910 colon_index = i
1911 break
1912 if colon_index is not None:
1913 lpar = Leaf(token.LPAR, "")
1914 rpar = Leaf(token.RPAR, "")
1915 context_managers = node.children[1:colon_index]
1916 for child in context_managers:
1917 child.remove()
1918 # After wrapping, the with_stmt will look like this:
1919 # with_stmt
1920 # NAME 'with'
1921 # atom
1922 # LPAR ''
1923 # testlist_gexp
1924 # ... <-- context_managers
1925 # /testlist_gexp
1926 # RPAR ''
1927 # /atom
1928 # COLON ':'
1929 new_child = Node(
1930 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1931 )
1932 node.insert_child(1, new_child)
1935def remove_with_parens(
1936 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1937) -> None:
1938 """Recursively hide optional parens in `with` statements."""
1939 # Removing all unnecessary parentheses in with statements in one pass is a tad
1940 # complex as different variations of bracketed statements result in pretty
1941 # different parse trees:
1942 #
1943 # with (open("file")) as f: # this is an asexpr_test
1944 # ...
1945 #
1946 # with (open("file") as f): # this is an atom containing an
1947 # ... # asexpr_test
1948 #
1949 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1950 # ... # asexpr_test
1951 #
1952 # with (open("file") as f, open("file") as f): # an atom containing a
1953 # ... # testlist_gexp which then
1954 # # contains multiple asexpr_test(s)
1955 if node.type == syms.atom:
1956 if maybe_make_parens_invisible_in_atom(
1957 node,
1958 parent=parent,
1959 mode=mode,
1960 features=features,
1961 remove_brackets_around_comma=True,
1962 ):
1963 wrap_in_parentheses(parent, node, visible=False)
1964 if isinstance(node.children[1], Node):
1965 remove_with_parens(node.children[1], node, mode=mode, features=features)
1966 elif node.type == syms.testlist_gexp:
1967 for child in node.children:
1968 if isinstance(child, Node):
1969 remove_with_parens(child, node, mode=mode, features=features)
1970 elif node.type == syms.asexpr_test and not any(
1971 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1972 ):
1973 if maybe_make_parens_invisible_in_atom(
1974 node.children[0],
1975 parent=node,
1976 mode=mode,
1977 features=features,
1978 remove_brackets_around_comma=True,
1979 ):
1980 wrap_in_parentheses(node, node.children[0], visible=False)
1983def _atom_has_magic_trailing_comma(node: LN, mode: Mode) -> bool:
1984 """Check if an atom node has a magic trailing comma.
1986 Returns True for single-element tuples with trailing commas like (a,),
1987 which should be preserved to maintain their tuple type.
1988 """
1989 if not mode.magic_trailing_comma:
1990 return False
1992 return is_one_tuple(node)
1995def _is_parenthesized_annotation_target(node: Node, child: Node) -> bool:
1996 """Is `child` a parenthesized plain name that `node` annotates?
1998 `(x): int = 5` and `x: int = 5` do not do the same thing: only the second one
1999 records `x` in `__annotations__`, because the parentheses make the target
2000 non-simple (`AnnAssign.simple` is 0). Dropping them changes what the module
2001 does at runtime, so they have to stay. Attribute and subscript targets are
2002 non-simple either way, so those are left alone here.
2003 """
2004 if len(node.children) < 2:
2005 return False
2007 annassign = node.children[1]
2008 if not isinstance(annassign, Node) or annassign.type != syms.annassign:
2009 return False
2011 # `((x)): int = 5` is non-simple too, so look through any nesting.
2012 target: LN = child
2013 while (
2014 isinstance(target, Node)
2015 and target.type == syms.atom
2016 and len(target.children) == 3
2017 ):
2018 target = target.children[1]
2020 return isinstance(target, Leaf) and target.type == token.NAME
2023def _is_atom_multiline(node: LN) -> bool:
2024 """Check if an atom node is multiline (indicating intentional formatting)."""
2025 if not isinstance(node, Node) or len(node.children) < 3:
2026 return False
2028 # Check the middle child (between LPAR and RPAR) for newlines in its subtree
2029 # The first child's prefix contains blank lines/comments before the opening paren
2030 middle = node.children[1]
2031 for child in middle.pre_order():
2032 if isinstance(child, Leaf) and "\n" in child.prefix:
2033 return True
2035 return False
2038def maybe_make_parens_invisible_in_atom(
2039 node: LN,
2040 parent: LN,
2041 mode: Mode,
2042 features: Collection[Feature],
2043 remove_brackets_around_comma: bool = False,
2044 allow_star_expr: bool = False,
2045 remove_generator_parens: bool = False,
2046) -> bool:
2047 """If it's safe, make the parens in the atom `node` invisible, recursively.
2048 Additionally, remove repeated, adjacent invisible parens from the atom `node`
2049 as they are redundant.
2051 Returns whether the node should itself be wrapped in invisible parentheses.
2052 """
2053 can_remove_generator_parens = remove_generator_parens and is_generator(node)
2054 if (
2055 node.type not in (syms.atom, syms.expr)
2056 or is_empty_tuple(node)
2057 or is_one_tuple(node)
2058 or (is_tuple(node) and parent.type == syms.asexpr_test)
2059 or (
2060 is_tuple(node)
2061 and parent.type == syms.with_stmt
2062 and has_sibling_with_type(node, token.COMMA)
2063 )
2064 or (is_yield(node) and parent.type != syms.expr_stmt)
2065 or (
2066 # This condition tries to prevent removing non-optional brackets
2067 # around a tuple, however, can be a bit overzealous so we provide
2068 # and option to skip this check for `for` and `with` statements.
2069 not remove_brackets_around_comma
2070 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
2071 and not can_remove_generator_parens
2072 # Remove parentheses around multiple exception types in except and
2073 # except* without as. See PEP 758 for details.
2074 and not (
2075 Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
2076 # is a tuple
2077 and is_tuple(node)
2078 # has a parent node
2079 and node.parent is not None
2080 # parent is an except clause
2081 and node.parent.type == syms.except_clause
2082 # is not immediately followed by as clause
2083 and not (
2084 node.next_sibling is not None
2085 and is_name_token(node.next_sibling)
2086 and node.next_sibling.value == "as"
2087 )
2088 )
2089 )
2090 or is_tuple_containing_walrus(node)
2091 or (not allow_star_expr and is_tuple_containing_star(node))
2092 or (not can_remove_generator_parens and is_generator(node))
2093 ):
2094 return False
2096 if is_walrus_assignment(node):
2097 if parent.type in [
2098 syms.annassign,
2099 syms.expr_stmt,
2100 syms.assert_stmt,
2101 syms.return_stmt,
2102 syms.yield_arg,
2103 syms.yield_expr,
2104 syms.except_clause,
2105 syms.funcdef,
2106 syms.with_stmt,
2107 syms.testlist_gexp,
2108 syms.tname,
2109 # these ones aren't useful to end users, but they do please fuzzers
2110 syms.for_stmt,
2111 syms.del_stmt,
2112 ]:
2113 return False
2115 first = node.children[0]
2116 last = node.children[-1]
2117 if is_lpar_token(first) and is_rpar_token(last):
2118 middle = node.children[1]
2119 # make parentheses invisible
2120 if (
2121 # If the prefix of `middle` includes a type comment with
2122 # ignore annotation, then we do not remove the parentheses
2123 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
2124 ):
2125 first.value = ""
2126 last.value = ""
2127 maybe_make_parens_invisible_in_atom(
2128 middle,
2129 parent=parent,
2130 mode=mode,
2131 features=features,
2132 remove_brackets_around_comma=remove_brackets_around_comma,
2133 )
2135 if is_atom_with_invisible_parens(middle):
2136 # Strip the invisible parens from `middle` by replacing
2137 # it with the child in-between the invisible parens
2138 middle.replace(middle.children[1])
2140 if middle.children[0].prefix.strip():
2141 # Preserve comments before first paren
2142 middle.children[1].prefix = (
2143 middle.children[0].prefix + middle.children[1].prefix
2144 )
2146 if middle.children[-1].prefix.strip():
2147 # Preserve comments before last paren
2148 last.prefix = middle.children[-1].prefix + last.prefix
2150 return False
2152 return True
2155def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
2156 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
2158 if not (opening_bracket.parent and opening_bracket.value in "[{("):
2159 return False
2161 # We're essentially checking if the body is delimited by commas and there's more
2162 # than one of them (we're excluding the trailing comma and if the delimiter priority
2163 # is still commas, that means there's more).
2164 exclude = set()
2165 trailing_comma = False
2166 try:
2167 last_leaf = line.leaves[-1]
2168 if last_leaf.type == token.COMMA:
2169 trailing_comma = True
2170 exclude.add(id(last_leaf))
2171 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
2172 except (IndexError, ValueError):
2173 return False
2175 return max_priority == COMMA_PRIORITY and (
2176 (line.mode.magic_trailing_comma and trailing_comma)
2177 # always explode imports
2178 or opening_bracket.parent.type in {syms.atom, syms.import_from}
2179 )
2182def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
2183 """Generate sets of closing bracket IDs that should be omitted in a RHS.
2185 Brackets can be omitted if the entire trailer up to and including
2186 a preceding closing bracket fits in one line.
2188 Yielded sets are cumulative (contain results of previous yields, too). First
2189 set is empty, unless the line should explode, in which case bracket pairs until
2190 the one that needs to explode are omitted.
2191 """
2193 omit: set[LeafID] = set()
2194 if not line.magic_trailing_comma:
2195 yield omit
2197 length = 4 * line.depth
2198 opening_bracket: Leaf | None = None
2199 closing_bracket: Leaf | None = None
2200 inner_brackets: set[LeafID] = set()
2201 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
2202 length += leaf_length
2203 if length > line_length:
2204 break
2206 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
2207 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
2208 break
2210 if opening_bracket:
2211 if leaf is opening_bracket:
2212 opening_bracket = None
2213 elif leaf.type in CLOSING_BRACKETS:
2214 prev = line.leaves[index - 1] if index > 0 else None
2215 if (
2216 prev
2217 and prev.type == token.COMMA
2218 and leaf.opening_bracket is not None
2219 and not is_one_sequence_between(
2220 leaf.opening_bracket, leaf, line.leaves
2221 )
2222 ):
2223 # Never omit bracket pairs with trailing commas.
2224 # We need to explode on those.
2225 break
2227 inner_brackets.add(id(leaf))
2228 elif leaf.type in CLOSING_BRACKETS:
2229 prev = line.leaves[index - 1] if index > 0 else None
2230 if prev and prev.type in OPENING_BRACKETS:
2231 # Empty brackets would fail a split so treat them as "inner"
2232 # brackets (e.g. only add them to the `omit` set if another
2233 # pair of brackets was good enough.
2234 inner_brackets.add(id(leaf))
2235 continue
2237 if closing_bracket:
2238 omit.add(id(closing_bracket))
2239 omit.update(inner_brackets)
2240 inner_brackets.clear()
2241 yield omit
2243 if (
2244 prev
2245 and prev.type == token.COMMA
2246 and leaf.opening_bracket is not None
2247 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
2248 ):
2249 # Never omit bracket pairs with trailing commas.
2250 # We need to explode on those.
2251 break
2253 if leaf.value:
2254 opening_bracket = leaf.opening_bracket
2255 closing_bracket = leaf
2258def _over_length_only_due_to_subscript_comment(line: Line, mode: Mode) -> bool:
2259 """Return True if `line` only exceeds `mode.line_length` because of an inline
2260 comment attached to a subscript opening bracket (`[`).
2262 This is the shape produced by the original of the issue #4733 reproducer:
2263 a comment inside the annotation's subscript brackets renders at the end of
2264 the head line after Black splits the statement, pushing it past the limit.
2265 Taking the FORCE_OPTIONAL_PARENTHESES "second opinion" in that case wraps
2266 the annotation in extra parens and migrates the comment outside the
2267 subscript, which then oscillates on the next formatter pass.
2268 """
2269 if not line.leaves:
2270 return False
2271 # The over-length must be caused entirely by a trailing comment.
2272 indent = " " * line.depth
2273 leaves_iter = iter(line.leaves)
2274 first = next(leaves_iter)
2275 text_without_comments = f"{first.prefix}{indent}{first.value}"
2276 text_without_comments += "".join(str(leaf) for leaf in leaves_iter)
2277 if str_width(text_without_comments) > mode.line_length:
2278 return False
2279 # And the comment must be attached to a subscript opening bracket.
2280 for leaf_id, comments in line.comments.items():
2281 if not comments:
2282 continue
2283 leaf = next((lf for lf in line.leaves if id(lf) == leaf_id), None)
2284 if leaf is None or leaf.type != token.LSQB:
2285 return False
2286 return True
2289def run_transformer(
2290 line: Line,
2291 transform: Transformer,
2292 mode: Mode,
2293 features: Collection[Feature],
2294 *,
2295 line_str: str = "",
2296) -> list[Line]:
2297 if not line_str:
2298 line_str = line_to_string(line)
2299 result: list[Line] = []
2300 for transformed_line in transform(line, features, mode):
2301 if str(transformed_line).strip("\n") == line_str:
2302 raise CannotTransform("Line transformer returned an unchanged result")
2304 result.extend(transform_line(transformed_line, mode=mode, features=features))
2306 features_set = set(features)
2307 if (
2308 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
2309 or transform.__class__.__name__ != "rhs"
2310 or not line.bracket_tracker.invisible
2311 or any(bracket.value for bracket in line.bracket_tracker.invisible)
2312 or line.contains_multiline_strings()
2313 or result[0].contains_uncollapsable_type_comments()
2314 or result[0].contains_unsplittable_type_ignore()
2315 or is_line_short_enough(result[0], mode=mode)
2316 # result[0] only exceeds the length because of a comment attached to a
2317 # subscript opening bracket. Taking the FORCE_OPTIONAL_PARENTHESES
2318 # "second opinion" wraps the annotation in extra invisible parens and
2319 # migrates the comment outside the subscript, which then oscillates with
2320 # a deeper-bracket split on the next formatter pass (issue #4733).
2321 or _over_length_only_due_to_subscript_comment(result[0], mode)
2322 # If any leaves have no parents (which _can_ occur since
2323 # `transform(line)` potentially destroys the line's underlying node
2324 # structure), then we can't proceed. Doing so would cause the below
2325 # call to `append_leaves()` to fail.
2326 or any(leaf.parent is None for leaf in line.leaves)
2327 ):
2328 return result
2330 line_copy = line.clone()
2331 append_leaves(line_copy, line, line.leaves)
2332 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
2333 second_opinion = run_transformer(
2334 line_copy, transform, mode, features_fop, line_str=line_str
2335 )
2336 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
2337 result = second_opinion
2338 return result