Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Optional, Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import FMT_OFF, generate_comments, list_comments
21from black.lines import (
22 Line,
23 RHSResult,
24 append_leaves,
25 can_be_split,
26 can_omit_invisible_parens,
27 is_line_short_enough,
28 line_to_string,
29)
30from black.mode import Feature, Mode, Preview
31from black.nodes import (
32 ASSIGNMENTS,
33 BRACKETS,
34 CLOSING_BRACKETS,
35 OPENING_BRACKETS,
36 STANDALONE_COMMENT,
37 STATEMENT,
38 WHITESPACE,
39 Visitor,
40 ensure_visible,
41 fstring_to_string,
42 get_annotation_type,
43 has_sibling_with_type,
44 is_arith_like,
45 is_async_stmt_or_funcdef,
46 is_atom_with_invisible_parens,
47 is_docstring,
48 is_empty_tuple,
49 is_generator,
50 is_lpar_token,
51 is_multiline_string,
52 is_name_token,
53 is_one_sequence_between,
54 is_one_tuple,
55 is_parent_function_or_class,
56 is_part_of_annotation,
57 is_rpar_token,
58 is_stub_body,
59 is_stub_suite,
60 is_tuple,
61 is_tuple_containing_star,
62 is_tuple_containing_walrus,
63 is_type_ignore_comment_string,
64 is_vararg,
65 is_walrus_assignment,
66 is_yield,
67 syms,
68 wrap_in_parentheses,
69)
70from black.numerics import normalize_numeric_literal
71from black.strings import (
72 fix_multiline_docstring,
73 get_string_prefix,
74 normalize_string_prefix,
75 normalize_string_quotes,
76 normalize_unicode_escape_sequences,
77)
78from black.trans import (
79 CannotTransform,
80 StringMerger,
81 StringParenStripper,
82 StringParenWrapper,
83 StringSplitter,
84 Transformer,
85 hug_power_op,
86)
87from blib2to3.pgen2 import token
88from blib2to3.pytree import Leaf, Node
90# types
91LeafID = int
92LN = Union[Leaf, Node]
95class CannotSplit(CannotTransform):
96 """A readable split that fits the allotted line length is impossible."""
99# This isn't a dataclass because @dataclass + Generic breaks mypyc.
100# See also https://github.com/mypyc/mypyc/issues/827.
101class LineGenerator(Visitor[Line]):
102 """Generates reformatted Line objects. Empty lines are not emitted.
104 Note: destroys the tree it's visiting by mutating prefixes of its leaves
105 in ways that will no longer stringify to valid Python code on the tree.
106 """
108 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
109 self.mode = mode
110 self.features = features
111 self.current_line: Line
112 self.__post_init__()
114 def line(self, indent: int = 0) -> Iterator[Line]:
115 """Generate a line.
117 If the line is empty, only emit if it makes sense.
118 If the line is too long, split it first and then generate.
120 If any lines were generated, set up a new current_line.
121 """
122 if not self.current_line:
123 self.current_line.depth += indent
124 return # Line is empty, don't emit. Creating a new one unnecessary.
126 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
127 self.current_line.leaves[0]
128 ):
129 # Special case for async def/for/with statements. `visit_async_stmt`
130 # adds an `ASYNC` leaf then visits the child def/for/with statement
131 # nodes. Line yields from those nodes shouldn't treat the former
132 # `ASYNC` leaf as a complete line.
133 return
135 complete_line = self.current_line
136 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
137 yield complete_line
139 def visit_default(self, node: LN) -> Iterator[Line]:
140 """Default `visit_*()` implementation. Recurses to children of `node`."""
141 if isinstance(node, Leaf):
142 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
143 for comment in generate_comments(node):
144 if any_open_brackets:
145 # any comment within brackets is subject to splitting
146 self.current_line.append(comment)
147 elif comment.type == token.COMMENT:
148 # regular trailing comment
149 self.current_line.append(comment)
150 yield from self.line()
152 else:
153 # regular standalone comment
154 yield from self.line()
156 self.current_line.append(comment)
157 yield from self.line()
159 if any_open_brackets:
160 node.prefix = ""
161 if node.type not in WHITESPACE:
162 self.current_line.append(node)
163 yield from super().visit_default(node)
165 def visit_test(self, node: Node) -> Iterator[Line]:
166 """Visit an `x if y else z` test"""
168 already_parenthesized = (
169 node.prev_sibling and node.prev_sibling.type == token.LPAR
170 )
172 if not already_parenthesized:
173 # Similar to logic in wrap_in_parentheses
174 lpar = Leaf(token.LPAR, "")
175 rpar = Leaf(token.RPAR, "")
176 prefix = node.prefix
177 node.prefix = ""
178 lpar.prefix = prefix
179 node.insert_child(0, lpar)
180 node.append_child(rpar)
182 yield from self.visit_default(node)
184 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
185 """Increase indentation level, maybe yield a line."""
186 # In blib2to3 INDENT never holds comments.
187 yield from self.line(+1)
188 yield from self.visit_default(node)
190 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
191 """Decrease indentation level, maybe yield a line."""
192 # The current line might still wait for trailing comments. At DEDENT time
193 # there won't be any (they would be prefixes on the preceding NEWLINE).
194 # Emit the line then.
195 yield from self.line()
197 # While DEDENT has no value, its prefix may contain standalone comments
198 # that belong to the current indentation level. Get 'em.
199 yield from self.visit_default(node)
201 # Finally, emit the dedent.
202 yield from self.line(-1)
204 def visit_stmt(
205 self, node: Node, keywords: set[str], parens: set[str]
206 ) -> Iterator[Line]:
207 """Visit a statement.
209 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
210 `def`, `with`, `class`, `assert`, and assignments.
212 The relevant Python language `keywords` for a given statement will be
213 NAME leaves within it. This methods puts those on a separate line.
215 `parens` holds a set of string leaf values immediately after which
216 invisible parens should be put.
217 """
218 normalize_invisible_parens(
219 node, parens_after=parens, mode=self.mode, features=self.features
220 )
221 for child in node.children:
222 if is_name_token(child) and child.value in keywords:
223 yield from self.line()
225 yield from self.visit(child)
227 def visit_typeparams(self, node: Node) -> Iterator[Line]:
228 yield from self.visit_default(node)
229 node.children[0].prefix = ""
231 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
232 yield from self.visit_default(node)
233 node.children[1].prefix = ""
235 def visit_paramspec(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[1].prefix = ""
239 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
240 if Preview.wrap_long_dict_values_in_parens in self.mode:
241 for i, child in enumerate(node.children):
242 if i == 0:
243 continue
244 if node.children[i - 1].type == token.COLON:
245 if (
246 child.type == syms.atom
247 and child.children[0].type in OPENING_BRACKETS
248 and not is_walrus_assignment(child)
249 ):
250 maybe_make_parens_invisible_in_atom(
251 child,
252 parent=node,
253 remove_brackets_around_comma=False,
254 )
255 else:
256 wrap_in_parentheses(node, child, visible=False)
257 yield from self.visit_default(node)
259 def visit_funcdef(self, node: Node) -> Iterator[Line]:
260 """Visit function definition."""
261 yield from self.line()
263 # Remove redundant brackets around return type annotation.
264 is_return_annotation = False
265 for child in node.children:
266 if child.type == token.RARROW:
267 is_return_annotation = True
268 elif is_return_annotation:
269 if child.type == syms.atom and child.children[0].type == token.LPAR:
270 if maybe_make_parens_invisible_in_atom(
271 child,
272 parent=node,
273 remove_brackets_around_comma=False,
274 ):
275 wrap_in_parentheses(node, child, visible=False)
276 else:
277 wrap_in_parentheses(node, child, visible=False)
278 is_return_annotation = False
280 for child in node.children:
281 yield from self.visit(child)
283 def visit_match_case(self, node: Node) -> Iterator[Line]:
284 """Visit either a match or case statement."""
285 normalize_invisible_parens(
286 node, parens_after=set(), mode=self.mode, features=self.features
287 )
289 yield from self.line()
290 for child in node.children:
291 yield from self.visit(child)
293 def visit_suite(self, node: Node) -> Iterator[Line]:
294 """Visit a suite."""
295 if is_stub_suite(node):
296 yield from self.visit(node.children[2])
297 else:
298 yield from self.visit_default(node)
300 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
301 """Visit a statement without nested statements."""
302 prev_type: Optional[int] = None
303 for child in node.children:
304 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
305 wrap_in_parentheses(node, child, visible=False)
306 prev_type = child.type
308 if node.parent and node.parent.type in STATEMENT:
309 if is_parent_function_or_class(node) and is_stub_body(node):
310 yield from self.visit_default(node)
311 else:
312 yield from self.line(+1)
313 yield from self.visit_default(node)
314 yield from self.line(-1)
316 else:
317 if node.parent and is_stub_suite(node.parent):
318 node.prefix = ""
319 yield from self.visit_default(node)
320 return
321 yield from self.line()
322 yield from self.visit_default(node)
324 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
325 """Visit `async def`, `async for`, `async with`."""
326 yield from self.line()
328 children = iter(node.children)
329 for child in children:
330 yield from self.visit(child)
332 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
333 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
334 # line.
335 break
337 internal_stmt = next(children)
338 yield from self.visit(internal_stmt)
340 def visit_decorators(self, node: Node) -> Iterator[Line]:
341 """Visit decorators."""
342 for child in node.children:
343 yield from self.line()
344 yield from self.visit(child)
346 def visit_power(self, node: Node) -> Iterator[Line]:
347 for idx, leaf in enumerate(node.children[:-1]):
348 next_leaf = node.children[idx + 1]
350 if not isinstance(leaf, Leaf):
351 continue
353 value = leaf.value.lower()
354 if (
355 leaf.type == token.NUMBER
356 and next_leaf.type == syms.trailer
357 # Ensure that we are in an attribute trailer
358 and next_leaf.children[0].type == token.DOT
359 # It shouldn't wrap hexadecimal, binary and octal literals
360 and not value.startswith(("0x", "0b", "0o"))
361 # It shouldn't wrap complex literals
362 and "j" not in value
363 ):
364 wrap_in_parentheses(node, leaf)
366 remove_await_parens(node)
368 yield from self.visit_default(node)
370 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
371 """Remove a semicolon and put the other statement on a separate line."""
372 yield from self.line()
374 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
375 """End of file. Process outstanding comments and end with a newline."""
376 yield from self.visit_default(leaf)
377 yield from self.line()
379 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
380 if not self.current_line.bracket_tracker.any_open_brackets():
381 yield from self.line()
382 yield from self.visit_default(leaf)
384 def visit_factor(self, node: Node) -> Iterator[Line]:
385 """Force parentheses between a unary op and a binary power:
387 -2 ** 8 -> -(2 ** 8)
388 """
389 _operator, operand = node.children
390 if (
391 operand.type == syms.power
392 and len(operand.children) == 3
393 and operand.children[1].type == token.DOUBLESTAR
394 ):
395 lpar = Leaf(token.LPAR, "(")
396 rpar = Leaf(token.RPAR, ")")
397 index = operand.remove() or 0
398 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
399 yield from self.visit_default(node)
401 def visit_tname(self, node: Node) -> Iterator[Line]:
402 """
403 Add potential parentheses around types in function parameter lists to be made
404 into real parentheses in case the type hint is too long to fit on a line
405 Examples:
406 def foo(a: int, b: float = 7): ...
408 ->
410 def foo(a: (int), b: (float) = 7): ...
411 """
412 assert len(node.children) == 3
413 if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):
414 wrap_in_parentheses(node, node.children[2], visible=False)
416 yield from self.visit_default(node)
418 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
419 normalize_unicode_escape_sequences(leaf)
421 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
422 # We're ignoring docstrings with backslash newline escapes because changing
423 # indentation of those changes the AST representation of the code.
424 if self.mode.string_normalization:
425 docstring = normalize_string_prefix(leaf.value)
426 # We handle string normalization at the end of this method, but since
427 # what we do right now acts differently depending on quote style (ex.
428 # see padding logic below), there's a possibility for unstable
429 # formatting. To avoid a situation where this function formats a
430 # docstring differently on the second pass, normalize it early.
431 docstring = normalize_string_quotes(docstring)
432 else:
433 docstring = leaf.value
434 prefix = get_string_prefix(docstring)
435 docstring = docstring[len(prefix) :] # Remove the prefix
436 quote_char = docstring[0]
437 # A natural way to remove the outer quotes is to do:
438 # docstring = docstring.strip(quote_char)
439 # but that breaks on """""x""" (which is '""x').
440 # So we actually need to remove the first character and the next two
441 # characters but only if they are the same as the first.
442 quote_len = 1 if docstring[1] != quote_char else 3
443 docstring = docstring[quote_len:-quote_len]
444 docstring_started_empty = not docstring
445 indent = " " * 4 * self.current_line.depth
447 if is_multiline_string(leaf):
448 docstring = fix_multiline_docstring(docstring, indent)
449 else:
450 docstring = docstring.strip()
452 has_trailing_backslash = False
453 if docstring:
454 # Add some padding if the docstring starts / ends with a quote mark.
455 if docstring[0] == quote_char:
456 docstring = " " + docstring
457 if docstring[-1] == quote_char:
458 docstring += " "
459 if docstring[-1] == "\\":
460 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
461 if backslash_count % 2:
462 # Odd number of tailing backslashes, add some padding to
463 # avoid escaping the closing string quote.
464 docstring += " "
465 has_trailing_backslash = True
466 elif not docstring_started_empty:
467 docstring = " "
469 # We could enforce triple quotes at this point.
470 quote = quote_char * quote_len
472 # It's invalid to put closing single-character quotes on a new line.
473 if quote_len == 3:
474 # We need to find the length of the last line of the docstring
475 # to find if we can add the closing quotes to the line without
476 # exceeding the maximum line length.
477 # If docstring is one line, we don't put the closing quotes on a
478 # separate line because it looks ugly (#3320).
479 lines = docstring.splitlines()
480 last_line_length = len(lines[-1]) if docstring else 0
482 # If adding closing quotes would cause the last line to exceed
483 # the maximum line length, and the closing quote is not
484 # prefixed by a newline then put a line break before
485 # the closing quotes
486 if (
487 len(lines) > 1
488 and last_line_length + quote_len > self.mode.line_length
489 and len(indent) + quote_len <= self.mode.line_length
490 and not has_trailing_backslash
491 ):
492 if leaf.value[-1 - quote_len] == "\n":
493 leaf.value = prefix + quote + docstring + quote
494 else:
495 leaf.value = prefix + quote + docstring + "\n" + indent + quote
496 else:
497 leaf.value = prefix + quote + docstring + quote
498 else:
499 leaf.value = prefix + quote + docstring + quote
501 if self.mode.string_normalization and leaf.type == token.STRING:
502 leaf.value = normalize_string_prefix(leaf.value)
503 leaf.value = normalize_string_quotes(leaf.value)
504 yield from self.visit_default(leaf)
506 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
507 normalize_numeric_literal(leaf)
508 yield from self.visit_default(leaf)
510 def visit_atom(self, node: Node) -> Iterator[Line]:
511 """Visit any atom"""
512 if len(node.children) == 3:
513 first = node.children[0]
514 last = node.children[-1]
515 if (first.type == token.LSQB and last.type == token.RSQB) or (
516 first.type == token.LBRACE and last.type == token.RBRACE
517 ):
518 # Lists or sets of one item
519 maybe_make_parens_invisible_in_atom(node.children[1], parent=node)
521 yield from self.visit_default(node)
523 def visit_fstring(self, node: Node) -> Iterator[Line]:
524 # currently we don't want to format and split f-strings at all.
525 string_leaf = fstring_to_string(node)
526 node.replace(string_leaf)
527 if "\\" in string_leaf.value and any(
528 "\\" in str(child)
529 for child in node.children
530 if child.type == syms.fstring_replacement_field
531 ):
532 # string normalization doesn't account for nested quotes,
533 # causing breakages. skip normalization when nested quotes exist
534 yield from self.visit_default(string_leaf)
535 return
536 yield from self.visit_STRING(string_leaf)
538 # TODO: Uncomment Implementation to format f-string children
539 # fstring_start = node.children[0]
540 # fstring_end = node.children[-1]
541 # assert isinstance(fstring_start, Leaf)
542 # assert isinstance(fstring_end, Leaf)
544 # quote_char = fstring_end.value[0]
545 # quote_idx = fstring_start.value.index(quote_char)
546 # prefix, quote = (
547 # fstring_start.value[:quote_idx],
548 # fstring_start.value[quote_idx:]
549 # )
551 # if not is_docstring(node, self.mode):
552 # prefix = normalize_string_prefix(prefix)
554 # assert quote == fstring_end.value
556 # is_raw_fstring = "r" in prefix or "R" in prefix
557 # middles = [
558 # leaf
559 # for leaf in node.leaves()
560 # if leaf.type == token.FSTRING_MIDDLE
561 # ]
563 # if self.mode.string_normalization:
564 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
566 # fstring_start.value = prefix + quote
567 # fstring_end.value = quote
569 # yield from self.visit_default(node)
571 def __post_init__(self) -> None:
572 """You are in a twisty little maze of passages."""
573 self.current_line = Line(mode=self.mode)
575 v = self.visit_stmt
576 Ø: set[str] = set()
577 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
578 self.visit_if_stmt = partial(
579 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
580 )
581 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
582 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
583 self.visit_try_stmt = partial(
584 v, keywords={"try", "except", "else", "finally"}, parens=Ø
585 )
586 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
587 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
588 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
590 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
591 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
592 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
593 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
594 self.visit_async_funcdef = self.visit_async_stmt
595 self.visit_decorated = self.visit_decorators
597 # PEP 634
598 self.visit_match_stmt = self.visit_match_case
599 self.visit_case_block = self.visit_match_case
600 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
603def _hugging_power_ops_line_to_string(
604 line: Line,
605 features: Collection[Feature],
606 mode: Mode,
607) -> Optional[str]:
608 try:
609 return line_to_string(next(hug_power_op(line, features, mode)))
610 except CannotTransform:
611 return None
614def transform_line(
615 line: Line, mode: Mode, features: Collection[Feature] = ()
616) -> Iterator[Line]:
617 """Transform a `line`, potentially splitting it into many lines.
619 They should fit in the allotted `line_length` but might not be able to.
621 `features` are syntactical features that may be used in the output.
622 """
623 if line.is_comment:
624 yield line
625 return
627 line_str = line_to_string(line)
629 # We need the line string when power operators are hugging to determine if we should
630 # split the line. Default to line_str, if no power operator are present on the line.
631 line_str_hugging_power_ops = (
632 _hugging_power_ops_line_to_string(line, features, mode) or line_str
633 )
635 ll = mode.line_length
636 sn = mode.string_normalization
637 string_merge = StringMerger(ll, sn)
638 string_paren_strip = StringParenStripper(ll, sn)
639 string_split = StringSplitter(ll, sn)
640 string_paren_wrap = StringParenWrapper(ll, sn)
642 transformers: list[Transformer]
643 if (
644 not line.contains_uncollapsable_type_comments()
645 and not line.should_split_rhs
646 and not line.magic_trailing_comma
647 and (
648 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
649 or line.contains_unsplittable_type_ignore()
650 )
651 and not (line.inside_brackets and line.contains_standalone_comments())
652 and not line.contains_implicit_multiline_string_with_comments()
653 ):
654 # Only apply basic string preprocessing, since lines shouldn't be split here.
655 if Preview.string_processing in mode:
656 transformers = [string_merge, string_paren_strip]
657 else:
658 transformers = []
659 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
660 transformers = [left_hand_split]
661 else:
663 def _rhs(
664 self: object, line: Line, features: Collection[Feature], mode: Mode
665 ) -> Iterator[Line]:
666 """Wraps calls to `right_hand_split`.
668 The calls increasingly `omit` right-hand trailers (bracket pairs with
669 content), meaning the trailers get glued together to split on another
670 bracket pair instead.
671 """
672 for omit in generate_trailers_to_omit(line, mode.line_length):
673 lines = list(right_hand_split(line, mode, features, omit=omit))
674 # Note: this check is only able to figure out if the first line of the
675 # *current* transformation fits in the line length. This is true only
676 # for simple cases. All others require running more transforms via
677 # `transform_line()`. This check doesn't know if those would succeed.
678 if is_line_short_enough(lines[0], mode=mode):
679 yield from lines
680 return
682 # All splits failed, best effort split with no omits.
683 # This mostly happens to multiline strings that are by definition
684 # reported as not fitting a single line, as well as lines that contain
685 # trailing commas (those have to be exploded).
686 yield from right_hand_split(line, mode, features=features)
688 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
689 # __name__ attribute which is needed in `run_transformer` further down.
690 # Unfortunately a nested class breaks mypyc too. So a class must be created
691 # via type ... https://github.com/mypyc/mypyc/issues/884
692 rhs = type("rhs", (), {"__call__": _rhs})()
694 if Preview.string_processing in mode:
695 if line.inside_brackets:
696 transformers = [
697 string_merge,
698 string_paren_strip,
699 string_split,
700 delimiter_split,
701 standalone_comment_split,
702 string_paren_wrap,
703 rhs,
704 ]
705 else:
706 transformers = [
707 string_merge,
708 string_paren_strip,
709 string_split,
710 string_paren_wrap,
711 rhs,
712 ]
713 else:
714 if line.inside_brackets:
715 transformers = [delimiter_split, standalone_comment_split, rhs]
716 else:
717 transformers = [rhs]
718 # It's always safe to attempt hugging of power operations and pretty much every line
719 # could match.
720 transformers.append(hug_power_op)
722 for transform in transformers:
723 # We are accumulating lines in `result` because we might want to abort
724 # mission and return the original line in the end, or attempt a different
725 # split altogether.
726 try:
727 result = run_transformer(line, transform, mode, features, line_str=line_str)
728 except CannotTransform:
729 continue
730 else:
731 yield from result
732 break
734 else:
735 yield line
738def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
739 """If a funcdef has a magic trailing comma in the return type, then we should first
740 split the line with rhs to respect the comma.
741 """
742 return_type_leaves: list[Leaf] = []
743 in_return_type = False
745 for leaf in line.leaves:
746 if leaf.type == token.COLON:
747 in_return_type = False
748 if in_return_type:
749 return_type_leaves.append(leaf)
750 if leaf.type == token.RARROW:
751 in_return_type = True
753 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
754 # couple lines from it.
755 result = Line(mode=line.mode, depth=line.depth)
756 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
757 for leaf in return_type_leaves:
758 result.append(
759 leaf,
760 preformatted=True,
761 track_bracket=id(leaf) in leaves_to_track,
762 )
764 # we could also return true if the line is too long, and the return type is longer
765 # than the param list. Or if `should_split_rhs` returns True.
766 return result.magic_trailing_comma is not None
769class _BracketSplitComponent(Enum):
770 head = auto()
771 body = auto()
772 tail = auto()
775def left_hand_split(
776 line: Line, _features: Collection[Feature], mode: Mode
777) -> Iterator[Line]:
778 """Split line into many lines, starting with the first matching bracket pair.
780 Note: this usually looks weird, only use this for function definitions.
781 Prefer RHS otherwise. This is why this function is not symmetrical with
782 :func:`right_hand_split` which also handles optional parentheses.
783 """
784 for leaf_type in [token.LPAR, token.LSQB]:
785 tail_leaves: list[Leaf] = []
786 body_leaves: list[Leaf] = []
787 head_leaves: list[Leaf] = []
788 current_leaves = head_leaves
789 matching_bracket: Optional[Leaf] = None
790 for leaf in line.leaves:
791 if (
792 current_leaves is body_leaves
793 and leaf.type in CLOSING_BRACKETS
794 and leaf.opening_bracket is matching_bracket
795 and isinstance(matching_bracket, Leaf)
796 ):
797 ensure_visible(leaf)
798 ensure_visible(matching_bracket)
799 current_leaves = tail_leaves if body_leaves else head_leaves
800 current_leaves.append(leaf)
801 if current_leaves is head_leaves:
802 if leaf.type == leaf_type:
803 matching_bracket = leaf
804 current_leaves = body_leaves
805 if matching_bracket and tail_leaves:
806 break
807 if not matching_bracket or not tail_leaves:
808 raise CannotSplit("No brackets found")
810 head = bracket_split_build_line(
811 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
812 )
813 body = bracket_split_build_line(
814 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
815 )
816 tail = bracket_split_build_line(
817 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
818 )
819 bracket_split_succeeded_or_raise(head, body, tail)
820 for result in (head, body, tail):
821 if result:
822 yield result
825def right_hand_split(
826 line: Line,
827 mode: Mode,
828 features: Collection[Feature] = (),
829 omit: Collection[LeafID] = (),
830) -> Iterator[Line]:
831 """Split line into many lines, starting with the last matching bracket pair.
833 If the split was by optional parentheses, attempt splitting without them, too.
834 `omit` is a collection of closing bracket IDs that shouldn't be considered for
835 this split.
837 Note: running this function modifies `bracket_depth` on the leaves of `line`.
838 """
839 rhs_result = _first_right_hand_split(line, omit=omit)
840 yield from _maybe_split_omitting_optional_parens(
841 rhs_result, line, mode, features=features, omit=omit
842 )
845def _first_right_hand_split(
846 line: Line,
847 omit: Collection[LeafID] = (),
848) -> RHSResult:
849 """Split the line into head, body, tail starting with the last bracket pair.
851 Note: this function should not have side effects. It's relied upon by
852 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
853 splitting on the right side of an assignment statement.
854 """
855 tail_leaves: list[Leaf] = []
856 body_leaves: list[Leaf] = []
857 head_leaves: list[Leaf] = []
858 current_leaves = tail_leaves
859 opening_bracket: Optional[Leaf] = None
860 closing_bracket: Optional[Leaf] = None
861 for leaf in reversed(line.leaves):
862 if current_leaves is body_leaves:
863 if leaf is opening_bracket:
864 current_leaves = head_leaves if body_leaves else tail_leaves
865 current_leaves.append(leaf)
866 if current_leaves is tail_leaves:
867 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
868 opening_bracket = leaf.opening_bracket
869 closing_bracket = leaf
870 current_leaves = body_leaves
871 if not (opening_bracket and closing_bracket and head_leaves):
872 # If there is no opening or closing_bracket that means the split failed and
873 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
874 # the matching `opening_bracket` wasn't available on `line` anymore.
875 raise CannotSplit("No brackets found")
877 tail_leaves.reverse()
878 body_leaves.reverse()
879 head_leaves.reverse()
881 body: Optional[Line] = None
882 if (
883 Preview.hug_parens_with_braces_and_square_brackets in line.mode
884 and tail_leaves[0].value
885 and tail_leaves[0].opening_bracket is head_leaves[-1]
886 ):
887 inner_body_leaves = list(body_leaves)
888 hugged_opening_leaves: list[Leaf] = []
889 hugged_closing_leaves: list[Leaf] = []
890 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
891 unpacking_offset: int = 1 if is_unpacking else 0
892 while (
893 len(inner_body_leaves) >= 2 + unpacking_offset
894 and inner_body_leaves[-1].type in CLOSING_BRACKETS
895 and inner_body_leaves[-1].opening_bracket
896 is inner_body_leaves[unpacking_offset]
897 ):
898 if unpacking_offset:
899 hugged_opening_leaves.append(inner_body_leaves.pop(0))
900 unpacking_offset = 0
901 hugged_opening_leaves.append(inner_body_leaves.pop(0))
902 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
904 if hugged_opening_leaves and inner_body_leaves:
905 inner_body = bracket_split_build_line(
906 inner_body_leaves,
907 line,
908 hugged_opening_leaves[-1],
909 component=_BracketSplitComponent.body,
910 )
911 if (
912 line.mode.magic_trailing_comma
913 and inner_body_leaves[-1].type == token.COMMA
914 ):
915 should_hug = True
916 else:
917 line_length = line.mode.line_length - sum(
918 len(str(leaf))
919 for leaf in hugged_opening_leaves + hugged_closing_leaves
920 )
921 if is_line_short_enough(
922 inner_body, mode=replace(line.mode, line_length=line_length)
923 ):
924 # Do not hug if it fits on a single line.
925 should_hug = False
926 else:
927 should_hug = True
928 if should_hug:
929 body_leaves = inner_body_leaves
930 head_leaves.extend(hugged_opening_leaves)
931 tail_leaves = hugged_closing_leaves + tail_leaves
932 body = inner_body # No need to re-calculate the body again later.
934 head = bracket_split_build_line(
935 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
936 )
937 if body is None:
938 body = bracket_split_build_line(
939 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
940 )
941 tail = bracket_split_build_line(
942 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
943 )
944 bracket_split_succeeded_or_raise(head, body, tail)
945 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
948def _maybe_split_omitting_optional_parens(
949 rhs: RHSResult,
950 line: Line,
951 mode: Mode,
952 features: Collection[Feature] = (),
953 omit: Collection[LeafID] = (),
954) -> Iterator[Line]:
955 if (
956 Feature.FORCE_OPTIONAL_PARENTHESES not in features
957 # the opening bracket is an optional paren
958 and rhs.opening_bracket.type == token.LPAR
959 and not rhs.opening_bracket.value
960 # the closing bracket is an optional paren
961 and rhs.closing_bracket.type == token.RPAR
962 and not rhs.closing_bracket.value
963 # it's not an import (optional parens are the only thing we can split on
964 # in this case; attempting a split without them is a waste of time)
965 and not line.is_import
966 # and we can actually remove the parens
967 and can_omit_invisible_parens(rhs, mode.line_length)
968 ):
969 omit = {id(rhs.closing_bracket), *omit}
970 try:
971 # The RHSResult Omitting Optional Parens.
972 rhs_oop = _first_right_hand_split(line, omit=omit)
973 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
974 yield from _maybe_split_omitting_optional_parens(
975 rhs_oop, line, mode, features=features, omit=omit
976 )
977 return
979 except CannotSplit as e:
980 # For chained assignments we want to use the previous successful split
981 if line.is_chained_assignment:
982 pass
984 elif (
985 not can_be_split(rhs.body)
986 and not is_line_short_enough(rhs.body, mode=mode)
987 and not (
988 Preview.wrap_long_dict_values_in_parens
989 and rhs.opening_bracket.parent
990 and rhs.opening_bracket.parent.parent
991 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
992 )
993 ):
994 raise CannotSplit(
995 "Splitting failed, body is still too long and can't be split."
996 ) from e
998 elif (
999 rhs.head.contains_multiline_strings()
1000 or rhs.tail.contains_multiline_strings()
1001 ):
1002 raise CannotSplit(
1003 "The current optional pair of parentheses is bound to fail to"
1004 " satisfy the splitting algorithm because the head or the tail"
1005 " contains multiline strings which by definition never fit one"
1006 " line."
1007 ) from e
1009 ensure_visible(rhs.opening_bracket)
1010 ensure_visible(rhs.closing_bracket)
1011 for result in (rhs.head, rhs.body, rhs.tail):
1012 if result:
1013 yield result
1016def _prefer_split_rhs_oop_over_rhs(
1017 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1018) -> bool:
1019 """
1020 Returns whether we should prefer the result from a split omitting optional parens
1021 (rhs_oop) over the original (rhs).
1022 """
1023 # contains unsplittable type ignore
1024 if (
1025 rhs_oop.head.contains_unsplittable_type_ignore()
1026 or rhs_oop.body.contains_unsplittable_type_ignore()
1027 or rhs_oop.tail.contains_unsplittable_type_ignore()
1028 ):
1029 return True
1031 # Retain optional parens around dictionary values
1032 if (
1033 Preview.wrap_long_dict_values_in_parens
1034 and rhs.opening_bracket.parent
1035 and rhs.opening_bracket.parent.parent
1036 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1037 and rhs.body.bracket_tracker.delimiters
1038 ):
1039 # Unless the split is inside the key
1040 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1042 # the split is right after `=`
1043 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1044 return True
1046 # the left side of assignment contains brackets
1047 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1048 return True
1050 # the left side of assignment is short enough (the -1 is for the ending optional
1051 # paren)
1052 if not is_line_short_enough(
1053 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1054 ):
1055 return True
1057 # the left side of assignment won't explode further because of magic trailing comma
1058 if rhs.head.magic_trailing_comma is not None:
1059 return True
1061 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1062 # the body
1063 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1064 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1065 token.EQUAL
1066 )
1067 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1068 return False
1070 has_closing_bracket_after_assign = False
1071 for leaf in reversed(rhs_oop.head.leaves):
1072 if leaf.type == token.EQUAL:
1073 break
1074 if leaf.type in CLOSING_BRACKETS:
1075 has_closing_bracket_after_assign = True
1076 break
1077 return (
1078 # contains matching brackets after the `=` (done by checking there is a
1079 # closing bracket)
1080 has_closing_bracket_after_assign
1081 or (
1082 # the split is actually from inside the optional parens (done by checking
1083 # the first line still contains the `=`)
1084 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1085 # the first line is short enough
1086 and is_line_short_enough(rhs_oop.head, mode=mode)
1087 )
1088 )
1091def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1092 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1094 Do nothing otherwise.
1096 A left- or right-hand split is based on a pair of brackets. Content before
1097 (and including) the opening bracket is left on one line, content inside the
1098 brackets is put on a separate line, and finally content starting with and
1099 following the closing bracket is put on a separate line.
1101 Those are called `head`, `body`, and `tail`, respectively. If the split
1102 produced the same line (all content in `head`) or ended up with an empty `body`
1103 and the `tail` is just the closing bracket, then it's considered failed.
1104 """
1105 tail_len = len(str(tail).strip())
1106 if not body:
1107 if tail_len == 0:
1108 raise CannotSplit("Splitting brackets produced the same line")
1110 elif tail_len < 3:
1111 raise CannotSplit(
1112 f"Splitting brackets on an empty body to save {tail_len} characters is"
1113 " not worth it"
1114 )
1117def _ensure_trailing_comma(
1118 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1119) -> bool:
1120 if not leaves:
1121 return False
1122 # Ensure a trailing comma for imports
1123 if original.is_import:
1124 return True
1125 # ...and standalone function arguments
1126 if not original.is_def:
1127 return False
1128 if opening_bracket.value != "(":
1129 return False
1130 # Don't add commas if we already have any commas
1131 if any(
1132 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1133 ):
1134 return False
1136 # Find a leaf with a parent (comments don't have parents)
1137 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1138 if leaf_with_parent is None:
1139 return True
1140 # Don't add commas inside parenthesized return annotations
1141 if get_annotation_type(leaf_with_parent) == "return":
1142 return False
1143 # Don't add commas inside PEP 604 unions
1144 if (
1145 leaf_with_parent.parent
1146 and leaf_with_parent.parent.next_sibling
1147 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1148 ):
1149 return False
1150 return True
1153def bracket_split_build_line(
1154 leaves: list[Leaf],
1155 original: Line,
1156 opening_bracket: Leaf,
1157 *,
1158 component: _BracketSplitComponent,
1159) -> Line:
1160 """Return a new line with given `leaves` and respective comments from `original`.
1162 If it's the head component, brackets will be tracked so trailing commas are
1163 respected.
1165 If it's the body component, the result line is one-indented inside brackets and as
1166 such has its first leaf's prefix normalized and a trailing comma added when
1167 expected.
1168 """
1169 result = Line(mode=original.mode, depth=original.depth)
1170 if component is _BracketSplitComponent.body:
1171 result.inside_brackets = True
1172 result.depth += 1
1173 if _ensure_trailing_comma(leaves, original, opening_bracket):
1174 for i in range(len(leaves) - 1, -1, -1):
1175 if leaves[i].type == STANDALONE_COMMENT:
1176 continue
1178 if leaves[i].type != token.COMMA:
1179 new_comma = Leaf(token.COMMA, ",")
1180 leaves.insert(i + 1, new_comma)
1181 break
1183 leaves_to_track: set[LeafID] = set()
1184 if component is _BracketSplitComponent.head:
1185 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1186 # Populate the line
1187 for leaf in leaves:
1188 result.append(
1189 leaf,
1190 preformatted=True,
1191 track_bracket=id(leaf) in leaves_to_track,
1192 )
1193 for comment_after in original.comments_after(leaf):
1194 result.append(comment_after, preformatted=True)
1195 if component is _BracketSplitComponent.body and should_split_line(
1196 result, opening_bracket
1197 ):
1198 result.should_split_rhs = True
1199 return result
1202def dont_increase_indentation(split_func: Transformer) -> Transformer:
1203 """Normalize prefix of the first leaf in every line returned by `split_func`.
1205 This is a decorator over relevant split functions.
1206 """
1208 @wraps(split_func)
1209 def split_wrapper(
1210 line: Line, features: Collection[Feature], mode: Mode
1211 ) -> Iterator[Line]:
1212 for split_line in split_func(line, features, mode):
1213 split_line.leaves[0].prefix = ""
1214 yield split_line
1216 return split_wrapper
1219def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1220 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1221 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1222 return leaf_idx
1223 return None
1226def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1227 if is_vararg(leaf, within={syms.typedargslist}):
1228 return Feature.TRAILING_COMMA_IN_DEF in features
1229 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1230 return Feature.TRAILING_COMMA_IN_CALL in features
1231 return True
1234def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1235 if (
1236 safe
1237 and delimiter_priority == COMMA_PRIORITY
1238 and line.leaves[-1].type != token.COMMA
1239 and line.leaves[-1].type != STANDALONE_COMMENT
1240 ):
1241 new_comma = Leaf(token.COMMA, ",")
1242 line.append(new_comma)
1243 return line
1246MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1249@dont_increase_indentation
1250def delimiter_split(
1251 line: Line, features: Collection[Feature], mode: Mode
1252) -> Iterator[Line]:
1253 """Split according to delimiters of the highest priority.
1255 If the appropriate Features are given, the split will add trailing commas
1256 also in function signatures and calls that contain `*` and `**`.
1257 """
1258 if len(line.leaves) == 0:
1259 raise CannotSplit("Line empty") from None
1260 last_leaf = line.leaves[-1]
1262 bt = line.bracket_tracker
1263 try:
1264 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1265 except ValueError:
1266 raise CannotSplit("No delimiters found") from None
1268 if (
1269 delimiter_priority == DOT_PRIORITY
1270 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1271 ):
1272 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1274 current_line = Line(
1275 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1276 )
1277 lowest_depth = sys.maxsize
1278 trailing_comma_safe = True
1280 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1281 """Append `leaf` to current line or to new line if appending impossible."""
1282 nonlocal current_line
1283 try:
1284 current_line.append_safe(leaf, preformatted=True)
1285 except ValueError:
1286 yield current_line
1288 current_line = Line(
1289 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1290 )
1291 current_line.append(leaf)
1293 def append_comments(leaf: Leaf) -> Iterator[Line]:
1294 for comment_after in line.comments_after(leaf):
1295 yield from append_to_line(comment_after)
1297 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1298 for leaf_idx, leaf in enumerate(line.leaves):
1299 yield from append_to_line(leaf)
1301 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1302 id(line.leaves[leaf_idx - 1])
1303 )
1304 if (
1305 previous_priority != delimiter_priority
1306 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1307 ):
1308 yield from append_comments(leaf)
1310 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1311 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1312 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1314 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1315 current_line = _safe_add_trailing_comma(
1316 trailing_comma_safe, delimiter_priority, current_line
1317 )
1319 leaf_priority = bt.delimiters.get(id(leaf))
1320 if leaf_priority == delimiter_priority:
1321 if (
1322 leaf_idx + 1 < len(line.leaves)
1323 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1324 ):
1325 yield from append_comments(line.leaves[leaf_idx + 1])
1327 yield current_line
1328 current_line = Line(
1329 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1330 )
1332 if current_line:
1333 current_line = _safe_add_trailing_comma(
1334 trailing_comma_safe, delimiter_priority, current_line
1335 )
1336 yield current_line
1339@dont_increase_indentation
1340def standalone_comment_split(
1341 line: Line, features: Collection[Feature], mode: Mode
1342) -> Iterator[Line]:
1343 """Split standalone comments from the rest of the line."""
1344 if not line.contains_standalone_comments():
1345 raise CannotSplit("Line does not have any standalone comments")
1347 current_line = Line(
1348 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1349 )
1351 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1352 """Append `leaf` to current line or to new line if appending impossible."""
1353 nonlocal current_line
1354 try:
1355 current_line.append_safe(leaf, preformatted=True)
1356 except ValueError:
1357 yield current_line
1359 current_line = Line(
1360 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1361 )
1362 current_line.append(leaf)
1364 for leaf in line.leaves:
1365 yield from append_to_line(leaf)
1367 for comment_after in line.comments_after(leaf):
1368 yield from append_to_line(comment_after)
1370 if current_line:
1371 yield current_line
1374def normalize_invisible_parens( # noqa: C901
1375 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1376) -> None:
1377 """Make existing optional parentheses invisible or create new ones.
1379 `parens_after` is a set of string leaf values immediately after which parens
1380 should be put.
1382 Standardizes on visible parentheses for single-element tuples, and keeps
1383 existing visible parentheses for other tuples and generator expressions.
1384 """
1385 for pc in list_comments(node.prefix, is_endmarker=False):
1386 if pc.value in FMT_OFF:
1387 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1388 return
1390 # The multiple context managers grammar has a different pattern, thus this is
1391 # separate from the for-loop below. This possibly wraps them in invisible parens,
1392 # and later will be removed in remove_with_parens when needed.
1393 if node.type == syms.with_stmt:
1394 _maybe_wrap_cms_in_parens(node, mode, features)
1396 check_lpar = False
1397 for index, child in enumerate(list(node.children)):
1398 # Fixes a bug where invisible parens are not properly stripped from
1399 # assignment statements that contain type annotations.
1400 if isinstance(child, Node) and child.type == syms.annassign:
1401 normalize_invisible_parens(
1402 child, parens_after=parens_after, mode=mode, features=features
1403 )
1405 # Fixes a bug where invisible parens are not properly wrapped around
1406 # case blocks.
1407 if isinstance(child, Node) and child.type == syms.case_block:
1408 normalize_invisible_parens(
1409 child, parens_after={"case"}, mode=mode, features=features
1410 )
1412 # Add parentheses around if guards in case blocks
1413 if isinstance(child, Node) and child.type == syms.guard:
1414 normalize_invisible_parens(
1415 child, parens_after={"if"}, mode=mode, features=features
1416 )
1418 # Add parentheses around long tuple unpacking in assignments.
1419 if (
1420 index == 0
1421 and isinstance(child, Node)
1422 and child.type == syms.testlist_star_expr
1423 ):
1424 check_lpar = True
1426 if check_lpar:
1427 if (
1428 child.type == syms.atom
1429 and node.type == syms.for_stmt
1430 and isinstance(child.prev_sibling, Leaf)
1431 and child.prev_sibling.type == token.NAME
1432 and child.prev_sibling.value == "for"
1433 ):
1434 if maybe_make_parens_invisible_in_atom(
1435 child,
1436 parent=node,
1437 remove_brackets_around_comma=True,
1438 ):
1439 wrap_in_parentheses(node, child, visible=False)
1440 elif isinstance(child, Node) and node.type == syms.with_stmt:
1441 remove_with_parens(child, node)
1442 elif child.type == syms.atom:
1443 if maybe_make_parens_invisible_in_atom(
1444 child,
1445 parent=node,
1446 ):
1447 wrap_in_parentheses(node, child, visible=False)
1448 elif is_one_tuple(child):
1449 wrap_in_parentheses(node, child, visible=True)
1450 elif node.type == syms.import_from:
1451 _normalize_import_from(node, child, index)
1452 break
1453 elif (
1454 index == 1
1455 and child.type == token.STAR
1456 and node.type == syms.except_clause
1457 ):
1458 # In except* (PEP 654), the star is actually part of
1459 # of the keyword. So we need to skip the insertion of
1460 # invisible parentheses to work more precisely.
1461 continue
1463 elif (
1464 isinstance(child, Leaf)
1465 and child.next_sibling is not None
1466 and child.next_sibling.type == token.COLON
1467 and child.value == "case"
1468 ):
1469 # A special patch for "case case:" scenario, the second occurrence
1470 # of case will be not parsed as a Python keyword.
1471 break
1473 elif not is_multiline_string(child):
1474 wrap_in_parentheses(node, child, visible=False)
1476 comma_check = child.type == token.COMMA
1478 check_lpar = isinstance(child, Leaf) and (
1479 child.value in parens_after or comma_check
1480 )
1483def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1484 # "import from" nodes store parentheses directly as part of
1485 # the statement
1486 if is_lpar_token(child):
1487 assert is_rpar_token(parent.children[-1])
1488 # make parentheses invisible
1489 child.value = ""
1490 parent.children[-1].value = ""
1491 elif child.type != token.STAR:
1492 # insert invisible parentheses
1493 parent.insert_child(index, Leaf(token.LPAR, ""))
1494 parent.append_child(Leaf(token.RPAR, ""))
1497def remove_await_parens(node: Node) -> None:
1498 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1499 if (
1500 node.children[1].type == syms.atom
1501 and node.children[1].children[0].type == token.LPAR
1502 ):
1503 if maybe_make_parens_invisible_in_atom(
1504 node.children[1],
1505 parent=node,
1506 remove_brackets_around_comma=True,
1507 ):
1508 wrap_in_parentheses(node, node.children[1], visible=False)
1510 # Since await is an expression we shouldn't remove
1511 # brackets in cases where this would change
1512 # the AST due to operator precedence.
1513 # Therefore we only aim to remove brackets around
1514 # power nodes that aren't also await expressions themselves.
1515 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1516 # N.B. We've still removed any redundant nested brackets though :)
1517 opening_bracket = cast(Leaf, node.children[1].children[0])
1518 closing_bracket = cast(Leaf, node.children[1].children[-1])
1519 bracket_contents = node.children[1].children[1]
1520 if isinstance(bracket_contents, Node) and (
1521 bracket_contents.type != syms.power
1522 or bracket_contents.children[0].type == token.AWAIT
1523 or any(
1524 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1525 for child in bracket_contents.children
1526 )
1527 ):
1528 ensure_visible(opening_bracket)
1529 ensure_visible(closing_bracket)
1532def _maybe_wrap_cms_in_parens(
1533 node: Node, mode: Mode, features: Collection[Feature]
1534) -> None:
1535 """When enabled and safe, wrap the multiple context managers in invisible parens.
1537 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1538 """
1539 if (
1540 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1541 or len(node.children) <= 2
1542 # If it's an atom, it's already wrapped in parens.
1543 or node.children[1].type == syms.atom
1544 ):
1545 return
1546 colon_index: Optional[int] = None
1547 for i in range(2, len(node.children)):
1548 if node.children[i].type == token.COLON:
1549 colon_index = i
1550 break
1551 if colon_index is not None:
1552 lpar = Leaf(token.LPAR, "")
1553 rpar = Leaf(token.RPAR, "")
1554 context_managers = node.children[1:colon_index]
1555 for child in context_managers:
1556 child.remove()
1557 # After wrapping, the with_stmt will look like this:
1558 # with_stmt
1559 # NAME 'with'
1560 # atom
1561 # LPAR ''
1562 # testlist_gexp
1563 # ... <-- context_managers
1564 # /testlist_gexp
1565 # RPAR ''
1566 # /atom
1567 # COLON ':'
1568 new_child = Node(
1569 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1570 )
1571 node.insert_child(1, new_child)
1574def remove_with_parens(node: Node, parent: Node) -> None:
1575 """Recursively hide optional parens in `with` statements."""
1576 # Removing all unnecessary parentheses in with statements in one pass is a tad
1577 # complex as different variations of bracketed statements result in pretty
1578 # different parse trees:
1579 #
1580 # with (open("file")) as f: # this is an asexpr_test
1581 # ...
1582 #
1583 # with (open("file") as f): # this is an atom containing an
1584 # ... # asexpr_test
1585 #
1586 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1587 # ... # asexpr_test
1588 #
1589 # with (open("file") as f, open("file") as f): # an atom containing a
1590 # ... # testlist_gexp which then
1591 # # contains multiple asexpr_test(s)
1592 if node.type == syms.atom:
1593 if maybe_make_parens_invisible_in_atom(
1594 node,
1595 parent=parent,
1596 remove_brackets_around_comma=True,
1597 ):
1598 wrap_in_parentheses(parent, node, visible=False)
1599 if isinstance(node.children[1], Node):
1600 remove_with_parens(node.children[1], node)
1601 elif node.type == syms.testlist_gexp:
1602 for child in node.children:
1603 if isinstance(child, Node):
1604 remove_with_parens(child, node)
1605 elif node.type == syms.asexpr_test and not any(
1606 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1607 ):
1608 if maybe_make_parens_invisible_in_atom(
1609 node.children[0],
1610 parent=node,
1611 remove_brackets_around_comma=True,
1612 ):
1613 wrap_in_parentheses(node, node.children[0], visible=False)
1616def maybe_make_parens_invisible_in_atom(
1617 node: LN,
1618 parent: LN,
1619 remove_brackets_around_comma: bool = False,
1620) -> bool:
1621 """If it's safe, make the parens in the atom `node` invisible, recursively.
1622 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1623 as they are redundant.
1625 Returns whether the node should itself be wrapped in invisible parentheses.
1626 """
1627 if (
1628 node.type not in (syms.atom, syms.expr)
1629 or is_empty_tuple(node)
1630 or is_one_tuple(node)
1631 or (is_tuple(node) and parent.type == syms.asexpr_test)
1632 or (
1633 is_tuple(node)
1634 and parent.type == syms.with_stmt
1635 and has_sibling_with_type(node, token.COMMA)
1636 )
1637 or (is_yield(node) and parent.type != syms.expr_stmt)
1638 or (
1639 # This condition tries to prevent removing non-optional brackets
1640 # around a tuple, however, can be a bit overzealous so we provide
1641 # and option to skip this check for `for` and `with` statements.
1642 not remove_brackets_around_comma
1643 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1644 )
1645 or is_tuple_containing_walrus(node)
1646 or is_tuple_containing_star(node)
1647 or is_generator(node)
1648 ):
1649 return False
1651 if is_walrus_assignment(node):
1652 if parent.type in [
1653 syms.annassign,
1654 syms.expr_stmt,
1655 syms.assert_stmt,
1656 syms.return_stmt,
1657 syms.except_clause,
1658 syms.funcdef,
1659 syms.with_stmt,
1660 syms.testlist_gexp,
1661 syms.tname,
1662 # these ones aren't useful to end users, but they do please fuzzers
1663 syms.for_stmt,
1664 syms.del_stmt,
1665 syms.for_stmt,
1666 ]:
1667 return False
1669 first = node.children[0]
1670 last = node.children[-1]
1671 if is_lpar_token(first) and is_rpar_token(last):
1672 middle = node.children[1]
1673 # make parentheses invisible
1674 if (
1675 # If the prefix of `middle` includes a type comment with
1676 # ignore annotation, then we do not remove the parentheses
1677 not is_type_ignore_comment_string(middle.prefix.strip())
1678 ):
1679 first.value = ""
1680 last.value = ""
1681 maybe_make_parens_invisible_in_atom(
1682 middle,
1683 parent=parent,
1684 remove_brackets_around_comma=remove_brackets_around_comma,
1685 )
1687 if is_atom_with_invisible_parens(middle):
1688 # Strip the invisible parens from `middle` by replacing
1689 # it with the child in-between the invisible parens
1690 middle.replace(middle.children[1])
1692 if middle.children[0].prefix.strip():
1693 # Preserve comments before first paren
1694 middle.children[1].prefix = (
1695 middle.children[0].prefix + middle.children[1].prefix
1696 )
1698 if middle.children[-1].prefix.strip():
1699 # Preserve comments before last paren
1700 last.prefix = middle.children[-1].prefix + last.prefix
1702 return False
1704 return True
1707def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1708 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1710 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1711 return False
1713 # We're essentially checking if the body is delimited by commas and there's more
1714 # than one of them (we're excluding the trailing comma and if the delimiter priority
1715 # is still commas, that means there's more).
1716 exclude = set()
1717 trailing_comma = False
1718 try:
1719 last_leaf = line.leaves[-1]
1720 if last_leaf.type == token.COMMA:
1721 trailing_comma = True
1722 exclude.add(id(last_leaf))
1723 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1724 except (IndexError, ValueError):
1725 return False
1727 return max_priority == COMMA_PRIORITY and (
1728 (line.mode.magic_trailing_comma and trailing_comma)
1729 # always explode imports
1730 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1731 )
1734def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1735 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1737 Brackets can be omitted if the entire trailer up to and including
1738 a preceding closing bracket fits in one line.
1740 Yielded sets are cumulative (contain results of previous yields, too). First
1741 set is empty, unless the line should explode, in which case bracket pairs until
1742 the one that needs to explode are omitted.
1743 """
1745 omit: set[LeafID] = set()
1746 if not line.magic_trailing_comma:
1747 yield omit
1749 length = 4 * line.depth
1750 opening_bracket: Optional[Leaf] = None
1751 closing_bracket: Optional[Leaf] = None
1752 inner_brackets: set[LeafID] = set()
1753 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1754 length += leaf_length
1755 if length > line_length:
1756 break
1758 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1759 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1760 break
1762 if opening_bracket:
1763 if leaf is opening_bracket:
1764 opening_bracket = None
1765 elif leaf.type in CLOSING_BRACKETS:
1766 prev = line.leaves[index - 1] if index > 0 else None
1767 if (
1768 prev
1769 and prev.type == token.COMMA
1770 and leaf.opening_bracket is not None
1771 and not is_one_sequence_between(
1772 leaf.opening_bracket, leaf, line.leaves
1773 )
1774 ):
1775 # Never omit bracket pairs with trailing commas.
1776 # We need to explode on those.
1777 break
1779 inner_brackets.add(id(leaf))
1780 elif leaf.type in CLOSING_BRACKETS:
1781 prev = line.leaves[index - 1] if index > 0 else None
1782 if prev and prev.type in OPENING_BRACKETS:
1783 # Empty brackets would fail a split so treat them as "inner"
1784 # brackets (e.g. only add them to the `omit` set if another
1785 # pair of brackets was good enough.
1786 inner_brackets.add(id(leaf))
1787 continue
1789 if closing_bracket:
1790 omit.add(id(closing_bracket))
1791 omit.update(inner_brackets)
1792 inner_brackets.clear()
1793 yield omit
1795 if (
1796 prev
1797 and prev.type == token.COMMA
1798 and leaf.opening_bracket is not None
1799 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1800 ):
1801 # Never omit bracket pairs with trailing commas.
1802 # We need to explode on those.
1803 break
1805 if leaf.value:
1806 opening_bracket = leaf.opening_bracket
1807 closing_bracket = leaf
1810def run_transformer(
1811 line: Line,
1812 transform: Transformer,
1813 mode: Mode,
1814 features: Collection[Feature],
1815 *,
1816 line_str: str = "",
1817) -> list[Line]:
1818 if not line_str:
1819 line_str = line_to_string(line)
1820 result: list[Line] = []
1821 for transformed_line in transform(line, features, mode):
1822 if str(transformed_line).strip("\n") == line_str:
1823 raise CannotTransform("Line transformer returned an unchanged result")
1825 result.extend(transform_line(transformed_line, mode=mode, features=features))
1827 features_set = set(features)
1828 if (
1829 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1830 or transform.__class__.__name__ != "rhs"
1831 or not line.bracket_tracker.invisible
1832 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1833 or line.contains_multiline_strings()
1834 or result[0].contains_uncollapsable_type_comments()
1835 or result[0].contains_unsplittable_type_ignore()
1836 or is_line_short_enough(result[0], mode=mode)
1837 # If any leaves have no parents (which _can_ occur since
1838 # `transform(line)` potentially destroys the line's underlying node
1839 # structure), then we can't proceed. Doing so would cause the below
1840 # call to `append_leaves()` to fail.
1841 or any(leaf.parent is None for leaf in line.leaves)
1842 ):
1843 return result
1845 line_copy = line.clone()
1846 append_leaves(line_copy, line, line.leaves)
1847 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1848 second_opinion = run_transformer(
1849 line_copy, transform, mode, features_fop, line_str=line_str
1850 )
1851 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1852 result = second_opinion
1853 return result