Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Optional, Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import FMT_OFF, generate_comments, list_comments
21from black.lines import (
22 Line,
23 RHSResult,
24 append_leaves,
25 can_be_split,
26 can_omit_invisible_parens,
27 is_line_short_enough,
28 line_to_string,
29)
30from black.mode import Feature, Mode, Preview
31from black.nodes import (
32 ASSIGNMENTS,
33 BRACKETS,
34 CLOSING_BRACKETS,
35 OPENING_BRACKETS,
36 STANDALONE_COMMENT,
37 STATEMENT,
38 WHITESPACE,
39 Visitor,
40 ensure_visible,
41 fstring_to_string,
42 get_annotation_type,
43 has_sibling_with_type,
44 is_arith_like,
45 is_async_stmt_or_funcdef,
46 is_atom_with_invisible_parens,
47 is_docstring,
48 is_empty_tuple,
49 is_generator,
50 is_lpar_token,
51 is_multiline_string,
52 is_name_token,
53 is_one_sequence_between,
54 is_one_tuple,
55 is_parent_function_or_class,
56 is_part_of_annotation,
57 is_rpar_token,
58 is_stub_body,
59 is_stub_suite,
60 is_tuple,
61 is_tuple_containing_star,
62 is_tuple_containing_walrus,
63 is_type_ignore_comment_string,
64 is_vararg,
65 is_walrus_assignment,
66 is_yield,
67 syms,
68 wrap_in_parentheses,
69)
70from black.numerics import normalize_numeric_literal
71from black.strings import (
72 fix_multiline_docstring,
73 get_string_prefix,
74 normalize_string_prefix,
75 normalize_string_quotes,
76 normalize_unicode_escape_sequences,
77)
78from black.trans import (
79 CannotTransform,
80 StringMerger,
81 StringParenStripper,
82 StringParenWrapper,
83 StringSplitter,
84 Transformer,
85 hug_power_op,
86)
87from blib2to3.pgen2 import token
88from blib2to3.pytree import Leaf, Node
90# types
91LeafID = int
92LN = Union[Leaf, Node]
95class CannotSplit(CannotTransform):
96 """A readable split that fits the allotted line length is impossible."""
99# This isn't a dataclass because @dataclass + Generic breaks mypyc.
100# See also https://github.com/mypyc/mypyc/issues/827.
101class LineGenerator(Visitor[Line]):
102 """Generates reformatted Line objects. Empty lines are not emitted.
104 Note: destroys the tree it's visiting by mutating prefixes of its leaves
105 in ways that will no longer stringify to valid Python code on the tree.
106 """
108 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
109 self.mode = mode
110 self.features = features
111 self.current_line: Line
112 self.__post_init__()
114 def line(self, indent: int = 0) -> Iterator[Line]:
115 """Generate a line.
117 If the line is empty, only emit if it makes sense.
118 If the line is too long, split it first and then generate.
120 If any lines were generated, set up a new current_line.
121 """
122 if not self.current_line:
123 self.current_line.depth += indent
124 return # Line is empty, don't emit. Creating a new one unnecessary.
126 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
127 self.current_line.leaves[0]
128 ):
129 # Special case for async def/for/with statements. `visit_async_stmt`
130 # adds an `ASYNC` leaf then visits the child def/for/with statement
131 # nodes. Line yields from those nodes shouldn't treat the former
132 # `ASYNC` leaf as a complete line.
133 return
135 complete_line = self.current_line
136 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
137 yield complete_line
139 def visit_default(self, node: LN) -> Iterator[Line]:
140 """Default `visit_*()` implementation. Recurses to children of `node`."""
141 if isinstance(node, Leaf):
142 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
143 for comment in generate_comments(node):
144 if any_open_brackets:
145 # any comment within brackets is subject to splitting
146 self.current_line.append(comment)
147 elif comment.type == token.COMMENT:
148 # regular trailing comment
149 self.current_line.append(comment)
150 yield from self.line()
152 else:
153 # regular standalone comment
154 yield from self.line()
156 self.current_line.append(comment)
157 yield from self.line()
159 if any_open_brackets:
160 node.prefix = ""
161 if node.type not in WHITESPACE:
162 self.current_line.append(node)
163 yield from super().visit_default(node)
165 def visit_test(self, node: Node) -> Iterator[Line]:
166 """Visit an `x if y else z` test"""
168 already_parenthesized = (
169 node.prev_sibling and node.prev_sibling.type == token.LPAR
170 )
172 if not already_parenthesized:
173 # Similar to logic in wrap_in_parentheses
174 lpar = Leaf(token.LPAR, "")
175 rpar = Leaf(token.RPAR, "")
176 prefix = node.prefix
177 node.prefix = ""
178 lpar.prefix = prefix
179 node.insert_child(0, lpar)
180 node.append_child(rpar)
182 yield from self.visit_default(node)
184 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
185 """Increase indentation level, maybe yield a line."""
186 # In blib2to3 INDENT never holds comments.
187 yield from self.line(+1)
188 yield from self.visit_default(node)
190 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
191 """Decrease indentation level, maybe yield a line."""
192 # The current line might still wait for trailing comments. At DEDENT time
193 # there won't be any (they would be prefixes on the preceding NEWLINE).
194 # Emit the line then.
195 yield from self.line()
197 # While DEDENT has no value, its prefix may contain standalone comments
198 # that belong to the current indentation level. Get 'em.
199 yield from self.visit_default(node)
201 # Finally, emit the dedent.
202 yield from self.line(-1)
204 def visit_stmt(
205 self, node: Node, keywords: set[str], parens: set[str]
206 ) -> Iterator[Line]:
207 """Visit a statement.
209 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
210 `def`, `with`, `class`, `assert`, and assignments.
212 The relevant Python language `keywords` for a given statement will be
213 NAME leaves within it. This methods puts those on a separate line.
215 `parens` holds a set of string leaf values immediately after which
216 invisible parens should be put.
217 """
218 normalize_invisible_parens(
219 node, parens_after=parens, mode=self.mode, features=self.features
220 )
221 for child in node.children:
222 if is_name_token(child) and child.value in keywords:
223 yield from self.line()
225 yield from self.visit(child)
227 def visit_typeparams(self, node: Node) -> Iterator[Line]:
228 yield from self.visit_default(node)
229 node.children[0].prefix = ""
231 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
232 yield from self.visit_default(node)
233 node.children[1].prefix = ""
235 def visit_paramspec(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[1].prefix = ""
239 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
240 if Preview.wrap_long_dict_values_in_parens in self.mode:
241 for i, child in enumerate(node.children):
242 if i == 0:
243 continue
244 if node.children[i - 1].type == token.COLON:
245 if (
246 child.type == syms.atom
247 and child.children[0].type in OPENING_BRACKETS
248 and not is_walrus_assignment(child)
249 ):
250 maybe_make_parens_invisible_in_atom(
251 child,
252 parent=node,
253 mode=self.mode,
254 features=self.features,
255 remove_brackets_around_comma=False,
256 )
257 else:
258 wrap_in_parentheses(node, child, visible=False)
259 yield from self.visit_default(node)
261 def visit_funcdef(self, node: Node) -> Iterator[Line]:
262 """Visit function definition."""
263 yield from self.line()
265 # Remove redundant brackets around return type annotation.
266 is_return_annotation = False
267 for child in node.children:
268 if child.type == token.RARROW:
269 is_return_annotation = True
270 elif is_return_annotation:
271 if child.type == syms.atom and child.children[0].type == token.LPAR:
272 if maybe_make_parens_invisible_in_atom(
273 child,
274 parent=node,
275 mode=self.mode,
276 features=self.features,
277 remove_brackets_around_comma=False,
278 ):
279 wrap_in_parentheses(node, child, visible=False)
280 else:
281 wrap_in_parentheses(node, child, visible=False)
282 is_return_annotation = False
284 for child in node.children:
285 yield from self.visit(child)
287 def visit_match_case(self, node: Node) -> Iterator[Line]:
288 """Visit either a match or case statement."""
289 normalize_invisible_parens(
290 node, parens_after=set(), mode=self.mode, features=self.features
291 )
293 yield from self.line()
294 for child in node.children:
295 yield from self.visit(child)
297 def visit_suite(self, node: Node) -> Iterator[Line]:
298 """Visit a suite."""
299 if is_stub_suite(node):
300 yield from self.visit(node.children[2])
301 else:
302 yield from self.visit_default(node)
304 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
305 """Visit a statement without nested statements."""
306 prev_type: Optional[int] = None
307 for child in node.children:
308 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
309 wrap_in_parentheses(node, child, visible=False)
310 prev_type = child.type
312 if node.parent and node.parent.type in STATEMENT:
313 if is_parent_function_or_class(node) and is_stub_body(node):
314 yield from self.visit_default(node)
315 else:
316 yield from self.line(+1)
317 yield from self.visit_default(node)
318 yield from self.line(-1)
320 else:
321 if node.parent and is_stub_suite(node.parent):
322 node.prefix = ""
323 yield from self.visit_default(node)
324 return
325 yield from self.line()
326 yield from self.visit_default(node)
328 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
329 """Visit `async def`, `async for`, `async with`."""
330 yield from self.line()
332 children = iter(node.children)
333 for child in children:
334 yield from self.visit(child)
336 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
337 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
338 # line.
339 break
341 internal_stmt = next(children)
342 yield from self.visit(internal_stmt)
344 def visit_decorators(self, node: Node) -> Iterator[Line]:
345 """Visit decorators."""
346 for child in node.children:
347 yield from self.line()
348 yield from self.visit(child)
350 def visit_power(self, node: Node) -> Iterator[Line]:
351 for idx, leaf in enumerate(node.children[:-1]):
352 next_leaf = node.children[idx + 1]
354 if not isinstance(leaf, Leaf):
355 continue
357 value = leaf.value.lower()
358 if (
359 leaf.type == token.NUMBER
360 and next_leaf.type == syms.trailer
361 # Ensure that we are in an attribute trailer
362 and next_leaf.children[0].type == token.DOT
363 # It shouldn't wrap hexadecimal, binary and octal literals
364 and not value.startswith(("0x", "0b", "0o"))
365 # It shouldn't wrap complex literals
366 and "j" not in value
367 ):
368 wrap_in_parentheses(node, leaf)
370 remove_await_parens(node, mode=self.mode, features=self.features)
372 yield from self.visit_default(node)
374 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
375 """Remove a semicolon and put the other statement on a separate line."""
376 yield from self.line()
378 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
379 """End of file. Process outstanding comments and end with a newline."""
380 yield from self.visit_default(leaf)
381 yield from self.line()
383 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
384 if not self.current_line.bracket_tracker.any_open_brackets():
385 yield from self.line()
386 yield from self.visit_default(leaf)
388 def visit_factor(self, node: Node) -> Iterator[Line]:
389 """Force parentheses between a unary op and a binary power:
391 -2 ** 8 -> -(2 ** 8)
392 """
393 _operator, operand = node.children
394 if (
395 operand.type == syms.power
396 and len(operand.children) == 3
397 and operand.children[1].type == token.DOUBLESTAR
398 ):
399 lpar = Leaf(token.LPAR, "(")
400 rpar = Leaf(token.RPAR, ")")
401 index = operand.remove() or 0
402 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
403 yield from self.visit_default(node)
405 def visit_tname(self, node: Node) -> Iterator[Line]:
406 """
407 Add potential parentheses around types in function parameter lists to be made
408 into real parentheses in case the type hint is too long to fit on a line
409 Examples:
410 def foo(a: int, b: float = 7): ...
412 ->
414 def foo(a: (int), b: (float) = 7): ...
415 """
416 assert len(node.children) == 3
417 if maybe_make_parens_invisible_in_atom(
418 node.children[2], parent=node, mode=self.mode, features=self.features
419 ):
420 wrap_in_parentheses(node, node.children[2], visible=False)
422 yield from self.visit_default(node)
424 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
425 normalize_unicode_escape_sequences(leaf)
427 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
428 # We're ignoring docstrings with backslash newline escapes because changing
429 # indentation of those changes the AST representation of the code.
430 if self.mode.string_normalization:
431 docstring = normalize_string_prefix(leaf.value)
432 # We handle string normalization at the end of this method, but since
433 # what we do right now acts differently depending on quote style (ex.
434 # see padding logic below), there's a possibility for unstable
435 # formatting. To avoid a situation where this function formats a
436 # docstring differently on the second pass, normalize it early.
437 docstring = normalize_string_quotes(docstring)
438 else:
439 docstring = leaf.value
440 prefix = get_string_prefix(docstring)
441 docstring = docstring[len(prefix) :] # Remove the prefix
442 quote_char = docstring[0]
443 # A natural way to remove the outer quotes is to do:
444 # docstring = docstring.strip(quote_char)
445 # but that breaks on """""x""" (which is '""x').
446 # So we actually need to remove the first character and the next two
447 # characters but only if they are the same as the first.
448 quote_len = 1 if docstring[1] != quote_char else 3
449 docstring = docstring[quote_len:-quote_len]
450 docstring_started_empty = not docstring
451 indent = " " * 4 * self.current_line.depth
453 if is_multiline_string(leaf):
454 docstring = fix_multiline_docstring(docstring, indent)
455 else:
456 docstring = docstring.strip()
458 has_trailing_backslash = False
459 if docstring:
460 # Add some padding if the docstring starts / ends with a quote mark.
461 if docstring[0] == quote_char:
462 docstring = " " + docstring
463 if docstring[-1] == quote_char:
464 docstring += " "
465 if docstring[-1] == "\\":
466 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
467 if backslash_count % 2:
468 # Odd number of tailing backslashes, add some padding to
469 # avoid escaping the closing string quote.
470 docstring += " "
471 has_trailing_backslash = True
472 elif not docstring_started_empty:
473 docstring = " "
475 # We could enforce triple quotes at this point.
476 quote = quote_char * quote_len
478 # It's invalid to put closing single-character quotes on a new line.
479 if quote_len == 3:
480 # We need to find the length of the last line of the docstring
481 # to find if we can add the closing quotes to the line without
482 # exceeding the maximum line length.
483 # If docstring is one line, we don't put the closing quotes on a
484 # separate line because it looks ugly (#3320).
485 lines = docstring.splitlines()
486 last_line_length = len(lines[-1]) if docstring else 0
488 # If adding closing quotes would cause the last line to exceed
489 # the maximum line length, and the closing quote is not
490 # prefixed by a newline then put a line break before
491 # the closing quotes
492 if (
493 len(lines) > 1
494 and last_line_length + quote_len > self.mode.line_length
495 and len(indent) + quote_len <= self.mode.line_length
496 and not has_trailing_backslash
497 ):
498 if leaf.value[-1 - quote_len] == "\n":
499 leaf.value = prefix + quote + docstring + quote
500 else:
501 leaf.value = prefix + quote + docstring + "\n" + indent + quote
502 else:
503 leaf.value = prefix + quote + docstring + quote
504 else:
505 leaf.value = prefix + quote + docstring + quote
507 if self.mode.string_normalization and leaf.type == token.STRING:
508 leaf.value = normalize_string_prefix(leaf.value)
509 leaf.value = normalize_string_quotes(leaf.value)
510 yield from self.visit_default(leaf)
512 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
513 normalize_numeric_literal(leaf)
514 yield from self.visit_default(leaf)
516 def visit_atom(self, node: Node) -> Iterator[Line]:
517 """Visit any atom"""
518 if len(node.children) == 3:
519 first = node.children[0]
520 last = node.children[-1]
521 if (first.type == token.LSQB and last.type == token.RSQB) or (
522 first.type == token.LBRACE and last.type == token.RBRACE
523 ):
524 # Lists or sets of one item
525 maybe_make_parens_invisible_in_atom(
526 node.children[1],
527 parent=node,
528 mode=self.mode,
529 features=self.features,
530 )
532 yield from self.visit_default(node)
534 def visit_fstring(self, node: Node) -> Iterator[Line]:
535 # currently we don't want to format and split f-strings at all.
536 string_leaf = fstring_to_string(node)
537 node.replace(string_leaf)
538 if "\\" in string_leaf.value and any(
539 "\\" in str(child)
540 for child in node.children
541 if child.type == syms.fstring_replacement_field
542 ):
543 # string normalization doesn't account for nested quotes,
544 # causing breakages. skip normalization when nested quotes exist
545 yield from self.visit_default(string_leaf)
546 return
547 yield from self.visit_STRING(string_leaf)
549 # TODO: Uncomment Implementation to format f-string children
550 # fstring_start = node.children[0]
551 # fstring_end = node.children[-1]
552 # assert isinstance(fstring_start, Leaf)
553 # assert isinstance(fstring_end, Leaf)
555 # quote_char = fstring_end.value[0]
556 # quote_idx = fstring_start.value.index(quote_char)
557 # prefix, quote = (
558 # fstring_start.value[:quote_idx],
559 # fstring_start.value[quote_idx:]
560 # )
562 # if not is_docstring(node, self.mode):
563 # prefix = normalize_string_prefix(prefix)
565 # assert quote == fstring_end.value
567 # is_raw_fstring = "r" in prefix or "R" in prefix
568 # middles = [
569 # leaf
570 # for leaf in node.leaves()
571 # if leaf.type == token.FSTRING_MIDDLE
572 # ]
574 # if self.mode.string_normalization:
575 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
577 # fstring_start.value = prefix + quote
578 # fstring_end.value = quote
580 # yield from self.visit_default(node)
582 def visit_comp_for(self, node: Node) -> Iterator[Line]:
583 if Preview.wrap_comprehension_in in self.mode:
584 normalize_invisible_parens(
585 node, parens_after={"in"}, mode=self.mode, features=self.features
586 )
587 yield from self.visit_default(node)
589 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
590 yield from self.visit_comp_for(node)
592 def __post_init__(self) -> None:
593 """You are in a twisty little maze of passages."""
594 self.current_line = Line(mode=self.mode)
596 v = self.visit_stmt
597 Ø: set[str] = set()
598 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
599 self.visit_if_stmt = partial(
600 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
601 )
602 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
603 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
604 self.visit_try_stmt = partial(
605 v, keywords={"try", "except", "else", "finally"}, parens=Ø
606 )
607 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
608 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
609 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
611 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
612 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
613 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
614 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
615 self.visit_async_funcdef = self.visit_async_stmt
616 self.visit_decorated = self.visit_decorators
618 # PEP 634
619 self.visit_match_stmt = self.visit_match_case
620 self.visit_case_block = self.visit_match_case
621 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
624def _hugging_power_ops_line_to_string(
625 line: Line,
626 features: Collection[Feature],
627 mode: Mode,
628) -> Optional[str]:
629 try:
630 return line_to_string(next(hug_power_op(line, features, mode)))
631 except CannotTransform:
632 return None
635def transform_line(
636 line: Line, mode: Mode, features: Collection[Feature] = ()
637) -> Iterator[Line]:
638 """Transform a `line`, potentially splitting it into many lines.
640 They should fit in the allotted `line_length` but might not be able to.
642 `features` are syntactical features that may be used in the output.
643 """
644 if line.is_comment:
645 yield line
646 return
648 line_str = line_to_string(line)
650 # We need the line string when power operators are hugging to determine if we should
651 # split the line. Default to line_str, if no power operator are present on the line.
652 line_str_hugging_power_ops = (
653 _hugging_power_ops_line_to_string(line, features, mode) or line_str
654 )
656 ll = mode.line_length
657 sn = mode.string_normalization
658 string_merge = StringMerger(ll, sn)
659 string_paren_strip = StringParenStripper(ll, sn)
660 string_split = StringSplitter(ll, sn)
661 string_paren_wrap = StringParenWrapper(ll, sn)
663 transformers: list[Transformer]
664 if (
665 not line.contains_uncollapsable_type_comments()
666 and not line.should_split_rhs
667 and not line.magic_trailing_comma
668 and (
669 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
670 or line.contains_unsplittable_type_ignore()
671 )
672 and not (line.inside_brackets and line.contains_standalone_comments())
673 and not line.contains_implicit_multiline_string_with_comments()
674 ):
675 # Only apply basic string preprocessing, since lines shouldn't be split here.
676 if Preview.string_processing in mode:
677 transformers = [string_merge, string_paren_strip]
678 else:
679 transformers = []
680 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
681 transformers = [left_hand_split]
682 else:
684 def _rhs(
685 self: object, line: Line, features: Collection[Feature], mode: Mode
686 ) -> Iterator[Line]:
687 """Wraps calls to `right_hand_split`.
689 The calls increasingly `omit` right-hand trailers (bracket pairs with
690 content), meaning the trailers get glued together to split on another
691 bracket pair instead.
692 """
693 for omit in generate_trailers_to_omit(line, mode.line_length):
694 lines = list(right_hand_split(line, mode, features, omit=omit))
695 # Note: this check is only able to figure out if the first line of the
696 # *current* transformation fits in the line length. This is true only
697 # for simple cases. All others require running more transforms via
698 # `transform_line()`. This check doesn't know if those would succeed.
699 if is_line_short_enough(lines[0], mode=mode):
700 yield from lines
701 return
703 # All splits failed, best effort split with no omits.
704 # This mostly happens to multiline strings that are by definition
705 # reported as not fitting a single line, as well as lines that contain
706 # trailing commas (those have to be exploded).
707 yield from right_hand_split(line, mode, features=features)
709 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
710 # __name__ attribute which is needed in `run_transformer` further down.
711 # Unfortunately a nested class breaks mypyc too. So a class must be created
712 # via type ... https://github.com/mypyc/mypyc/issues/884
713 rhs = type("rhs", (), {"__call__": _rhs})()
715 if Preview.string_processing in mode:
716 if line.inside_brackets:
717 transformers = [
718 string_merge,
719 string_paren_strip,
720 string_split,
721 delimiter_split,
722 standalone_comment_split,
723 string_paren_wrap,
724 rhs,
725 ]
726 else:
727 transformers = [
728 string_merge,
729 string_paren_strip,
730 string_split,
731 string_paren_wrap,
732 rhs,
733 ]
734 else:
735 if line.inside_brackets:
736 transformers = [delimiter_split, standalone_comment_split, rhs]
737 else:
738 transformers = [rhs]
739 # It's always safe to attempt hugging of power operations and pretty much every line
740 # could match.
741 transformers.append(hug_power_op)
743 for transform in transformers:
744 # We are accumulating lines in `result` because we might want to abort
745 # mission and return the original line in the end, or attempt a different
746 # split altogether.
747 try:
748 result = run_transformer(line, transform, mode, features, line_str=line_str)
749 except CannotTransform:
750 continue
751 else:
752 yield from result
753 break
755 else:
756 yield line
759def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
760 """If a funcdef has a magic trailing comma in the return type, then we should first
761 split the line with rhs to respect the comma.
762 """
763 return_type_leaves: list[Leaf] = []
764 in_return_type = False
766 for leaf in line.leaves:
767 if leaf.type == token.COLON:
768 in_return_type = False
769 if in_return_type:
770 return_type_leaves.append(leaf)
771 if leaf.type == token.RARROW:
772 in_return_type = True
774 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
775 # couple lines from it.
776 result = Line(mode=line.mode, depth=line.depth)
777 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
778 for leaf in return_type_leaves:
779 result.append(
780 leaf,
781 preformatted=True,
782 track_bracket=id(leaf) in leaves_to_track,
783 )
785 # we could also return true if the line is too long, and the return type is longer
786 # than the param list. Or if `should_split_rhs` returns True.
787 return result.magic_trailing_comma is not None
790class _BracketSplitComponent(Enum):
791 head = auto()
792 body = auto()
793 tail = auto()
796def left_hand_split(
797 line: Line, _features: Collection[Feature], mode: Mode
798) -> Iterator[Line]:
799 """Split line into many lines, starting with the first matching bracket pair.
801 Note: this usually looks weird, only use this for function definitions.
802 Prefer RHS otherwise. This is why this function is not symmetrical with
803 :func:`right_hand_split` which also handles optional parentheses.
804 """
805 for leaf_type in [token.LPAR, token.LSQB]:
806 tail_leaves: list[Leaf] = []
807 body_leaves: list[Leaf] = []
808 head_leaves: list[Leaf] = []
809 current_leaves = head_leaves
810 matching_bracket: Optional[Leaf] = None
811 depth = 0
812 for index, leaf in enumerate(line.leaves):
813 if index == 2 and leaf.type == token.LSQB:
814 # A [ at index 2 means this is a type param, so start
815 # tracking the depth
816 depth += 1
817 elif depth > 0:
818 if leaf.type == token.LSQB:
819 depth += 1
820 elif leaf.type == token.RSQB:
821 depth -= 1
822 if (
823 current_leaves is body_leaves
824 and leaf.type in CLOSING_BRACKETS
825 and leaf.opening_bracket is matching_bracket
826 and isinstance(matching_bracket, Leaf)
827 # If the code is still on LPAR and we are inside a type
828 # param, ignore the match since this is searching
829 # for the function arguments
830 and not (leaf_type == token.LPAR and depth > 0)
831 ):
832 ensure_visible(leaf)
833 ensure_visible(matching_bracket)
834 current_leaves = tail_leaves if body_leaves else head_leaves
835 current_leaves.append(leaf)
836 if current_leaves is head_leaves:
837 if leaf.type == leaf_type and (
838 Preview.fix_type_expansion_split not in mode
839 or not (leaf_type == token.LPAR and depth > 0)
840 ):
841 matching_bracket = leaf
842 current_leaves = body_leaves
843 if matching_bracket and tail_leaves:
844 break
845 if not matching_bracket or not tail_leaves:
846 raise CannotSplit("No brackets found")
848 head = bracket_split_build_line(
849 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
850 )
851 body = bracket_split_build_line(
852 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
853 )
854 tail = bracket_split_build_line(
855 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
856 )
857 bracket_split_succeeded_or_raise(head, body, tail)
858 for result in (head, body, tail):
859 if result:
860 yield result
863def right_hand_split(
864 line: Line,
865 mode: Mode,
866 features: Collection[Feature] = (),
867 omit: Collection[LeafID] = (),
868) -> Iterator[Line]:
869 """Split line into many lines, starting with the last matching bracket pair.
871 If the split was by optional parentheses, attempt splitting without them, too.
872 `omit` is a collection of closing bracket IDs that shouldn't be considered for
873 this split.
875 Note: running this function modifies `bracket_depth` on the leaves of `line`.
876 """
877 rhs_result = _first_right_hand_split(line, omit=omit)
878 yield from _maybe_split_omitting_optional_parens(
879 rhs_result, line, mode, features=features, omit=omit
880 )
883def _first_right_hand_split(
884 line: Line,
885 omit: Collection[LeafID] = (),
886) -> RHSResult:
887 """Split the line into head, body, tail starting with the last bracket pair.
889 Note: this function should not have side effects. It's relied upon by
890 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
891 splitting on the right side of an assignment statement.
892 """
893 tail_leaves: list[Leaf] = []
894 body_leaves: list[Leaf] = []
895 head_leaves: list[Leaf] = []
896 current_leaves = tail_leaves
897 opening_bracket: Optional[Leaf] = None
898 closing_bracket: Optional[Leaf] = None
899 for leaf in reversed(line.leaves):
900 if current_leaves is body_leaves:
901 if leaf is opening_bracket:
902 current_leaves = head_leaves if body_leaves else tail_leaves
903 current_leaves.append(leaf)
904 if current_leaves is tail_leaves:
905 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
906 opening_bracket = leaf.opening_bracket
907 closing_bracket = leaf
908 current_leaves = body_leaves
909 if not (opening_bracket and closing_bracket and head_leaves):
910 # If there is no opening or closing_bracket that means the split failed and
911 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
912 # the matching `opening_bracket` wasn't available on `line` anymore.
913 raise CannotSplit("No brackets found")
915 tail_leaves.reverse()
916 body_leaves.reverse()
917 head_leaves.reverse()
919 body: Optional[Line] = None
920 if (
921 Preview.hug_parens_with_braces_and_square_brackets in line.mode
922 and tail_leaves[0].value
923 and tail_leaves[0].opening_bracket is head_leaves[-1]
924 ):
925 inner_body_leaves = list(body_leaves)
926 hugged_opening_leaves: list[Leaf] = []
927 hugged_closing_leaves: list[Leaf] = []
928 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
929 unpacking_offset: int = 1 if is_unpacking else 0
930 while (
931 len(inner_body_leaves) >= 2 + unpacking_offset
932 and inner_body_leaves[-1].type in CLOSING_BRACKETS
933 and inner_body_leaves[-1].opening_bracket
934 is inner_body_leaves[unpacking_offset]
935 ):
936 if unpacking_offset:
937 hugged_opening_leaves.append(inner_body_leaves.pop(0))
938 unpacking_offset = 0
939 hugged_opening_leaves.append(inner_body_leaves.pop(0))
940 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
942 if hugged_opening_leaves and inner_body_leaves:
943 inner_body = bracket_split_build_line(
944 inner_body_leaves,
945 line,
946 hugged_opening_leaves[-1],
947 component=_BracketSplitComponent.body,
948 )
949 if (
950 line.mode.magic_trailing_comma
951 and inner_body_leaves[-1].type == token.COMMA
952 ):
953 should_hug = True
954 else:
955 line_length = line.mode.line_length - sum(
956 len(str(leaf))
957 for leaf in hugged_opening_leaves + hugged_closing_leaves
958 )
959 if is_line_short_enough(
960 inner_body, mode=replace(line.mode, line_length=line_length)
961 ):
962 # Do not hug if it fits on a single line.
963 should_hug = False
964 else:
965 should_hug = True
966 if should_hug:
967 body_leaves = inner_body_leaves
968 head_leaves.extend(hugged_opening_leaves)
969 tail_leaves = hugged_closing_leaves + tail_leaves
970 body = inner_body # No need to re-calculate the body again later.
972 head = bracket_split_build_line(
973 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
974 )
975 if body is None:
976 body = bracket_split_build_line(
977 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
978 )
979 tail = bracket_split_build_line(
980 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
981 )
982 bracket_split_succeeded_or_raise(head, body, tail)
983 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
986def _maybe_split_omitting_optional_parens(
987 rhs: RHSResult,
988 line: Line,
989 mode: Mode,
990 features: Collection[Feature] = (),
991 omit: Collection[LeafID] = (),
992) -> Iterator[Line]:
993 if (
994 Feature.FORCE_OPTIONAL_PARENTHESES not in features
995 # the opening bracket is an optional paren
996 and rhs.opening_bracket.type == token.LPAR
997 and not rhs.opening_bracket.value
998 # the closing bracket is an optional paren
999 and rhs.closing_bracket.type == token.RPAR
1000 and not rhs.closing_bracket.value
1001 # it's not an import (optional parens are the only thing we can split on
1002 # in this case; attempting a split without them is a waste of time)
1003 and not line.is_import
1004 # and we can actually remove the parens
1005 and can_omit_invisible_parens(rhs, mode.line_length)
1006 ):
1007 omit = {id(rhs.closing_bracket), *omit}
1008 try:
1009 # The RHSResult Omitting Optional Parens.
1010 rhs_oop = _first_right_hand_split(line, omit=omit)
1011 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1012 yield from _maybe_split_omitting_optional_parens(
1013 rhs_oop, line, mode, features=features, omit=omit
1014 )
1015 return
1017 except CannotSplit as e:
1018 # For chained assignments we want to use the previous successful split
1019 if line.is_chained_assignment:
1020 pass
1022 elif (
1023 not can_be_split(rhs.body)
1024 and not is_line_short_enough(rhs.body, mode=mode)
1025 and not (
1026 Preview.wrap_long_dict_values_in_parens
1027 and rhs.opening_bracket.parent
1028 and rhs.opening_bracket.parent.parent
1029 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1030 )
1031 ):
1032 raise CannotSplit(
1033 "Splitting failed, body is still too long and can't be split."
1034 ) from e
1036 elif (
1037 rhs.head.contains_multiline_strings()
1038 or rhs.tail.contains_multiline_strings()
1039 ):
1040 raise CannotSplit(
1041 "The current optional pair of parentheses is bound to fail to"
1042 " satisfy the splitting algorithm because the head or the tail"
1043 " contains multiline strings which by definition never fit one"
1044 " line."
1045 ) from e
1047 ensure_visible(rhs.opening_bracket)
1048 ensure_visible(rhs.closing_bracket)
1049 for result in (rhs.head, rhs.body, rhs.tail):
1050 if result:
1051 yield result
1054def _prefer_split_rhs_oop_over_rhs(
1055 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1056) -> bool:
1057 """
1058 Returns whether we should prefer the result from a split omitting optional parens
1059 (rhs_oop) over the original (rhs).
1060 """
1061 # contains unsplittable type ignore
1062 if (
1063 rhs_oop.head.contains_unsplittable_type_ignore()
1064 or rhs_oop.body.contains_unsplittable_type_ignore()
1065 or rhs_oop.tail.contains_unsplittable_type_ignore()
1066 ):
1067 return True
1069 # Retain optional parens around dictionary values
1070 if (
1071 Preview.wrap_long_dict_values_in_parens
1072 and rhs.opening_bracket.parent
1073 and rhs.opening_bracket.parent.parent
1074 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1075 and rhs.body.bracket_tracker.delimiters
1076 ):
1077 # Unless the split is inside the key
1078 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1080 # the split is right after `=`
1081 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1082 return True
1084 # the left side of assignment contains brackets
1085 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1086 return True
1088 # the left side of assignment is short enough (the -1 is for the ending optional
1089 # paren)
1090 if not is_line_short_enough(
1091 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1092 ):
1093 return True
1095 # the left side of assignment won't explode further because of magic trailing comma
1096 if rhs.head.magic_trailing_comma is not None:
1097 return True
1099 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1100 # the body
1101 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1102 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1103 token.EQUAL
1104 )
1105 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1106 return False
1108 has_closing_bracket_after_assign = False
1109 for leaf in reversed(rhs_oop.head.leaves):
1110 if leaf.type == token.EQUAL:
1111 break
1112 if leaf.type in CLOSING_BRACKETS:
1113 has_closing_bracket_after_assign = True
1114 break
1115 return (
1116 # contains matching brackets after the `=` (done by checking there is a
1117 # closing bracket)
1118 has_closing_bracket_after_assign
1119 or (
1120 # the split is actually from inside the optional parens (done by checking
1121 # the first line still contains the `=`)
1122 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1123 # the first line is short enough
1124 and is_line_short_enough(rhs_oop.head, mode=mode)
1125 )
1126 )
1129def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1130 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1132 Do nothing otherwise.
1134 A left- or right-hand split is based on a pair of brackets. Content before
1135 (and including) the opening bracket is left on one line, content inside the
1136 brackets is put on a separate line, and finally content starting with and
1137 following the closing bracket is put on a separate line.
1139 Those are called `head`, `body`, and `tail`, respectively. If the split
1140 produced the same line (all content in `head`) or ended up with an empty `body`
1141 and the `tail` is just the closing bracket, then it's considered failed.
1142 """
1143 tail_len = len(str(tail).strip())
1144 if not body:
1145 if tail_len == 0:
1146 raise CannotSplit("Splitting brackets produced the same line")
1148 elif tail_len < 3:
1149 raise CannotSplit(
1150 f"Splitting brackets on an empty body to save {tail_len} characters is"
1151 " not worth it"
1152 )
1155def _ensure_trailing_comma(
1156 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1157) -> bool:
1158 if not leaves:
1159 return False
1160 # Ensure a trailing comma for imports
1161 if original.is_import:
1162 return True
1163 # ...and standalone function arguments
1164 if not original.is_def:
1165 return False
1166 if opening_bracket.value != "(":
1167 return False
1168 # Don't add commas if we already have any commas
1169 if any(
1170 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1171 ):
1172 return False
1174 # Find a leaf with a parent (comments don't have parents)
1175 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1176 if leaf_with_parent is None:
1177 return True
1178 # Don't add commas inside parenthesized return annotations
1179 if get_annotation_type(leaf_with_parent) == "return":
1180 return False
1181 # Don't add commas inside PEP 604 unions
1182 if (
1183 leaf_with_parent.parent
1184 and leaf_with_parent.parent.next_sibling
1185 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1186 ):
1187 return False
1188 return True
1191def bracket_split_build_line(
1192 leaves: list[Leaf],
1193 original: Line,
1194 opening_bracket: Leaf,
1195 *,
1196 component: _BracketSplitComponent,
1197) -> Line:
1198 """Return a new line with given `leaves` and respective comments from `original`.
1200 If it's the head component, brackets will be tracked so trailing commas are
1201 respected.
1203 If it's the body component, the result line is one-indented inside brackets and as
1204 such has its first leaf's prefix normalized and a trailing comma added when
1205 expected.
1206 """
1207 result = Line(mode=original.mode, depth=original.depth)
1208 if component is _BracketSplitComponent.body:
1209 result.inside_brackets = True
1210 result.depth += 1
1211 if _ensure_trailing_comma(leaves, original, opening_bracket):
1212 for i in range(len(leaves) - 1, -1, -1):
1213 if leaves[i].type == STANDALONE_COMMENT:
1214 continue
1216 if leaves[i].type != token.COMMA:
1217 new_comma = Leaf(token.COMMA, ",")
1218 leaves.insert(i + 1, new_comma)
1219 break
1221 leaves_to_track: set[LeafID] = set()
1222 if component is _BracketSplitComponent.head:
1223 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1224 # Populate the line
1225 for leaf in leaves:
1226 result.append(
1227 leaf,
1228 preformatted=True,
1229 track_bracket=id(leaf) in leaves_to_track,
1230 )
1231 for comment_after in original.comments_after(leaf):
1232 result.append(comment_after, preformatted=True)
1233 if component is _BracketSplitComponent.body and should_split_line(
1234 result, opening_bracket
1235 ):
1236 result.should_split_rhs = True
1237 return result
1240def dont_increase_indentation(split_func: Transformer) -> Transformer:
1241 """Normalize prefix of the first leaf in every line returned by `split_func`.
1243 This is a decorator over relevant split functions.
1244 """
1246 @wraps(split_func)
1247 def split_wrapper(
1248 line: Line, features: Collection[Feature], mode: Mode
1249 ) -> Iterator[Line]:
1250 for split_line in split_func(line, features, mode):
1251 split_line.leaves[0].prefix = ""
1252 yield split_line
1254 return split_wrapper
1257def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1258 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1259 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1260 return leaf_idx
1261 return None
1264def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1265 if is_vararg(leaf, within={syms.typedargslist}):
1266 return Feature.TRAILING_COMMA_IN_DEF in features
1267 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1268 return Feature.TRAILING_COMMA_IN_CALL in features
1269 return True
1272def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1273 if (
1274 safe
1275 and delimiter_priority == COMMA_PRIORITY
1276 and line.leaves[-1].type != token.COMMA
1277 and line.leaves[-1].type != STANDALONE_COMMENT
1278 ):
1279 new_comma = Leaf(token.COMMA, ",")
1280 line.append(new_comma)
1281 return line
1284MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1287@dont_increase_indentation
1288def delimiter_split(
1289 line: Line, features: Collection[Feature], mode: Mode
1290) -> Iterator[Line]:
1291 """Split according to delimiters of the highest priority.
1293 If the appropriate Features are given, the split will add trailing commas
1294 also in function signatures and calls that contain `*` and `**`.
1295 """
1296 if len(line.leaves) == 0:
1297 raise CannotSplit("Line empty") from None
1298 last_leaf = line.leaves[-1]
1300 bt = line.bracket_tracker
1301 try:
1302 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1303 except ValueError:
1304 raise CannotSplit("No delimiters found") from None
1306 if (
1307 delimiter_priority == DOT_PRIORITY
1308 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1309 ):
1310 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1312 current_line = Line(
1313 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1314 )
1315 lowest_depth = sys.maxsize
1316 trailing_comma_safe = True
1318 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1319 """Append `leaf` to current line or to new line if appending impossible."""
1320 nonlocal current_line
1321 try:
1322 current_line.append_safe(leaf, preformatted=True)
1323 except ValueError:
1324 yield current_line
1326 current_line = Line(
1327 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1328 )
1329 current_line.append(leaf)
1331 def append_comments(leaf: Leaf) -> Iterator[Line]:
1332 for comment_after in line.comments_after(leaf):
1333 yield from append_to_line(comment_after)
1335 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1336 for leaf_idx, leaf in enumerate(line.leaves):
1337 yield from append_to_line(leaf)
1339 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1340 id(line.leaves[leaf_idx - 1])
1341 )
1342 if (
1343 previous_priority != delimiter_priority
1344 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1345 ):
1346 yield from append_comments(leaf)
1348 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1349 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1350 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1352 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1353 current_line = _safe_add_trailing_comma(
1354 trailing_comma_safe, delimiter_priority, current_line
1355 )
1357 leaf_priority = bt.delimiters.get(id(leaf))
1358 if leaf_priority == delimiter_priority:
1359 if (
1360 leaf_idx + 1 < len(line.leaves)
1361 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1362 ):
1363 yield from append_comments(line.leaves[leaf_idx + 1])
1365 yield current_line
1366 current_line = Line(
1367 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1368 )
1370 if current_line:
1371 current_line = _safe_add_trailing_comma(
1372 trailing_comma_safe, delimiter_priority, current_line
1373 )
1374 yield current_line
1377@dont_increase_indentation
1378def standalone_comment_split(
1379 line: Line, features: Collection[Feature], mode: Mode
1380) -> Iterator[Line]:
1381 """Split standalone comments from the rest of the line."""
1382 if not line.contains_standalone_comments():
1383 raise CannotSplit("Line does not have any standalone comments")
1385 current_line = Line(
1386 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1387 )
1389 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1390 """Append `leaf` to current line or to new line if appending impossible."""
1391 nonlocal current_line
1392 try:
1393 current_line.append_safe(leaf, preformatted=True)
1394 except ValueError:
1395 yield current_line
1397 current_line = Line(
1398 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1399 )
1400 current_line.append(leaf)
1402 for leaf in line.leaves:
1403 yield from append_to_line(leaf)
1405 for comment_after in line.comments_after(leaf):
1406 yield from append_to_line(comment_after)
1408 if current_line:
1409 yield current_line
1412def normalize_invisible_parens( # noqa: C901
1413 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1414) -> None:
1415 """Make existing optional parentheses invisible or create new ones.
1417 `parens_after` is a set of string leaf values immediately after which parens
1418 should be put.
1420 Standardizes on visible parentheses for single-element tuples, and keeps
1421 existing visible parentheses for other tuples and generator expressions.
1422 """
1423 for pc in list_comments(node.prefix, is_endmarker=False):
1424 if pc.value in FMT_OFF:
1425 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1426 return
1428 # The multiple context managers grammar has a different pattern, thus this is
1429 # separate from the for-loop below. This possibly wraps them in invisible parens,
1430 # and later will be removed in remove_with_parens when needed.
1431 if node.type == syms.with_stmt:
1432 _maybe_wrap_cms_in_parens(node, mode, features)
1434 check_lpar = False
1435 for index, child in enumerate(list(node.children)):
1436 # Fixes a bug where invisible parens are not properly stripped from
1437 # assignment statements that contain type annotations.
1438 if isinstance(child, Node) and child.type == syms.annassign:
1439 normalize_invisible_parens(
1440 child, parens_after=parens_after, mode=mode, features=features
1441 )
1443 # Fixes a bug where invisible parens are not properly wrapped around
1444 # case blocks.
1445 if isinstance(child, Node) and child.type == syms.case_block:
1446 normalize_invisible_parens(
1447 child, parens_after={"case"}, mode=mode, features=features
1448 )
1450 # Add parentheses around if guards in case blocks
1451 if isinstance(child, Node) and child.type == syms.guard:
1452 normalize_invisible_parens(
1453 child, parens_after={"if"}, mode=mode, features=features
1454 )
1456 # Add parentheses around long tuple unpacking in assignments.
1457 if (
1458 index == 0
1459 and isinstance(child, Node)
1460 and child.type == syms.testlist_star_expr
1461 ):
1462 check_lpar = True
1464 if check_lpar:
1465 if (
1466 child.type == syms.atom
1467 and node.type == syms.for_stmt
1468 and isinstance(child.prev_sibling, Leaf)
1469 and child.prev_sibling.type == token.NAME
1470 and child.prev_sibling.value == "for"
1471 ):
1472 if maybe_make_parens_invisible_in_atom(
1473 child,
1474 parent=node,
1475 mode=mode,
1476 features=features,
1477 remove_brackets_around_comma=True,
1478 ):
1479 wrap_in_parentheses(node, child, visible=False)
1480 elif isinstance(child, Node) and node.type == syms.with_stmt:
1481 remove_with_parens(child, node, mode=mode, features=features)
1482 elif child.type == syms.atom and not (
1483 "in" in parens_after
1484 and len(child.children) == 3
1485 and is_lpar_token(child.children[0])
1486 and is_rpar_token(child.children[-1])
1487 and child.children[1].type == syms.test
1488 ):
1489 if maybe_make_parens_invisible_in_atom(
1490 child, parent=node, mode=mode, features=features
1491 ):
1492 wrap_in_parentheses(node, child, visible=False)
1493 elif is_one_tuple(child):
1494 wrap_in_parentheses(node, child, visible=True)
1495 elif node.type == syms.import_from:
1496 _normalize_import_from(node, child, index)
1497 break
1498 elif (
1499 index == 1
1500 and child.type == token.STAR
1501 and node.type == syms.except_clause
1502 ):
1503 # In except* (PEP 654), the star is actually part of
1504 # of the keyword. So we need to skip the insertion of
1505 # invisible parentheses to work more precisely.
1506 continue
1508 elif (
1509 isinstance(child, Leaf)
1510 and child.next_sibling is not None
1511 and child.next_sibling.type == token.COLON
1512 and child.value == "case"
1513 ):
1514 # A special patch for "case case:" scenario, the second occurrence
1515 # of case will be not parsed as a Python keyword.
1516 break
1518 elif not is_multiline_string(child):
1519 wrap_in_parentheses(node, child, visible=False)
1521 comma_check = child.type == token.COMMA
1523 check_lpar = isinstance(child, Leaf) and (
1524 child.value in parens_after or comma_check
1525 )
1528def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1529 # "import from" nodes store parentheses directly as part of
1530 # the statement
1531 if is_lpar_token(child):
1532 assert is_rpar_token(parent.children[-1])
1533 # make parentheses invisible
1534 child.value = ""
1535 parent.children[-1].value = ""
1536 elif child.type != token.STAR:
1537 # insert invisible parentheses
1538 parent.insert_child(index, Leaf(token.LPAR, ""))
1539 parent.append_child(Leaf(token.RPAR, ""))
1542def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1543 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1544 if (
1545 node.children[1].type == syms.atom
1546 and node.children[1].children[0].type == token.LPAR
1547 ):
1548 if maybe_make_parens_invisible_in_atom(
1549 node.children[1],
1550 parent=node,
1551 mode=mode,
1552 features=features,
1553 remove_brackets_around_comma=True,
1554 ):
1555 wrap_in_parentheses(node, node.children[1], visible=False)
1557 # Since await is an expression we shouldn't remove
1558 # brackets in cases where this would change
1559 # the AST due to operator precedence.
1560 # Therefore we only aim to remove brackets around
1561 # power nodes that aren't also await expressions themselves.
1562 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1563 # N.B. We've still removed any redundant nested brackets though :)
1564 opening_bracket = cast(Leaf, node.children[1].children[0])
1565 closing_bracket = cast(Leaf, node.children[1].children[-1])
1566 bracket_contents = node.children[1].children[1]
1567 if isinstance(bracket_contents, Node) and (
1568 bracket_contents.type != syms.power
1569 or bracket_contents.children[0].type == token.AWAIT
1570 or any(
1571 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1572 for child in bracket_contents.children
1573 )
1574 ):
1575 ensure_visible(opening_bracket)
1576 ensure_visible(closing_bracket)
1579def _maybe_wrap_cms_in_parens(
1580 node: Node, mode: Mode, features: Collection[Feature]
1581) -> None:
1582 """When enabled and safe, wrap the multiple context managers in invisible parens.
1584 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1585 """
1586 if (
1587 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1588 or len(node.children) <= 2
1589 # If it's an atom, it's already wrapped in parens.
1590 or node.children[1].type == syms.atom
1591 ):
1592 return
1593 colon_index: Optional[int] = None
1594 for i in range(2, len(node.children)):
1595 if node.children[i].type == token.COLON:
1596 colon_index = i
1597 break
1598 if colon_index is not None:
1599 lpar = Leaf(token.LPAR, "")
1600 rpar = Leaf(token.RPAR, "")
1601 context_managers = node.children[1:colon_index]
1602 for child in context_managers:
1603 child.remove()
1604 # After wrapping, the with_stmt will look like this:
1605 # with_stmt
1606 # NAME 'with'
1607 # atom
1608 # LPAR ''
1609 # testlist_gexp
1610 # ... <-- context_managers
1611 # /testlist_gexp
1612 # RPAR ''
1613 # /atom
1614 # COLON ':'
1615 new_child = Node(
1616 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1617 )
1618 node.insert_child(1, new_child)
1621def remove_with_parens(
1622 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1623) -> None:
1624 """Recursively hide optional parens in `with` statements."""
1625 # Removing all unnecessary parentheses in with statements in one pass is a tad
1626 # complex as different variations of bracketed statements result in pretty
1627 # different parse trees:
1628 #
1629 # with (open("file")) as f: # this is an asexpr_test
1630 # ...
1631 #
1632 # with (open("file") as f): # this is an atom containing an
1633 # ... # asexpr_test
1634 #
1635 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1636 # ... # asexpr_test
1637 #
1638 # with (open("file") as f, open("file") as f): # an atom containing a
1639 # ... # testlist_gexp which then
1640 # # contains multiple asexpr_test(s)
1641 if node.type == syms.atom:
1642 if maybe_make_parens_invisible_in_atom(
1643 node,
1644 parent=parent,
1645 mode=mode,
1646 features=features,
1647 remove_brackets_around_comma=True,
1648 ):
1649 wrap_in_parentheses(parent, node, visible=False)
1650 if isinstance(node.children[1], Node):
1651 remove_with_parens(node.children[1], node, mode=mode, features=features)
1652 elif node.type == syms.testlist_gexp:
1653 for child in node.children:
1654 if isinstance(child, Node):
1655 remove_with_parens(child, node, mode=mode, features=features)
1656 elif node.type == syms.asexpr_test and not any(
1657 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1658 ):
1659 if maybe_make_parens_invisible_in_atom(
1660 node.children[0],
1661 parent=node,
1662 mode=mode,
1663 features=features,
1664 remove_brackets_around_comma=True,
1665 ):
1666 wrap_in_parentheses(node, node.children[0], visible=False)
1669def maybe_make_parens_invisible_in_atom(
1670 node: LN,
1671 parent: LN,
1672 mode: Mode,
1673 features: Collection[Feature],
1674 remove_brackets_around_comma: bool = False,
1675) -> bool:
1676 """If it's safe, make the parens in the atom `node` invisible, recursively.
1677 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1678 as they are redundant.
1680 Returns whether the node should itself be wrapped in invisible parentheses.
1681 """
1682 if (
1683 node.type not in (syms.atom, syms.expr)
1684 or is_empty_tuple(node)
1685 or is_one_tuple(node)
1686 or (is_tuple(node) and parent.type == syms.asexpr_test)
1687 or (
1688 is_tuple(node)
1689 and parent.type == syms.with_stmt
1690 and has_sibling_with_type(node, token.COMMA)
1691 )
1692 or (is_yield(node) and parent.type != syms.expr_stmt)
1693 or (
1694 # This condition tries to prevent removing non-optional brackets
1695 # around a tuple, however, can be a bit overzealous so we provide
1696 # and option to skip this check for `for` and `with` statements.
1697 not remove_brackets_around_comma
1698 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1699 # Skip this check in Preview mode in order to
1700 # Remove parentheses around multiple exception types in except and
1701 # except* without as. See PEP 758 for details.
1702 and not (
1703 Preview.remove_parens_around_except_types in mode
1704 and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1705 # is a tuple
1706 and is_tuple(node)
1707 # has a parent node
1708 and node.parent is not None
1709 # parent is an except clause
1710 and node.parent.type == syms.except_clause
1711 # is not immediately followed by as clause
1712 and not (
1713 node.next_sibling is not None
1714 and is_name_token(node.next_sibling)
1715 and node.next_sibling.value == "as"
1716 )
1717 )
1718 )
1719 or is_tuple_containing_walrus(node)
1720 or is_tuple_containing_star(node)
1721 or is_generator(node)
1722 ):
1723 return False
1725 if is_walrus_assignment(node):
1726 if parent.type in [
1727 syms.annassign,
1728 syms.expr_stmt,
1729 syms.assert_stmt,
1730 syms.return_stmt,
1731 syms.except_clause,
1732 syms.funcdef,
1733 syms.with_stmt,
1734 syms.testlist_gexp,
1735 syms.tname,
1736 # these ones aren't useful to end users, but they do please fuzzers
1737 syms.for_stmt,
1738 syms.del_stmt,
1739 syms.for_stmt,
1740 ]:
1741 return False
1743 first = node.children[0]
1744 last = node.children[-1]
1745 if is_lpar_token(first) and is_rpar_token(last):
1746 middle = node.children[1]
1747 # make parentheses invisible
1748 if (
1749 # If the prefix of `middle` includes a type comment with
1750 # ignore annotation, then we do not remove the parentheses
1751 not is_type_ignore_comment_string(middle.prefix.strip())
1752 ):
1753 first.value = ""
1754 last.value = ""
1755 maybe_make_parens_invisible_in_atom(
1756 middle,
1757 parent=parent,
1758 mode=mode,
1759 features=features,
1760 remove_brackets_around_comma=remove_brackets_around_comma,
1761 )
1763 if is_atom_with_invisible_parens(middle):
1764 # Strip the invisible parens from `middle` by replacing
1765 # it with the child in-between the invisible parens
1766 middle.replace(middle.children[1])
1768 if middle.children[0].prefix.strip():
1769 # Preserve comments before first paren
1770 middle.children[1].prefix = (
1771 middle.children[0].prefix + middle.children[1].prefix
1772 )
1774 if middle.children[-1].prefix.strip():
1775 # Preserve comments before last paren
1776 last.prefix = middle.children[-1].prefix + last.prefix
1778 return False
1780 return True
1783def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1784 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1786 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1787 return False
1789 # We're essentially checking if the body is delimited by commas and there's more
1790 # than one of them (we're excluding the trailing comma and if the delimiter priority
1791 # is still commas, that means there's more).
1792 exclude = set()
1793 trailing_comma = False
1794 try:
1795 last_leaf = line.leaves[-1]
1796 if last_leaf.type == token.COMMA:
1797 trailing_comma = True
1798 exclude.add(id(last_leaf))
1799 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1800 except (IndexError, ValueError):
1801 return False
1803 return max_priority == COMMA_PRIORITY and (
1804 (line.mode.magic_trailing_comma and trailing_comma)
1805 # always explode imports
1806 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1807 )
1810def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1811 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1813 Brackets can be omitted if the entire trailer up to and including
1814 a preceding closing bracket fits in one line.
1816 Yielded sets are cumulative (contain results of previous yields, too). First
1817 set is empty, unless the line should explode, in which case bracket pairs until
1818 the one that needs to explode are omitted.
1819 """
1821 omit: set[LeafID] = set()
1822 if not line.magic_trailing_comma:
1823 yield omit
1825 length = 4 * line.depth
1826 opening_bracket: Optional[Leaf] = None
1827 closing_bracket: Optional[Leaf] = None
1828 inner_brackets: set[LeafID] = set()
1829 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1830 length += leaf_length
1831 if length > line_length:
1832 break
1834 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1835 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1836 break
1838 if opening_bracket:
1839 if leaf is opening_bracket:
1840 opening_bracket = None
1841 elif leaf.type in CLOSING_BRACKETS:
1842 prev = line.leaves[index - 1] if index > 0 else None
1843 if (
1844 prev
1845 and prev.type == token.COMMA
1846 and leaf.opening_bracket is not None
1847 and not is_one_sequence_between(
1848 leaf.opening_bracket, leaf, line.leaves
1849 )
1850 ):
1851 # Never omit bracket pairs with trailing commas.
1852 # We need to explode on those.
1853 break
1855 inner_brackets.add(id(leaf))
1856 elif leaf.type in CLOSING_BRACKETS:
1857 prev = line.leaves[index - 1] if index > 0 else None
1858 if prev and prev.type in OPENING_BRACKETS:
1859 # Empty brackets would fail a split so treat them as "inner"
1860 # brackets (e.g. only add them to the `omit` set if another
1861 # pair of brackets was good enough.
1862 inner_brackets.add(id(leaf))
1863 continue
1865 if closing_bracket:
1866 omit.add(id(closing_bracket))
1867 omit.update(inner_brackets)
1868 inner_brackets.clear()
1869 yield omit
1871 if (
1872 prev
1873 and prev.type == token.COMMA
1874 and leaf.opening_bracket is not None
1875 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1876 ):
1877 # Never omit bracket pairs with trailing commas.
1878 # We need to explode on those.
1879 break
1881 if leaf.value:
1882 opening_bracket = leaf.opening_bracket
1883 closing_bracket = leaf
1886def run_transformer(
1887 line: Line,
1888 transform: Transformer,
1889 mode: Mode,
1890 features: Collection[Feature],
1891 *,
1892 line_str: str = "",
1893) -> list[Line]:
1894 if not line_str:
1895 line_str = line_to_string(line)
1896 result: list[Line] = []
1897 for transformed_line in transform(line, features, mode):
1898 if str(transformed_line).strip("\n") == line_str:
1899 raise CannotTransform("Line transformer returned an unchanged result")
1901 result.extend(transform_line(transformed_line, mode=mode, features=features))
1903 features_set = set(features)
1904 if (
1905 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1906 or transform.__class__.__name__ != "rhs"
1907 or not line.bracket_tracker.invisible
1908 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1909 or line.contains_multiline_strings()
1910 or result[0].contains_uncollapsable_type_comments()
1911 or result[0].contains_unsplittable_type_ignore()
1912 or is_line_short_enough(result[0], mode=mode)
1913 # If any leaves have no parents (which _can_ occur since
1914 # `transform(line)` potentially destroys the line's underlying node
1915 # structure), then we can't proceed. Doing so would cause the below
1916 # call to `append_leaves()` to fail.
1917 or any(leaf.parent is None for leaf in line.leaves)
1918 ):
1919 return result
1921 line_copy = line.clone()
1922 append_leaves(line_copy, line, line.leaves)
1923 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1924 second_opinion = run_transformer(
1925 line_copy, transform, mode, features_fop, line_str=line_str
1926 )
1927 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1928 result = second_opinion
1929 return result