Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import (
21 FMT_OFF,
22 FMT_ON,
23 contains_fmt_directive,
24 generate_comments,
25 list_comments,
26)
27from black.lines import (
28 Line,
29 RHSResult,
30 append_leaves,
31 can_be_split,
32 can_omit_invisible_parens,
33 is_line_short_enough,
34 line_to_string,
35)
36from black.mode import Feature, Mode, Preview
37from black.nodes import (
38 ASSIGNMENTS,
39 BRACKETS,
40 CLOSING_BRACKETS,
41 OPENING_BRACKETS,
42 STANDALONE_COMMENT,
43 STATEMENT,
44 WHITESPACE,
45 Visitor,
46 ensure_visible,
47 fstring_tstring_to_string,
48 get_annotation_type,
49 has_sibling_with_type,
50 is_arith_like,
51 is_async_stmt_or_funcdef,
52 is_atom_with_invisible_parens,
53 is_docstring,
54 is_empty_tuple,
55 is_generator,
56 is_lpar_token,
57 is_multiline_string,
58 is_name_token,
59 is_one_sequence_between,
60 is_one_tuple,
61 is_parent_function_or_class,
62 is_part_of_annotation,
63 is_rpar_token,
64 is_stub_body,
65 is_stub_suite,
66 is_tuple,
67 is_tuple_containing_star,
68 is_tuple_containing_walrus,
69 is_type_ignore_comment_string,
70 is_vararg,
71 is_walrus_assignment,
72 is_yield,
73 syms,
74 wrap_in_parentheses,
75)
76from black.numerics import normalize_numeric_literal
77from black.strings import (
78 fix_multiline_docstring,
79 get_string_prefix,
80 normalize_string_prefix,
81 normalize_string_quotes,
82 normalize_unicode_escape_sequences,
83)
84from black.trans import (
85 CannotTransform,
86 StringMerger,
87 StringParenStripper,
88 StringParenWrapper,
89 StringSplitter,
90 Transformer,
91 hug_power_op,
92)
93from blib2to3.pgen2 import token
94from blib2to3.pytree import Leaf, Node
96# types
97LeafID = int
98LN = Union[Leaf, Node]
101class CannotSplit(CannotTransform):
102 """A readable split that fits the allotted line length is impossible."""
105# This isn't a dataclass because @dataclass + Generic breaks mypyc.
106# See also https://github.com/mypyc/mypyc/issues/827.
107class LineGenerator(Visitor[Line]):
108 """Generates reformatted Line objects. Empty lines are not emitted.
110 Note: destroys the tree it's visiting by mutating prefixes of its leaves
111 in ways that will no longer stringify to valid Python code on the tree.
112 """
114 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
115 self.mode = mode
116 self.features = features
117 self.current_line: Line
118 self.__post_init__()
120 def line(self, indent: int = 0) -> Iterator[Line]:
121 """Generate a line.
123 If the line is empty, only emit if it makes sense.
124 If the line is too long, split it first and then generate.
126 If any lines were generated, set up a new current_line.
127 """
128 if not self.current_line:
129 self.current_line.depth += indent
130 return # Line is empty, don't emit. Creating a new one unnecessary.
132 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
133 self.current_line.leaves[0]
134 ):
135 # Special case for async def/for/with statements. `visit_async_stmt`
136 # adds an `ASYNC` leaf then visits the child def/for/with statement
137 # nodes. Line yields from those nodes shouldn't treat the former
138 # `ASYNC` leaf as a complete line.
139 return
141 complete_line = self.current_line
142 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
143 yield complete_line
145 def visit_default(self, node: LN) -> Iterator[Line]:
146 """Default `visit_*()` implementation. Recurses to children of `node`."""
147 if isinstance(node, Leaf):
148 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
149 for comment in generate_comments(node, mode=self.mode):
150 if any_open_brackets:
151 # any comment within brackets is subject to splitting
152 self.current_line.append(comment)
153 elif comment.type == token.COMMENT:
154 # regular trailing comment
155 self.current_line.append(comment)
156 yield from self.line()
158 else:
159 # regular standalone comment
160 yield from self.line()
162 self.current_line.append(comment)
163 yield from self.line()
165 if any_open_brackets:
166 node.prefix = ""
167 if node.type not in WHITESPACE:
168 self.current_line.append(node)
169 yield from super().visit_default(node)
171 def visit_test(self, node: Node) -> Iterator[Line]:
172 """Visit an `x if y else z` test"""
174 already_parenthesized = (
175 node.prev_sibling and node.prev_sibling.type == token.LPAR
176 )
178 if not already_parenthesized:
179 # Similar to logic in wrap_in_parentheses
180 lpar = Leaf(token.LPAR, "")
181 rpar = Leaf(token.RPAR, "")
182 prefix = node.prefix
183 node.prefix = ""
184 lpar.prefix = prefix
185 node.insert_child(0, lpar)
186 node.append_child(rpar)
188 yield from self.visit_default(node)
190 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
191 """Increase indentation level, maybe yield a line."""
192 # In blib2to3 INDENT never holds comments.
193 yield from self.line(+1)
194 yield from self.visit_default(node)
196 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
197 """Decrease indentation level, maybe yield a line."""
198 # The current line might still wait for trailing comments. At DEDENT time
199 # there won't be any (they would be prefixes on the preceding NEWLINE).
200 # Emit the line then.
201 yield from self.line()
203 # While DEDENT has no value, its prefix may contain standalone comments
204 # that belong to the current indentation level. Get 'em.
205 yield from self.visit_default(node)
207 # Finally, emit the dedent.
208 yield from self.line(-1)
210 def visit_stmt(
211 self, node: Node, keywords: set[str], parens: set[str]
212 ) -> Iterator[Line]:
213 """Visit a statement.
215 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
216 `def`, `with`, `class`, `assert`, and assignments.
218 The relevant Python language `keywords` for a given statement will be
219 NAME leaves within it. This methods puts those on a separate line.
221 `parens` holds a set of string leaf values immediately after which
222 invisible parens should be put.
223 """
224 normalize_invisible_parens(
225 node, parens_after=parens, mode=self.mode, features=self.features
226 )
227 for child in node.children:
228 if is_name_token(child) and child.value in keywords:
229 yield from self.line()
231 yield from self.visit(child)
233 def visit_typeparams(self, node: Node) -> Iterator[Line]:
234 yield from self.visit_default(node)
235 node.children[0].prefix = ""
237 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
238 yield from self.visit_default(node)
239 node.children[1].prefix = ""
241 def visit_paramspec(self, node: Node) -> Iterator[Line]:
242 yield from self.visit_default(node)
243 node.children[1].prefix = ""
245 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
246 if Preview.wrap_long_dict_values_in_parens in self.mode:
247 for i, child in enumerate(node.children):
248 if i == 0:
249 continue
250 if node.children[i - 1].type == token.COLON:
251 if (
252 child.type == syms.atom
253 and child.children[0].type in OPENING_BRACKETS
254 and not is_walrus_assignment(child)
255 ):
256 maybe_make_parens_invisible_in_atom(
257 child,
258 parent=node,
259 mode=self.mode,
260 features=self.features,
261 remove_brackets_around_comma=False,
262 )
263 else:
264 wrap_in_parentheses(node, child, visible=False)
265 yield from self.visit_default(node)
267 def visit_funcdef(self, node: Node) -> Iterator[Line]:
268 """Visit function definition."""
269 yield from self.line()
271 # Remove redundant brackets around return type annotation.
272 is_return_annotation = False
273 for child in node.children:
274 if child.type == token.RARROW:
275 is_return_annotation = True
276 elif is_return_annotation:
277 if child.type == syms.atom and child.children[0].type == token.LPAR:
278 if maybe_make_parens_invisible_in_atom(
279 child,
280 parent=node,
281 mode=self.mode,
282 features=self.features,
283 remove_brackets_around_comma=False,
284 ):
285 wrap_in_parentheses(node, child, visible=False)
286 else:
287 wrap_in_parentheses(node, child, visible=False)
288 is_return_annotation = False
290 for child in node.children:
291 yield from self.visit(child)
293 def visit_match_case(self, node: Node) -> Iterator[Line]:
294 """Visit either a match or case statement."""
295 normalize_invisible_parens(
296 node, parens_after=set(), mode=self.mode, features=self.features
297 )
299 yield from self.line()
300 for child in node.children:
301 yield from self.visit(child)
303 def visit_suite(self, node: Node) -> Iterator[Line]:
304 """Visit a suite."""
305 if is_stub_suite(node):
306 yield from self.visit(node.children[2])
307 else:
308 yield from self.visit_default(node)
310 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
311 """Visit a statement without nested statements."""
312 prev_type: int | None = None
313 for child in node.children:
314 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
315 wrap_in_parentheses(node, child, visible=False)
316 prev_type = child.type
318 if node.parent and node.parent.type in STATEMENT:
319 if is_parent_function_or_class(node) and is_stub_body(node):
320 yield from self.visit_default(node)
321 else:
322 yield from self.line(+1)
323 yield from self.visit_default(node)
324 yield from self.line(-1)
326 else:
327 if node.parent and is_stub_suite(node.parent):
328 node.prefix = ""
329 yield from self.visit_default(node)
330 return
331 yield from self.line()
332 yield from self.visit_default(node)
334 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
335 """Visit `async def`, `async for`, `async with`."""
336 yield from self.line()
338 children = iter(node.children)
339 for child in children:
340 yield from self.visit(child)
342 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
343 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
344 # line.
345 break
347 internal_stmt = next(children)
348 yield from self.visit(internal_stmt)
350 def visit_decorators(self, node: Node) -> Iterator[Line]:
351 """Visit decorators."""
352 for child in node.children:
353 yield from self.line()
354 yield from self.visit(child)
356 def visit_power(self, node: Node) -> Iterator[Line]:
357 for idx, leaf in enumerate(node.children[:-1]):
358 next_leaf = node.children[idx + 1]
360 if not isinstance(leaf, Leaf):
361 continue
363 value = leaf.value.lower()
364 if (
365 leaf.type == token.NUMBER
366 and next_leaf.type == syms.trailer
367 # Ensure that we are in an attribute trailer
368 and next_leaf.children[0].type == token.DOT
369 # It shouldn't wrap hexadecimal, binary and octal literals
370 and not value.startswith(("0x", "0b", "0o"))
371 # It shouldn't wrap complex literals
372 and "j" not in value
373 ):
374 wrap_in_parentheses(node, leaf)
376 remove_await_parens(node, mode=self.mode, features=self.features)
378 yield from self.visit_default(node)
380 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
381 """Remove a semicolon and put the other statement on a separate line."""
382 yield from self.line()
384 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
385 """End of file. Process outstanding comments and end with a newline."""
386 yield from self.visit_default(leaf)
387 yield from self.line()
389 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
390 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
391 if not any_open_brackets:
392 yield from self.line()
393 # STANDALONE_COMMENT nodes created by our special handling in
394 # normalize_fmt_off for comment-only blocks have fmt:off as the first
395 # line and fmt:on as the last line (each directive on its own line,
396 # not embedded in other text). These should be appended directly
397 # without calling visit_default, which would process their prefix and
398 # lose indentation. Normal STANDALONE_COMMENT nodes go through
399 # visit_default.
400 value = leaf.value
401 lines = value.splitlines()
402 is_fmt_off_block = (
403 len(lines) >= 2
404 and contains_fmt_directive(lines[0], FMT_OFF)
405 and contains_fmt_directive(lines[-1], FMT_ON)
406 )
407 if is_fmt_off_block:
408 # This is a fmt:off/on block from normalize_fmt_off - we still need
409 # to process any prefix comments (like markdown comments) but append
410 # the fmt block itself directly to preserve its formatting
412 # Only process prefix comments if there actually is a prefix with comments
413 if leaf.prefix and any(
414 line.strip().startswith("#")
415 and not contains_fmt_directive(line.strip())
416 for line in leaf.prefix.split("\n")
417 ):
418 for comment in generate_comments(leaf, mode=self.mode):
419 yield from self.line()
420 self.current_line.append(comment)
421 yield from self.line()
422 # Clear the prefix since we've processed it as comments above
423 leaf.prefix = ""
425 self.current_line.append(leaf)
426 if not any_open_brackets:
427 yield from self.line()
428 else:
429 # Normal standalone comment - process through visit_default
430 yield from self.visit_default(leaf)
432 def visit_factor(self, node: Node) -> Iterator[Line]:
433 """Force parentheses between a unary op and a binary power:
435 -2 ** 8 -> -(2 ** 8)
436 """
437 _operator, operand = node.children
438 if (
439 operand.type == syms.power
440 and len(operand.children) == 3
441 and operand.children[1].type == token.DOUBLESTAR
442 ):
443 lpar = Leaf(token.LPAR, "(")
444 rpar = Leaf(token.RPAR, ")")
445 index = operand.remove() or 0
446 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
447 yield from self.visit_default(node)
449 def visit_tname(self, node: Node) -> Iterator[Line]:
450 """
451 Add potential parentheses around types in function parameter lists to be made
452 into real parentheses in case the type hint is too long to fit on a line
453 Examples:
454 def foo(a: int, b: float = 7): ...
456 ->
458 def foo(a: (int), b: (float) = 7): ...
459 """
460 if len(node.children) == 3 and maybe_make_parens_invisible_in_atom(
461 node.children[2], parent=node, mode=self.mode, features=self.features
462 ):
463 wrap_in_parentheses(node, node.children[2], visible=False)
465 yield from self.visit_default(node)
467 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
468 normalize_unicode_escape_sequences(leaf)
470 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
471 # We're ignoring docstrings with backslash newline escapes because changing
472 # indentation of those changes the AST representation of the code.
473 if self.mode.string_normalization:
474 docstring = normalize_string_prefix(leaf.value)
475 # We handle string normalization at the end of this method, but since
476 # what we do right now acts differently depending on quote style (ex.
477 # see padding logic below), there's a possibility for unstable
478 # formatting. To avoid a situation where this function formats a
479 # docstring differently on the second pass, normalize it early.
480 docstring = normalize_string_quotes(docstring)
481 else:
482 docstring = leaf.value
483 prefix = get_string_prefix(docstring)
484 docstring = docstring[len(prefix) :] # Remove the prefix
485 quote_char = docstring[0]
486 # A natural way to remove the outer quotes is to do:
487 # docstring = docstring.strip(quote_char)
488 # but that breaks on """""x""" (which is '""x').
489 # So we actually need to remove the first character and the next two
490 # characters but only if they are the same as the first.
491 quote_len = 1 if docstring[1] != quote_char else 3
492 docstring = docstring[quote_len:-quote_len]
493 docstring_started_empty = not docstring
494 indent = " " * 4 * self.current_line.depth
496 if is_multiline_string(leaf):
497 docstring = fix_multiline_docstring(docstring, indent)
498 else:
499 docstring = docstring.strip()
501 has_trailing_backslash = False
502 if docstring:
503 # Add some padding if the docstring starts / ends with a quote mark.
504 if docstring[0] == quote_char:
505 docstring = " " + docstring
506 if docstring[-1] == quote_char:
507 docstring += " "
508 if docstring[-1] == "\\":
509 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
510 if backslash_count % 2:
511 # Odd number of tailing backslashes, add some padding to
512 # avoid escaping the closing string quote.
513 docstring += " "
514 has_trailing_backslash = True
515 elif not docstring_started_empty:
516 docstring = " "
518 # We could enforce triple quotes at this point.
519 quote = quote_char * quote_len
521 # It's invalid to put closing single-character quotes on a new line.
522 if quote_len == 3:
523 # We need to find the length of the last line of the docstring
524 # to find if we can add the closing quotes to the line without
525 # exceeding the maximum line length.
526 # If docstring is one line, we don't put the closing quotes on a
527 # separate line because it looks ugly (#3320).
528 lines = docstring.splitlines()
529 last_line_length = len(lines[-1]) if docstring else 0
531 # If adding closing quotes would cause the last line to exceed
532 # the maximum line length, and the closing quote is not
533 # prefixed by a newline then put a line break before
534 # the closing quotes
535 if (
536 len(lines) > 1
537 and last_line_length + quote_len > self.mode.line_length
538 and len(indent) + quote_len <= self.mode.line_length
539 and not has_trailing_backslash
540 ):
541 if leaf.value[-1 - quote_len] == "\n":
542 leaf.value = prefix + quote + docstring + quote
543 else:
544 leaf.value = prefix + quote + docstring + "\n" + indent + quote
545 else:
546 leaf.value = prefix + quote + docstring + quote
547 else:
548 leaf.value = prefix + quote + docstring + quote
550 if self.mode.string_normalization and leaf.type == token.STRING:
551 leaf.value = normalize_string_prefix(leaf.value)
552 leaf.value = normalize_string_quotes(leaf.value)
553 yield from self.visit_default(leaf)
555 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
556 normalize_numeric_literal(leaf)
557 yield from self.visit_default(leaf)
559 def visit_atom(self, node: Node) -> Iterator[Line]:
560 """Visit any atom"""
561 if len(node.children) == 3:
562 first = node.children[0]
563 last = node.children[-1]
564 if (first.type == token.LSQB and last.type == token.RSQB) or (
565 first.type == token.LBRACE and last.type == token.RBRACE
566 ):
567 # Lists or sets of one item
568 maybe_make_parens_invisible_in_atom(
569 node.children[1],
570 parent=node,
571 mode=self.mode,
572 features=self.features,
573 )
575 yield from self.visit_default(node)
577 def visit_fstring(self, node: Node) -> Iterator[Line]:
578 # currently we don't want to format and split f-strings at all.
579 string_leaf = fstring_tstring_to_string(node)
580 node.replace(string_leaf)
581 if "\\" in string_leaf.value and any(
582 "\\" in str(child)
583 for child in node.children
584 if child.type == syms.fstring_replacement_field
585 ):
586 # string normalization doesn't account for nested quotes,
587 # causing breakages. skip normalization when nested quotes exist
588 yield from self.visit_default(string_leaf)
589 return
590 yield from self.visit_STRING(string_leaf)
592 def visit_tstring(self, node: Node) -> Iterator[Line]:
593 # currently we don't want to format and split t-strings at all.
594 string_leaf = fstring_tstring_to_string(node)
595 node.replace(string_leaf)
596 if "\\" in string_leaf.value and any(
597 "\\" in str(child)
598 for child in node.children
599 if child.type == syms.fstring_replacement_field
600 ):
601 # string normalization doesn't account for nested quotes,
602 # causing breakages. skip normalization when nested quotes exist
603 yield from self.visit_default(string_leaf)
604 return
605 yield from self.visit_STRING(string_leaf)
607 # TODO: Uncomment Implementation to format f-string children
608 # fstring_start = node.children[0]
609 # fstring_end = node.children[-1]
610 # assert isinstance(fstring_start, Leaf)
611 # assert isinstance(fstring_end, Leaf)
613 # quote_char = fstring_end.value[0]
614 # quote_idx = fstring_start.value.index(quote_char)
615 # prefix, quote = (
616 # fstring_start.value[:quote_idx],
617 # fstring_start.value[quote_idx:]
618 # )
620 # if not is_docstring(node, self.mode):
621 # prefix = normalize_string_prefix(prefix)
623 # assert quote == fstring_end.value
625 # is_raw_fstring = "r" in prefix or "R" in prefix
626 # middles = [
627 # leaf
628 # for leaf in node.leaves()
629 # if leaf.type == token.FSTRING_MIDDLE
630 # ]
632 # if self.mode.string_normalization:
633 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
635 # fstring_start.value = prefix + quote
636 # fstring_end.value = quote
638 # yield from self.visit_default(node)
640 def visit_comp_for(self, node: Node) -> Iterator[Line]:
641 if Preview.wrap_comprehension_in in self.mode:
642 normalize_invisible_parens(
643 node, parens_after={"in"}, mode=self.mode, features=self.features
644 )
645 yield from self.visit_default(node)
647 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
648 yield from self.visit_comp_for(node)
650 def __post_init__(self) -> None:
651 """You are in a twisty little maze of passages."""
652 self.current_line = Line(mode=self.mode)
654 v = self.visit_stmt
655 Ø: set[str] = set()
656 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
657 self.visit_if_stmt = partial(
658 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
659 )
660 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
661 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
662 self.visit_try_stmt = partial(
663 v, keywords={"try", "except", "else", "finally"}, parens=Ø
664 )
665 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
666 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
667 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
669 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
670 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
671 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
672 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
673 self.visit_async_funcdef = self.visit_async_stmt
674 self.visit_decorated = self.visit_decorators
676 # PEP 634
677 self.visit_match_stmt = self.visit_match_case
678 self.visit_case_block = self.visit_match_case
679 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
682def _hugging_power_ops_line_to_string(
683 line: Line,
684 features: Collection[Feature],
685 mode: Mode,
686) -> str | None:
687 try:
688 return line_to_string(next(hug_power_op(line, features, mode)))
689 except CannotTransform:
690 return None
693def transform_line(
694 line: Line, mode: Mode, features: Collection[Feature] = ()
695) -> Iterator[Line]:
696 """Transform a `line`, potentially splitting it into many lines.
698 They should fit in the allotted `line_length` but might not be able to.
700 `features` are syntactical features that may be used in the output.
701 """
702 if line.is_comment:
703 yield line
704 return
706 line_str = line_to_string(line)
708 # We need the line string when power operators are hugging to determine if we should
709 # split the line. Default to line_str, if no power operator are present on the line.
710 line_str_hugging_power_ops = (
711 _hugging_power_ops_line_to_string(line, features, mode) or line_str
712 )
714 ll = mode.line_length
715 sn = mode.string_normalization
716 string_merge = StringMerger(ll, sn)
717 string_paren_strip = StringParenStripper(ll, sn)
718 string_split = StringSplitter(ll, sn)
719 string_paren_wrap = StringParenWrapper(ll, sn)
721 transformers: list[Transformer]
722 if (
723 not line.contains_uncollapsable_type_comments()
724 and not line.should_split_rhs
725 and not line.magic_trailing_comma
726 and (
727 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
728 or line.contains_unsplittable_type_ignore()
729 )
730 and not (line.inside_brackets and line.contains_standalone_comments())
731 and not line.contains_implicit_multiline_string_with_comments()
732 ):
733 # Only apply basic string preprocessing, since lines shouldn't be split here.
734 if Preview.string_processing in mode:
735 transformers = [string_merge, string_paren_strip]
736 else:
737 transformers = []
738 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
739 transformers = [left_hand_split]
740 else:
742 def _rhs(
743 self: object, line: Line, features: Collection[Feature], mode: Mode
744 ) -> Iterator[Line]:
745 """Wraps calls to `right_hand_split`.
747 The calls increasingly `omit` right-hand trailers (bracket pairs with
748 content), meaning the trailers get glued together to split on another
749 bracket pair instead.
750 """
751 for omit in generate_trailers_to_omit(line, mode.line_length):
752 lines = list(right_hand_split(line, mode, features, omit=omit))
753 # Note: this check is only able to figure out if the first line of the
754 # *current* transformation fits in the line length. This is true only
755 # for simple cases. All others require running more transforms via
756 # `transform_line()`. This check doesn't know if those would succeed.
757 if is_line_short_enough(lines[0], mode=mode):
758 yield from lines
759 return
761 # All splits failed, best effort split with no omits.
762 # This mostly happens to multiline strings that are by definition
763 # reported as not fitting a single line, as well as lines that contain
764 # trailing commas (those have to be exploded).
765 yield from right_hand_split(line, mode, features=features)
767 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
768 # __name__ attribute which is needed in `run_transformer` further down.
769 # Unfortunately a nested class breaks mypyc too. So a class must be created
770 # via type ... https://github.com/mypyc/mypyc/issues/884
771 rhs = type("rhs", (), {"__call__": _rhs})()
773 if Preview.string_processing in mode:
774 if line.inside_brackets:
775 transformers = [
776 string_merge,
777 string_paren_strip,
778 string_split,
779 delimiter_split,
780 standalone_comment_split,
781 string_paren_wrap,
782 rhs,
783 ]
784 else:
785 transformers = [
786 string_merge,
787 string_paren_strip,
788 string_split,
789 string_paren_wrap,
790 rhs,
791 ]
792 else:
793 if line.inside_brackets:
794 transformers = [delimiter_split, standalone_comment_split, rhs]
795 else:
796 transformers = [rhs]
797 # It's always safe to attempt hugging of power operations and pretty much every line
798 # could match.
799 transformers.append(hug_power_op)
801 for transform in transformers:
802 # We are accumulating lines in `result` because we might want to abort
803 # mission and return the original line in the end, or attempt a different
804 # split altogether.
805 try:
806 result = run_transformer(line, transform, mode, features, line_str=line_str)
807 except CannotTransform:
808 continue
809 else:
810 yield from result
811 break
813 else:
814 yield line
817def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
818 """If a funcdef has a magic trailing comma in the return type, then we should first
819 split the line with rhs to respect the comma.
820 """
821 return_type_leaves: list[Leaf] = []
822 in_return_type = False
824 for leaf in line.leaves:
825 if leaf.type == token.COLON:
826 in_return_type = False
827 if in_return_type:
828 return_type_leaves.append(leaf)
829 if leaf.type == token.RARROW:
830 in_return_type = True
832 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
833 # couple lines from it.
834 result = Line(mode=line.mode, depth=line.depth)
835 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
836 for leaf in return_type_leaves:
837 result.append(
838 leaf,
839 preformatted=True,
840 track_bracket=id(leaf) in leaves_to_track,
841 )
843 # we could also return true if the line is too long, and the return type is longer
844 # than the param list. Or if `should_split_rhs` returns True.
845 return result.magic_trailing_comma is not None
848class _BracketSplitComponent(Enum):
849 head = auto()
850 body = auto()
851 tail = auto()
854def left_hand_split(
855 line: Line, _features: Collection[Feature], mode: Mode
856) -> Iterator[Line]:
857 """Split line into many lines, starting with the first matching bracket pair.
859 Note: this usually looks weird, only use this for function definitions.
860 Prefer RHS otherwise. This is why this function is not symmetrical with
861 :func:`right_hand_split` which also handles optional parentheses.
862 """
863 for leaf_type in [token.LPAR, token.LSQB]:
864 tail_leaves: list[Leaf] = []
865 body_leaves: list[Leaf] = []
866 head_leaves: list[Leaf] = []
867 current_leaves = head_leaves
868 matching_bracket: Leaf | None = None
869 depth = 0
870 for index, leaf in enumerate(line.leaves):
871 if index == 2 and leaf.type == token.LSQB:
872 # A [ at index 2 means this is a type param, so start
873 # tracking the depth
874 depth += 1
875 elif depth > 0:
876 if leaf.type == token.LSQB:
877 depth += 1
878 elif leaf.type == token.RSQB:
879 depth -= 1
880 if (
881 current_leaves is body_leaves
882 and leaf.type in CLOSING_BRACKETS
883 and leaf.opening_bracket is matching_bracket
884 and isinstance(matching_bracket, Leaf)
885 # If the code is still on LPAR and we are inside a type
886 # param, ignore the match since this is searching
887 # for the function arguments
888 and not (leaf_type == token.LPAR and depth > 0)
889 ):
890 ensure_visible(leaf)
891 ensure_visible(matching_bracket)
892 current_leaves = tail_leaves if body_leaves else head_leaves
893 current_leaves.append(leaf)
894 if current_leaves is head_leaves:
895 if leaf.type == leaf_type and (
896 Preview.fix_type_expansion_split not in mode
897 or not (leaf_type == token.LPAR and depth > 0)
898 ):
899 matching_bracket = leaf
900 current_leaves = body_leaves
901 if matching_bracket and tail_leaves:
902 break
903 if not matching_bracket or not tail_leaves:
904 raise CannotSplit("No brackets found")
906 head = bracket_split_build_line(
907 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
908 )
909 body = bracket_split_build_line(
910 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
911 )
912 tail = bracket_split_build_line(
913 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
914 )
915 bracket_split_succeeded_or_raise(head, body, tail)
916 for result in (head, body, tail):
917 if result:
918 yield result
921def right_hand_split(
922 line: Line,
923 mode: Mode,
924 features: Collection[Feature] = (),
925 omit: Collection[LeafID] = (),
926) -> Iterator[Line]:
927 """Split line into many lines, starting with the last matching bracket pair.
929 If the split was by optional parentheses, attempt splitting without them, too.
930 `omit` is a collection of closing bracket IDs that shouldn't be considered for
931 this split.
933 Note: running this function modifies `bracket_depth` on the leaves of `line`.
934 """
935 rhs_result = _first_right_hand_split(line, omit=omit)
936 yield from _maybe_split_omitting_optional_parens(
937 rhs_result, line, mode, features=features, omit=omit
938 )
941def _first_right_hand_split(
942 line: Line,
943 omit: Collection[LeafID] = (),
944) -> RHSResult:
945 """Split the line into head, body, tail starting with the last bracket pair.
947 Note: this function should not have side effects. It's relied upon by
948 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
949 splitting on the right side of an assignment statement.
950 """
951 tail_leaves: list[Leaf] = []
952 body_leaves: list[Leaf] = []
953 head_leaves: list[Leaf] = []
954 current_leaves = tail_leaves
955 opening_bracket: Leaf | None = None
956 closing_bracket: Leaf | None = None
957 for leaf in reversed(line.leaves):
958 if current_leaves is body_leaves:
959 if leaf is opening_bracket:
960 current_leaves = head_leaves if body_leaves else tail_leaves
961 current_leaves.append(leaf)
962 if current_leaves is tail_leaves:
963 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
964 opening_bracket = leaf.opening_bracket
965 closing_bracket = leaf
966 current_leaves = body_leaves
967 if not (opening_bracket and closing_bracket and head_leaves):
968 # If there is no opening or closing_bracket that means the split failed and
969 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
970 # the matching `opening_bracket` wasn't available on `line` anymore.
971 raise CannotSplit("No brackets found")
973 tail_leaves.reverse()
974 body_leaves.reverse()
975 head_leaves.reverse()
977 body: Line | None = None
978 if (
979 Preview.hug_parens_with_braces_and_square_brackets in line.mode
980 and tail_leaves[0].value
981 and tail_leaves[0].opening_bracket is head_leaves[-1]
982 ):
983 inner_body_leaves = list(body_leaves)
984 hugged_opening_leaves: list[Leaf] = []
985 hugged_closing_leaves: list[Leaf] = []
986 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
987 unpacking_offset: int = 1 if is_unpacking else 0
988 while (
989 len(inner_body_leaves) >= 2 + unpacking_offset
990 and inner_body_leaves[-1].type in CLOSING_BRACKETS
991 and inner_body_leaves[-1].opening_bracket
992 is inner_body_leaves[unpacking_offset]
993 ):
994 if unpacking_offset:
995 hugged_opening_leaves.append(inner_body_leaves.pop(0))
996 unpacking_offset = 0
997 hugged_opening_leaves.append(inner_body_leaves.pop(0))
998 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
1000 if hugged_opening_leaves and inner_body_leaves:
1001 inner_body = bracket_split_build_line(
1002 inner_body_leaves,
1003 line,
1004 hugged_opening_leaves[-1],
1005 component=_BracketSplitComponent.body,
1006 )
1007 if (
1008 line.mode.magic_trailing_comma
1009 and inner_body_leaves[-1].type == token.COMMA
1010 ):
1011 should_hug = True
1012 else:
1013 line_length = line.mode.line_length - sum(
1014 len(str(leaf))
1015 for leaf in hugged_opening_leaves + hugged_closing_leaves
1016 )
1017 if is_line_short_enough(
1018 inner_body, mode=replace(line.mode, line_length=line_length)
1019 ):
1020 # Do not hug if it fits on a single line.
1021 should_hug = False
1022 else:
1023 should_hug = True
1024 if should_hug:
1025 body_leaves = inner_body_leaves
1026 head_leaves.extend(hugged_opening_leaves)
1027 tail_leaves = hugged_closing_leaves + tail_leaves
1028 body = inner_body # No need to re-calculate the body again later.
1030 head = bracket_split_build_line(
1031 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1032 )
1033 if body is None:
1034 body = bracket_split_build_line(
1035 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1036 )
1037 tail = bracket_split_build_line(
1038 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1039 )
1040 bracket_split_succeeded_or_raise(head, body, tail)
1041 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1044def _maybe_split_omitting_optional_parens(
1045 rhs: RHSResult,
1046 line: Line,
1047 mode: Mode,
1048 features: Collection[Feature] = (),
1049 omit: Collection[LeafID] = (),
1050) -> Iterator[Line]:
1051 if (
1052 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1053 # the opening bracket is an optional paren
1054 and rhs.opening_bracket.type == token.LPAR
1055 and not rhs.opening_bracket.value
1056 # the closing bracket is an optional paren
1057 and rhs.closing_bracket.type == token.RPAR
1058 and not rhs.closing_bracket.value
1059 # it's not an import (optional parens are the only thing we can split on
1060 # in this case; attempting a split without them is a waste of time)
1061 and not line.is_import
1062 # and we can actually remove the parens
1063 and can_omit_invisible_parens(rhs, mode.line_length)
1064 ):
1065 omit = {id(rhs.closing_bracket), *omit}
1066 try:
1067 # The RHSResult Omitting Optional Parens.
1068 rhs_oop = _first_right_hand_split(line, omit=omit)
1069 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1070 yield from _maybe_split_omitting_optional_parens(
1071 rhs_oop, line, mode, features=features, omit=omit
1072 )
1073 return
1075 except CannotSplit as e:
1076 # For chained assignments we want to use the previous successful split
1077 if line.is_chained_assignment:
1078 pass
1080 elif (
1081 not can_be_split(rhs.body)
1082 and not is_line_short_enough(rhs.body, mode=mode)
1083 and not (
1084 Preview.wrap_long_dict_values_in_parens
1085 and rhs.opening_bracket.parent
1086 and rhs.opening_bracket.parent.parent
1087 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1088 )
1089 ):
1090 raise CannotSplit(
1091 "Splitting failed, body is still too long and can't be split."
1092 ) from e
1094 elif (
1095 rhs.head.contains_multiline_strings()
1096 or rhs.tail.contains_multiline_strings()
1097 ):
1098 raise CannotSplit(
1099 "The current optional pair of parentheses is bound to fail to"
1100 " satisfy the splitting algorithm because the head or the tail"
1101 " contains multiline strings which by definition never fit one"
1102 " line."
1103 ) from e
1105 ensure_visible(rhs.opening_bracket)
1106 ensure_visible(rhs.closing_bracket)
1107 for result in (rhs.head, rhs.body, rhs.tail):
1108 if result:
1109 yield result
1112def _prefer_split_rhs_oop_over_rhs(
1113 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1114) -> bool:
1115 """
1116 Returns whether we should prefer the result from a split omitting optional parens
1117 (rhs_oop) over the original (rhs).
1118 """
1119 # contains unsplittable type ignore
1120 if (
1121 rhs_oop.head.contains_unsplittable_type_ignore()
1122 or rhs_oop.body.contains_unsplittable_type_ignore()
1123 or rhs_oop.tail.contains_unsplittable_type_ignore()
1124 ):
1125 return True
1127 # Retain optional parens around dictionary values
1128 if (
1129 Preview.wrap_long_dict_values_in_parens
1130 and rhs.opening_bracket.parent
1131 and rhs.opening_bracket.parent.parent
1132 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1133 and rhs.body.bracket_tracker.delimiters
1134 ):
1135 # Unless the split is inside the key
1136 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1138 # the split is right after `=`
1139 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1140 return True
1142 # the left side of assignment contains brackets
1143 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1144 return True
1146 # the left side of assignment is short enough (the -1 is for the ending optional
1147 # paren)
1148 if not is_line_short_enough(
1149 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1150 ):
1151 return True
1153 # the left side of assignment won't explode further because of magic trailing comma
1154 if rhs.head.magic_trailing_comma is not None:
1155 return True
1157 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1158 # the body
1159 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1160 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1161 token.EQUAL
1162 )
1163 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1164 return False
1166 has_closing_bracket_after_assign = False
1167 for leaf in reversed(rhs_oop.head.leaves):
1168 if leaf.type == token.EQUAL:
1169 break
1170 if leaf.type in CLOSING_BRACKETS:
1171 has_closing_bracket_after_assign = True
1172 break
1173 return (
1174 # contains matching brackets after the `=` (done by checking there is a
1175 # closing bracket)
1176 has_closing_bracket_after_assign
1177 or (
1178 # the split is actually from inside the optional parens (done by checking
1179 # the first line still contains the `=`)
1180 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1181 # the first line is short enough
1182 and is_line_short_enough(rhs_oop.head, mode=mode)
1183 )
1184 )
1187def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1188 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1190 Do nothing otherwise.
1192 A left- or right-hand split is based on a pair of brackets. Content before
1193 (and including) the opening bracket is left on one line, content inside the
1194 brackets is put on a separate line, and finally content starting with and
1195 following the closing bracket is put on a separate line.
1197 Those are called `head`, `body`, and `tail`, respectively. If the split
1198 produced the same line (all content in `head`) or ended up with an empty `body`
1199 and the `tail` is just the closing bracket, then it's considered failed.
1200 """
1201 tail_len = len(str(tail).strip())
1202 if not body:
1203 if tail_len == 0:
1204 raise CannotSplit("Splitting brackets produced the same line")
1206 elif tail_len < 3:
1207 raise CannotSplit(
1208 f"Splitting brackets on an empty body to save {tail_len} characters is"
1209 " not worth it"
1210 )
1213def _ensure_trailing_comma(
1214 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1215) -> bool:
1216 if not leaves:
1217 return False
1218 # Ensure a trailing comma for imports
1219 if original.is_import:
1220 return True
1221 # ...and standalone function arguments
1222 if not original.is_def:
1223 return False
1224 if opening_bracket.value != "(":
1225 return False
1226 # Don't add commas if we already have any commas
1227 if any(
1228 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1229 ):
1230 return False
1232 # Find a leaf with a parent (comments don't have parents)
1233 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1234 if leaf_with_parent is None:
1235 return True
1236 # Don't add commas inside parenthesized return annotations
1237 if get_annotation_type(leaf_with_parent) == "return":
1238 return False
1239 # Don't add commas inside PEP 604 unions
1240 if (
1241 leaf_with_parent.parent
1242 and leaf_with_parent.parent.next_sibling
1243 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1244 ):
1245 return False
1246 return True
1249def bracket_split_build_line(
1250 leaves: list[Leaf],
1251 original: Line,
1252 opening_bracket: Leaf,
1253 *,
1254 component: _BracketSplitComponent,
1255) -> Line:
1256 """Return a new line with given `leaves` and respective comments from `original`.
1258 If it's the head component, brackets will be tracked so trailing commas are
1259 respected.
1261 If it's the body component, the result line is one-indented inside brackets and as
1262 such has its first leaf's prefix normalized and a trailing comma added when
1263 expected.
1264 """
1265 result = Line(mode=original.mode, depth=original.depth)
1266 if component is _BracketSplitComponent.body:
1267 result.inside_brackets = True
1268 result.depth += 1
1269 if _ensure_trailing_comma(leaves, original, opening_bracket):
1270 for i in range(len(leaves) - 1, -1, -1):
1271 if leaves[i].type == STANDALONE_COMMENT:
1272 continue
1274 if leaves[i].type != token.COMMA:
1275 new_comma = Leaf(token.COMMA, ",")
1276 leaves.insert(i + 1, new_comma)
1277 break
1279 leaves_to_track: set[LeafID] = set()
1280 if component is _BracketSplitComponent.head:
1281 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1282 # Populate the line
1283 for leaf in leaves:
1284 result.append(
1285 leaf,
1286 preformatted=True,
1287 track_bracket=id(leaf) in leaves_to_track,
1288 )
1289 for comment_after in original.comments_after(leaf):
1290 result.append(comment_after, preformatted=True)
1291 if component is _BracketSplitComponent.body and should_split_line(
1292 result, opening_bracket
1293 ):
1294 result.should_split_rhs = True
1295 return result
1298def dont_increase_indentation(split_func: Transformer) -> Transformer:
1299 """Normalize prefix of the first leaf in every line returned by `split_func`.
1301 This is a decorator over relevant split functions.
1302 """
1304 @wraps(split_func)
1305 def split_wrapper(
1306 line: Line, features: Collection[Feature], mode: Mode
1307 ) -> Iterator[Line]:
1308 for split_line in split_func(line, features, mode):
1309 split_line.leaves[0].prefix = ""
1310 yield split_line
1312 return split_wrapper
1315def _get_last_non_comment_leaf(line: Line) -> int | None:
1316 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1317 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1318 return leaf_idx
1319 return None
1322def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1323 if is_vararg(leaf, within={syms.typedargslist}):
1324 return Feature.TRAILING_COMMA_IN_DEF in features
1325 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1326 return Feature.TRAILING_COMMA_IN_CALL in features
1327 return True
1330def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1331 if (
1332 safe
1333 and delimiter_priority == COMMA_PRIORITY
1334 and line.leaves[-1].type != token.COMMA
1335 and line.leaves[-1].type != STANDALONE_COMMENT
1336 ):
1337 new_comma = Leaf(token.COMMA, ",")
1338 line.append(new_comma)
1339 return line
1342MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1345@dont_increase_indentation
1346def delimiter_split(
1347 line: Line, features: Collection[Feature], mode: Mode
1348) -> Iterator[Line]:
1349 """Split according to delimiters of the highest priority.
1351 If the appropriate Features are given, the split will add trailing commas
1352 also in function signatures and calls that contain `*` and `**`.
1353 """
1354 if len(line.leaves) == 0:
1355 raise CannotSplit("Line empty") from None
1356 last_leaf = line.leaves[-1]
1358 bt = line.bracket_tracker
1359 try:
1360 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1361 except ValueError:
1362 raise CannotSplit("No delimiters found") from None
1364 if (
1365 delimiter_priority == DOT_PRIORITY
1366 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1367 ):
1368 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1370 current_line = Line(
1371 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1372 )
1373 lowest_depth = sys.maxsize
1374 trailing_comma_safe = True
1376 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1377 """Append `leaf` to current line or to new line if appending impossible."""
1378 nonlocal current_line
1379 try:
1380 current_line.append_safe(leaf, preformatted=True)
1381 except ValueError:
1382 yield current_line
1384 current_line = Line(
1385 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1386 )
1387 current_line.append(leaf)
1389 def append_comments(leaf: Leaf) -> Iterator[Line]:
1390 for comment_after in line.comments_after(leaf):
1391 yield from append_to_line(comment_after)
1393 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1394 for leaf_idx, leaf in enumerate(line.leaves):
1395 yield from append_to_line(leaf)
1397 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1398 id(line.leaves[leaf_idx - 1])
1399 )
1400 if (
1401 previous_priority != delimiter_priority
1402 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1403 ):
1404 yield from append_comments(leaf)
1406 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1407 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1408 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1410 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1411 current_line = _safe_add_trailing_comma(
1412 trailing_comma_safe, delimiter_priority, current_line
1413 )
1415 leaf_priority = bt.delimiters.get(id(leaf))
1416 if leaf_priority == delimiter_priority:
1417 if (
1418 leaf_idx + 1 < len(line.leaves)
1419 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1420 ):
1421 yield from append_comments(line.leaves[leaf_idx + 1])
1423 yield current_line
1424 current_line = Line(
1425 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1426 )
1428 if current_line:
1429 current_line = _safe_add_trailing_comma(
1430 trailing_comma_safe, delimiter_priority, current_line
1431 )
1432 yield current_line
1435@dont_increase_indentation
1436def standalone_comment_split(
1437 line: Line, features: Collection[Feature], mode: Mode
1438) -> Iterator[Line]:
1439 """Split standalone comments from the rest of the line."""
1440 if not line.contains_standalone_comments():
1441 raise CannotSplit("Line does not have any standalone comments")
1443 current_line = Line(
1444 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1445 )
1447 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1448 """Append `leaf` to current line or to new line if appending impossible."""
1449 nonlocal current_line
1450 try:
1451 current_line.append_safe(leaf, preformatted=True)
1452 except ValueError:
1453 yield current_line
1455 current_line = Line(
1456 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1457 )
1458 current_line.append(leaf)
1460 for leaf in line.leaves:
1461 yield from append_to_line(leaf)
1463 for comment_after in line.comments_after(leaf):
1464 yield from append_to_line(comment_after)
1466 if current_line:
1467 yield current_line
1470def normalize_invisible_parens(
1471 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1472) -> None:
1473 """Make existing optional parentheses invisible or create new ones.
1475 `parens_after` is a set of string leaf values immediately after which parens
1476 should be put.
1478 Standardizes on visible parentheses for single-element tuples, and keeps
1479 existing visible parentheses for other tuples and generator expressions.
1480 """
1481 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1482 if contains_fmt_directive(pc.value, FMT_OFF):
1483 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1484 return
1486 # The multiple context managers grammar has a different pattern, thus this is
1487 # separate from the for-loop below. This possibly wraps them in invisible parens,
1488 # and later will be removed in remove_with_parens when needed.
1489 if node.type == syms.with_stmt:
1490 _maybe_wrap_cms_in_parens(node, mode, features)
1492 check_lpar = False
1493 for index, child in enumerate(list(node.children)):
1494 # Fixes a bug where invisible parens are not properly stripped from
1495 # assignment statements that contain type annotations.
1496 if isinstance(child, Node) and child.type == syms.annassign:
1497 normalize_invisible_parens(
1498 child, parens_after=parens_after, mode=mode, features=features
1499 )
1501 # Fixes a bug where invisible parens are not properly wrapped around
1502 # case blocks.
1503 if isinstance(child, Node) and child.type == syms.case_block:
1504 normalize_invisible_parens(
1505 child, parens_after={"case"}, mode=mode, features=features
1506 )
1508 # Add parentheses around if guards in case blocks
1509 if isinstance(child, Node) and child.type == syms.guard:
1510 normalize_invisible_parens(
1511 child, parens_after={"if"}, mode=mode, features=features
1512 )
1514 # Add parentheses around long tuple unpacking in assignments.
1515 if (
1516 index == 0
1517 and isinstance(child, Node)
1518 and child.type == syms.testlist_star_expr
1519 ):
1520 check_lpar = True
1522 # Check for assignment LHS with preview feature enabled
1523 if (
1524 Preview.remove_parens_from_assignment_lhs in mode
1525 and index == 0
1526 and isinstance(child, Node)
1527 and child.type == syms.atom
1528 and node.type == syms.expr_stmt
1529 and not _atom_has_magic_trailing_comma(child, mode)
1530 and not _is_atom_multiline(child)
1531 ):
1532 if maybe_make_parens_invisible_in_atom(
1533 child,
1534 parent=node,
1535 mode=mode,
1536 features=features,
1537 remove_brackets_around_comma=True,
1538 allow_star_expr=True,
1539 ):
1540 wrap_in_parentheses(node, child, visible=False)
1542 if check_lpar:
1543 if (
1544 child.type == syms.atom
1545 and node.type == syms.for_stmt
1546 and isinstance(child.prev_sibling, Leaf)
1547 and child.prev_sibling.type == token.NAME
1548 and child.prev_sibling.value == "for"
1549 ):
1550 if maybe_make_parens_invisible_in_atom(
1551 child,
1552 parent=node,
1553 mode=mode,
1554 features=features,
1555 remove_brackets_around_comma=True,
1556 ):
1557 wrap_in_parentheses(node, child, visible=False)
1558 elif isinstance(child, Node) and node.type == syms.with_stmt:
1559 remove_with_parens(child, node, mode=mode, features=features)
1560 elif child.type == syms.atom and not (
1561 "in" in parens_after
1562 and len(child.children) == 3
1563 and is_lpar_token(child.children[0])
1564 and is_rpar_token(child.children[-1])
1565 and child.children[1].type == syms.test
1566 ):
1567 if maybe_make_parens_invisible_in_atom(
1568 child, parent=node, mode=mode, features=features
1569 ):
1570 wrap_in_parentheses(node, child, visible=False)
1571 elif is_one_tuple(child):
1572 wrap_in_parentheses(node, child, visible=True)
1573 elif node.type == syms.import_from:
1574 _normalize_import_from(node, child, index)
1575 break
1576 elif (
1577 index == 1
1578 and child.type == token.STAR
1579 and node.type == syms.except_clause
1580 ):
1581 # In except* (PEP 654), the star is actually part of
1582 # of the keyword. So we need to skip the insertion of
1583 # invisible parentheses to work more precisely.
1584 continue
1586 elif (
1587 isinstance(child, Leaf)
1588 and child.next_sibling is not None
1589 and child.next_sibling.type == token.COLON
1590 and child.value == "case"
1591 ):
1592 # A special patch for "case case:" scenario, the second occurrence
1593 # of case will be not parsed as a Python keyword.
1594 break
1596 elif not is_multiline_string(child):
1597 wrap_in_parentheses(node, child, visible=False)
1599 comma_check = child.type == token.COMMA
1601 check_lpar = isinstance(child, Leaf) and (
1602 child.value in parens_after or comma_check
1603 )
1606def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1607 # "import from" nodes store parentheses directly as part of
1608 # the statement
1609 if is_lpar_token(child):
1610 assert is_rpar_token(parent.children[-1])
1611 # make parentheses invisible
1612 child.value = ""
1613 parent.children[-1].value = ""
1614 elif child.type != token.STAR:
1615 # insert invisible parentheses
1616 parent.insert_child(index, Leaf(token.LPAR, ""))
1617 parent.append_child(Leaf(token.RPAR, ""))
1620def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1621 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1622 if (
1623 node.children[1].type == syms.atom
1624 and node.children[1].children[0].type == token.LPAR
1625 ):
1626 if maybe_make_parens_invisible_in_atom(
1627 node.children[1],
1628 parent=node,
1629 mode=mode,
1630 features=features,
1631 remove_brackets_around_comma=True,
1632 ):
1633 wrap_in_parentheses(node, node.children[1], visible=False)
1635 # Since await is an expression we shouldn't remove
1636 # brackets in cases where this would change
1637 # the AST due to operator precedence.
1638 # Therefore we only aim to remove brackets around
1639 # power nodes that aren't also await expressions themselves.
1640 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1641 # N.B. We've still removed any redundant nested brackets though :)
1642 opening_bracket = cast(Leaf, node.children[1].children[0])
1643 closing_bracket = cast(Leaf, node.children[1].children[-1])
1644 bracket_contents = node.children[1].children[1]
1645 if isinstance(bracket_contents, Node) and (
1646 bracket_contents.type != syms.power
1647 or bracket_contents.children[0].type == token.AWAIT
1648 or any(
1649 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1650 for child in bracket_contents.children
1651 )
1652 ):
1653 ensure_visible(opening_bracket)
1654 ensure_visible(closing_bracket)
1657def _maybe_wrap_cms_in_parens(
1658 node: Node, mode: Mode, features: Collection[Feature]
1659) -> None:
1660 """When enabled and safe, wrap the multiple context managers in invisible parens.
1662 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1663 """
1664 if (
1665 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1666 or len(node.children) <= 2
1667 # If it's an atom, it's already wrapped in parens.
1668 or node.children[1].type == syms.atom
1669 ):
1670 return
1671 colon_index: int | None = None
1672 for i in range(2, len(node.children)):
1673 if node.children[i].type == token.COLON:
1674 colon_index = i
1675 break
1676 if colon_index is not None:
1677 lpar = Leaf(token.LPAR, "")
1678 rpar = Leaf(token.RPAR, "")
1679 context_managers = node.children[1:colon_index]
1680 for child in context_managers:
1681 child.remove()
1682 # After wrapping, the with_stmt will look like this:
1683 # with_stmt
1684 # NAME 'with'
1685 # atom
1686 # LPAR ''
1687 # testlist_gexp
1688 # ... <-- context_managers
1689 # /testlist_gexp
1690 # RPAR ''
1691 # /atom
1692 # COLON ':'
1693 new_child = Node(
1694 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1695 )
1696 node.insert_child(1, new_child)
1699def remove_with_parens(
1700 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1701) -> None:
1702 """Recursively hide optional parens in `with` statements."""
1703 # Removing all unnecessary parentheses in with statements in one pass is a tad
1704 # complex as different variations of bracketed statements result in pretty
1705 # different parse trees:
1706 #
1707 # with (open("file")) as f: # this is an asexpr_test
1708 # ...
1709 #
1710 # with (open("file") as f): # this is an atom containing an
1711 # ... # asexpr_test
1712 #
1713 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1714 # ... # asexpr_test
1715 #
1716 # with (open("file") as f, open("file") as f): # an atom containing a
1717 # ... # testlist_gexp which then
1718 # # contains multiple asexpr_test(s)
1719 if node.type == syms.atom:
1720 if maybe_make_parens_invisible_in_atom(
1721 node,
1722 parent=parent,
1723 mode=mode,
1724 features=features,
1725 remove_brackets_around_comma=True,
1726 ):
1727 wrap_in_parentheses(parent, node, visible=False)
1728 if isinstance(node.children[1], Node):
1729 remove_with_parens(node.children[1], node, mode=mode, features=features)
1730 elif node.type == syms.testlist_gexp:
1731 for child in node.children:
1732 if isinstance(child, Node):
1733 remove_with_parens(child, node, mode=mode, features=features)
1734 elif node.type == syms.asexpr_test and not any(
1735 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1736 ):
1737 if maybe_make_parens_invisible_in_atom(
1738 node.children[0],
1739 parent=node,
1740 mode=mode,
1741 features=features,
1742 remove_brackets_around_comma=True,
1743 ):
1744 wrap_in_parentheses(node, node.children[0], visible=False)
1747def _atom_has_magic_trailing_comma(node: LN, mode: Mode) -> bool:
1748 """Check if an atom node has a magic trailing comma.
1750 Returns True for single-element tuples with trailing commas like (a,),
1751 which should be preserved to maintain their tuple type.
1752 """
1753 if not mode.magic_trailing_comma:
1754 return False
1756 return is_one_tuple(node)
1759def _is_atom_multiline(node: LN) -> bool:
1760 """Check if an atom node is multiline (indicating intentional formatting)."""
1761 if not isinstance(node, Node) or len(node.children) < 3:
1762 return False
1764 # Check the middle child (between LPAR and RPAR) for newlines in its subtree
1765 # The first child's prefix contains blank lines/comments before the opening paren
1766 middle = node.children[1]
1767 for child in middle.pre_order():
1768 if isinstance(child, Leaf) and "\n" in child.prefix:
1769 return True
1771 return False
1774def maybe_make_parens_invisible_in_atom(
1775 node: LN,
1776 parent: LN,
1777 mode: Mode,
1778 features: Collection[Feature],
1779 remove_brackets_around_comma: bool = False,
1780 allow_star_expr: bool = False,
1781) -> bool:
1782 """If it's safe, make the parens in the atom `node` invisible, recursively.
1783 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1784 as they are redundant.
1786 Returns whether the node should itself be wrapped in invisible parentheses.
1787 """
1788 if (
1789 node.type not in (syms.atom, syms.expr)
1790 or is_empty_tuple(node)
1791 or is_one_tuple(node)
1792 or (is_tuple(node) and parent.type == syms.asexpr_test)
1793 or (
1794 is_tuple(node)
1795 and parent.type == syms.with_stmt
1796 and has_sibling_with_type(node, token.COMMA)
1797 )
1798 or (is_yield(node) and parent.type != syms.expr_stmt)
1799 or (
1800 # This condition tries to prevent removing non-optional brackets
1801 # around a tuple, however, can be a bit overzealous so we provide
1802 # and option to skip this check for `for` and `with` statements.
1803 not remove_brackets_around_comma
1804 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1805 # Skip this check in Preview mode in order to
1806 # Remove parentheses around multiple exception types in except and
1807 # except* without as. See PEP 758 for details.
1808 and not (
1809 Preview.remove_parens_around_except_types in mode
1810 and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1811 # is a tuple
1812 and is_tuple(node)
1813 # has a parent node
1814 and node.parent is not None
1815 # parent is an except clause
1816 and node.parent.type == syms.except_clause
1817 # is not immediately followed by as clause
1818 and not (
1819 node.next_sibling is not None
1820 and is_name_token(node.next_sibling)
1821 and node.next_sibling.value == "as"
1822 )
1823 )
1824 )
1825 or is_tuple_containing_walrus(node)
1826 or (not allow_star_expr and is_tuple_containing_star(node))
1827 or is_generator(node)
1828 ):
1829 return False
1831 if is_walrus_assignment(node):
1832 if parent.type in [
1833 syms.annassign,
1834 syms.expr_stmt,
1835 syms.assert_stmt,
1836 syms.return_stmt,
1837 syms.except_clause,
1838 syms.funcdef,
1839 syms.with_stmt,
1840 syms.testlist_gexp,
1841 syms.tname,
1842 # these ones aren't useful to end users, but they do please fuzzers
1843 syms.for_stmt,
1844 syms.del_stmt,
1845 syms.for_stmt,
1846 ]:
1847 return False
1849 first = node.children[0]
1850 last = node.children[-1]
1851 if is_lpar_token(first) and is_rpar_token(last):
1852 middle = node.children[1]
1853 # make parentheses invisible
1854 if (
1855 # If the prefix of `middle` includes a type comment with
1856 # ignore annotation, then we do not remove the parentheses
1857 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
1858 ):
1859 first.value = ""
1860 last.value = ""
1861 maybe_make_parens_invisible_in_atom(
1862 middle,
1863 parent=parent,
1864 mode=mode,
1865 features=features,
1866 remove_brackets_around_comma=remove_brackets_around_comma,
1867 )
1869 if is_atom_with_invisible_parens(middle):
1870 # Strip the invisible parens from `middle` by replacing
1871 # it with the child in-between the invisible parens
1872 middle.replace(middle.children[1])
1874 if middle.children[0].prefix.strip():
1875 # Preserve comments before first paren
1876 middle.children[1].prefix = (
1877 middle.children[0].prefix + middle.children[1].prefix
1878 )
1880 if middle.children[-1].prefix.strip():
1881 # Preserve comments before last paren
1882 last.prefix = middle.children[-1].prefix + last.prefix
1884 return False
1886 return True
1889def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1890 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1892 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1893 return False
1895 # We're essentially checking if the body is delimited by commas and there's more
1896 # than one of them (we're excluding the trailing comma and if the delimiter priority
1897 # is still commas, that means there's more).
1898 exclude = set()
1899 trailing_comma = False
1900 try:
1901 last_leaf = line.leaves[-1]
1902 if last_leaf.type == token.COMMA:
1903 trailing_comma = True
1904 exclude.add(id(last_leaf))
1905 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1906 except (IndexError, ValueError):
1907 return False
1909 return max_priority == COMMA_PRIORITY and (
1910 (line.mode.magic_trailing_comma and trailing_comma)
1911 # always explode imports
1912 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1913 )
1916def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1917 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1919 Brackets can be omitted if the entire trailer up to and including
1920 a preceding closing bracket fits in one line.
1922 Yielded sets are cumulative (contain results of previous yields, too). First
1923 set is empty, unless the line should explode, in which case bracket pairs until
1924 the one that needs to explode are omitted.
1925 """
1927 omit: set[LeafID] = set()
1928 if not line.magic_trailing_comma:
1929 yield omit
1931 length = 4 * line.depth
1932 opening_bracket: Leaf | None = None
1933 closing_bracket: Leaf | None = None
1934 inner_brackets: set[LeafID] = set()
1935 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1936 length += leaf_length
1937 if length > line_length:
1938 break
1940 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1941 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1942 break
1944 if opening_bracket:
1945 if leaf is opening_bracket:
1946 opening_bracket = None
1947 elif leaf.type in CLOSING_BRACKETS:
1948 prev = line.leaves[index - 1] if index > 0 else None
1949 if (
1950 prev
1951 and prev.type == token.COMMA
1952 and leaf.opening_bracket is not None
1953 and not is_one_sequence_between(
1954 leaf.opening_bracket, leaf, line.leaves
1955 )
1956 ):
1957 # Never omit bracket pairs with trailing commas.
1958 # We need to explode on those.
1959 break
1961 inner_brackets.add(id(leaf))
1962 elif leaf.type in CLOSING_BRACKETS:
1963 prev = line.leaves[index - 1] if index > 0 else None
1964 if prev and prev.type in OPENING_BRACKETS:
1965 # Empty brackets would fail a split so treat them as "inner"
1966 # brackets (e.g. only add them to the `omit` set if another
1967 # pair of brackets was good enough.
1968 inner_brackets.add(id(leaf))
1969 continue
1971 if closing_bracket:
1972 omit.add(id(closing_bracket))
1973 omit.update(inner_brackets)
1974 inner_brackets.clear()
1975 yield omit
1977 if (
1978 prev
1979 and prev.type == token.COMMA
1980 and leaf.opening_bracket is not None
1981 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1982 ):
1983 # Never omit bracket pairs with trailing commas.
1984 # We need to explode on those.
1985 break
1987 if leaf.value:
1988 opening_bracket = leaf.opening_bracket
1989 closing_bracket = leaf
1992def run_transformer(
1993 line: Line,
1994 transform: Transformer,
1995 mode: Mode,
1996 features: Collection[Feature],
1997 *,
1998 line_str: str = "",
1999) -> list[Line]:
2000 if not line_str:
2001 line_str = line_to_string(line)
2002 result: list[Line] = []
2003 for transformed_line in transform(line, features, mode):
2004 if str(transformed_line).strip("\n") == line_str:
2005 raise CannotTransform("Line transformer returned an unchanged result")
2007 result.extend(transform_line(transformed_line, mode=mode, features=features))
2009 features_set = set(features)
2010 if (
2011 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
2012 or transform.__class__.__name__ != "rhs"
2013 or not line.bracket_tracker.invisible
2014 or any(bracket.value for bracket in line.bracket_tracker.invisible)
2015 or line.contains_multiline_strings()
2016 or result[0].contains_uncollapsable_type_comments()
2017 or result[0].contains_unsplittable_type_ignore()
2018 or is_line_short_enough(result[0], mode=mode)
2019 # If any leaves have no parents (which _can_ occur since
2020 # `transform(line)` potentially destroys the line's underlying node
2021 # structure), then we can't proceed. Doing so would cause the below
2022 # call to `append_leaves()` to fail.
2023 or any(leaf.parent is None for leaf in line.leaves)
2024 ):
2025 return result
2027 line_copy = line.clone()
2028 append_leaves(line_copy, line, line.leaves)
2029 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
2030 second_opinion = run_transformer(
2031 line_copy, transform, mode, features_fop, line_str=line_str
2032 )
2033 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
2034 result = second_opinion
2035 return result