Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import (
21 FMT_OFF,
22 FMT_ON,
23 _contains_fmt_directive,
24 generate_comments,
25 list_comments,
26)
27from black.lines import (
28 Line,
29 RHSResult,
30 append_leaves,
31 can_be_split,
32 can_omit_invisible_parens,
33 is_line_short_enough,
34 line_to_string,
35)
36from black.mode import Feature, Mode, Preview
37from black.nodes import (
38 ASSIGNMENTS,
39 BRACKETS,
40 CLOSING_BRACKETS,
41 OPENING_BRACKETS,
42 STANDALONE_COMMENT,
43 STATEMENT,
44 WHITESPACE,
45 Visitor,
46 ensure_visible,
47 fstring_tstring_to_string,
48 get_annotation_type,
49 has_sibling_with_type,
50 is_arith_like,
51 is_async_stmt_or_funcdef,
52 is_atom_with_invisible_parens,
53 is_docstring,
54 is_empty_tuple,
55 is_generator,
56 is_lpar_token,
57 is_multiline_string,
58 is_name_token,
59 is_one_sequence_between,
60 is_one_tuple,
61 is_parent_function_or_class,
62 is_part_of_annotation,
63 is_rpar_token,
64 is_stub_body,
65 is_stub_suite,
66 is_tuple,
67 is_tuple_containing_star,
68 is_tuple_containing_walrus,
69 is_type_ignore_comment_string,
70 is_vararg,
71 is_walrus_assignment,
72 is_yield,
73 syms,
74 wrap_in_parentheses,
75)
76from black.numerics import normalize_numeric_literal
77from black.strings import (
78 fix_multiline_docstring,
79 get_string_prefix,
80 normalize_string_prefix,
81 normalize_string_quotes,
82 normalize_unicode_escape_sequences,
83)
84from black.trans import (
85 CannotTransform,
86 StringMerger,
87 StringParenStripper,
88 StringParenWrapper,
89 StringSplitter,
90 Transformer,
91 hug_power_op,
92)
93from blib2to3.pgen2 import token
94from blib2to3.pytree import Leaf, Node
96# types
97LeafID = int
98LN = Union[Leaf, Node]
101class CannotSplit(CannotTransform):
102 """A readable split that fits the allotted line length is impossible."""
105# This isn't a dataclass because @dataclass + Generic breaks mypyc.
106# See also https://github.com/mypyc/mypyc/issues/827.
107class LineGenerator(Visitor[Line]):
108 """Generates reformatted Line objects. Empty lines are not emitted.
110 Note: destroys the tree it's visiting by mutating prefixes of its leaves
111 in ways that will no longer stringify to valid Python code on the tree.
112 """
114 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
115 self.mode = mode
116 self.features = features
117 self.current_line: Line
118 self.__post_init__()
120 def line(self, indent: int = 0) -> Iterator[Line]:
121 """Generate a line.
123 If the line is empty, only emit if it makes sense.
124 If the line is too long, split it first and then generate.
126 If any lines were generated, set up a new current_line.
127 """
128 if not self.current_line:
129 self.current_line.depth += indent
130 return # Line is empty, don't emit. Creating a new one unnecessary.
132 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
133 self.current_line.leaves[0]
134 ):
135 # Special case for async def/for/with statements. `visit_async_stmt`
136 # adds an `ASYNC` leaf then visits the child def/for/with statement
137 # nodes. Line yields from those nodes shouldn't treat the former
138 # `ASYNC` leaf as a complete line.
139 return
141 complete_line = self.current_line
142 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
143 yield complete_line
145 def visit_default(self, node: LN) -> Iterator[Line]:
146 """Default `visit_*()` implementation. Recurses to children of `node`."""
147 if isinstance(node, Leaf):
148 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
149 for comment in generate_comments(node, mode=self.mode):
150 if any_open_brackets:
151 # any comment within brackets is subject to splitting
152 self.current_line.append(comment)
153 elif comment.type == token.COMMENT:
154 # regular trailing comment
155 self.current_line.append(comment)
156 yield from self.line()
158 else:
159 # regular standalone comment
160 yield from self.line()
162 self.current_line.append(comment)
163 yield from self.line()
165 if any_open_brackets:
166 node.prefix = ""
167 if node.type not in WHITESPACE:
168 self.current_line.append(node)
169 yield from super().visit_default(node)
171 def visit_test(self, node: Node) -> Iterator[Line]:
172 """Visit an `x if y else z` test"""
174 already_parenthesized = (
175 node.prev_sibling and node.prev_sibling.type == token.LPAR
176 )
178 if not already_parenthesized:
179 # Similar to logic in wrap_in_parentheses
180 lpar = Leaf(token.LPAR, "")
181 rpar = Leaf(token.RPAR, "")
182 prefix = node.prefix
183 node.prefix = ""
184 lpar.prefix = prefix
185 node.insert_child(0, lpar)
186 node.append_child(rpar)
188 yield from self.visit_default(node)
190 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
191 """Increase indentation level, maybe yield a line."""
192 # In blib2to3 INDENT never holds comments.
193 yield from self.line(+1)
194 yield from self.visit_default(node)
196 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
197 """Decrease indentation level, maybe yield a line."""
198 # The current line might still wait for trailing comments. At DEDENT time
199 # there won't be any (they would be prefixes on the preceding NEWLINE).
200 # Emit the line then.
201 yield from self.line()
203 # While DEDENT has no value, its prefix may contain standalone comments
204 # that belong to the current indentation level. Get 'em.
205 yield from self.visit_default(node)
207 # Finally, emit the dedent.
208 yield from self.line(-1)
210 def visit_stmt(
211 self, node: Node, keywords: set[str], parens: set[str]
212 ) -> Iterator[Line]:
213 """Visit a statement.
215 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
216 `def`, `with`, `class`, `assert`, and assignments.
218 The relevant Python language `keywords` for a given statement will be
219 NAME leaves within it. This methods puts those on a separate line.
221 `parens` holds a set of string leaf values immediately after which
222 invisible parens should be put.
223 """
224 normalize_invisible_parens(
225 node, parens_after=parens, mode=self.mode, features=self.features
226 )
227 for child in node.children:
228 if is_name_token(child) and child.value in keywords:
229 yield from self.line()
231 yield from self.visit(child)
233 def visit_typeparams(self, node: Node) -> Iterator[Line]:
234 yield from self.visit_default(node)
235 node.children[0].prefix = ""
237 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
238 yield from self.visit_default(node)
239 node.children[1].prefix = ""
241 def visit_paramspec(self, node: Node) -> Iterator[Line]:
242 yield from self.visit_default(node)
243 node.children[1].prefix = ""
245 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
246 if Preview.wrap_long_dict_values_in_parens in self.mode:
247 for i, child in enumerate(node.children):
248 if i == 0:
249 continue
250 if node.children[i - 1].type == token.COLON:
251 if (
252 child.type == syms.atom
253 and child.children[0].type in OPENING_BRACKETS
254 and not is_walrus_assignment(child)
255 ):
256 maybe_make_parens_invisible_in_atom(
257 child,
258 parent=node,
259 mode=self.mode,
260 features=self.features,
261 remove_brackets_around_comma=False,
262 )
263 else:
264 wrap_in_parentheses(node, child, visible=False)
265 yield from self.visit_default(node)
267 def visit_funcdef(self, node: Node) -> Iterator[Line]:
268 """Visit function definition."""
269 yield from self.line()
271 # Remove redundant brackets around return type annotation.
272 is_return_annotation = False
273 for child in node.children:
274 if child.type == token.RARROW:
275 is_return_annotation = True
276 elif is_return_annotation:
277 if child.type == syms.atom and child.children[0].type == token.LPAR:
278 if maybe_make_parens_invisible_in_atom(
279 child,
280 parent=node,
281 mode=self.mode,
282 features=self.features,
283 remove_brackets_around_comma=False,
284 ):
285 wrap_in_parentheses(node, child, visible=False)
286 else:
287 wrap_in_parentheses(node, child, visible=False)
288 is_return_annotation = False
290 for child in node.children:
291 yield from self.visit(child)
293 def visit_match_case(self, node: Node) -> Iterator[Line]:
294 """Visit either a match or case statement."""
295 normalize_invisible_parens(
296 node, parens_after=set(), mode=self.mode, features=self.features
297 )
299 yield from self.line()
300 for child in node.children:
301 yield from self.visit(child)
303 def visit_suite(self, node: Node) -> Iterator[Line]:
304 """Visit a suite."""
305 if is_stub_suite(node):
306 yield from self.visit(node.children[2])
307 else:
308 yield from self.visit_default(node)
310 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
311 """Visit a statement without nested statements."""
312 prev_type: int | None = None
313 for child in node.children:
314 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
315 wrap_in_parentheses(node, child, visible=False)
316 prev_type = child.type
318 if node.parent and node.parent.type in STATEMENT:
319 if is_parent_function_or_class(node) and is_stub_body(node):
320 yield from self.visit_default(node)
321 else:
322 yield from self.line(+1)
323 yield from self.visit_default(node)
324 yield from self.line(-1)
326 else:
327 if node.parent and is_stub_suite(node.parent):
328 node.prefix = ""
329 yield from self.visit_default(node)
330 return
331 yield from self.line()
332 yield from self.visit_default(node)
334 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
335 """Visit `async def`, `async for`, `async with`."""
336 yield from self.line()
338 children = iter(node.children)
339 for child in children:
340 yield from self.visit(child)
342 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
343 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
344 # line.
345 break
347 internal_stmt = next(children)
348 yield from self.visit(internal_stmt)
350 def visit_decorators(self, node: Node) -> Iterator[Line]:
351 """Visit decorators."""
352 for child in node.children:
353 yield from self.line()
354 yield from self.visit(child)
356 def visit_power(self, node: Node) -> Iterator[Line]:
357 for idx, leaf in enumerate(node.children[:-1]):
358 next_leaf = node.children[idx + 1]
360 if not isinstance(leaf, Leaf):
361 continue
363 value = leaf.value.lower()
364 if (
365 leaf.type == token.NUMBER
366 and next_leaf.type == syms.trailer
367 # Ensure that we are in an attribute trailer
368 and next_leaf.children[0].type == token.DOT
369 # It shouldn't wrap hexadecimal, binary and octal literals
370 and not value.startswith(("0x", "0b", "0o"))
371 # It shouldn't wrap complex literals
372 and "j" not in value
373 ):
374 wrap_in_parentheses(node, leaf)
376 remove_await_parens(node, mode=self.mode, features=self.features)
378 yield from self.visit_default(node)
380 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
381 """Remove a semicolon and put the other statement on a separate line."""
382 yield from self.line()
384 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
385 """End of file. Process outstanding comments and end with a newline."""
386 yield from self.visit_default(leaf)
387 yield from self.line()
389 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
390 if not self.current_line.bracket_tracker.any_open_brackets():
391 yield from self.line()
392 # STANDALONE_COMMENT nodes created by our special handling in
393 # normalize_fmt_off for comment-only blocks have fmt:off as the first
394 # line and fmt:on as the last line (each directive on its own line,
395 # not embedded in other text). These should be appended directly
396 # without calling visit_default, which would process their prefix and
397 # lose indentation. Normal STANDALONE_COMMENT nodes go through
398 # visit_default.
399 value = leaf.value
400 lines = value.splitlines()
401 if len(lines) >= 2:
402 # Check if first line (after stripping whitespace) is exactly a
403 # fmt:off directive
404 first_line = lines[0].lstrip()
405 first_is_fmt_off = first_line in FMT_OFF
406 # Check if last line (after stripping whitespace) is exactly a
407 # fmt:on directive
408 last_line = lines[-1].lstrip()
409 last_is_fmt_on = last_line in FMT_ON
410 is_fmt_off_block = first_is_fmt_off and last_is_fmt_on
411 else:
412 is_fmt_off_block = False
413 if is_fmt_off_block:
414 # This is a fmt:off/on block from normalize_fmt_off - we still need
415 # to process any prefix comments (like markdown comments) but append
416 # the fmt block itself directly to preserve its formatting
418 # Only process prefix comments if there actually is a prefix with comments
419 if leaf.prefix and any(
420 line.strip().startswith("#")
421 and not _contains_fmt_directive(line.strip())
422 for line in leaf.prefix.split("\n")
423 ):
424 for comment in generate_comments(leaf, mode=self.mode):
425 yield from self.line()
426 self.current_line.append(comment)
427 yield from self.line()
428 # Clear the prefix since we've processed it as comments above
429 leaf.prefix = ""
431 self.current_line.append(leaf)
432 yield from self.line()
433 else:
434 # Normal standalone comment - process through visit_default
435 yield from self.visit_default(leaf)
437 def visit_factor(self, node: Node) -> Iterator[Line]:
438 """Force parentheses between a unary op and a binary power:
440 -2 ** 8 -> -(2 ** 8)
441 """
442 _operator, operand = node.children
443 if (
444 operand.type == syms.power
445 and len(operand.children) == 3
446 and operand.children[1].type == token.DOUBLESTAR
447 ):
448 lpar = Leaf(token.LPAR, "(")
449 rpar = Leaf(token.RPAR, ")")
450 index = operand.remove() or 0
451 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
452 yield from self.visit_default(node)
454 def visit_tname(self, node: Node) -> Iterator[Line]:
455 """
456 Add potential parentheses around types in function parameter lists to be made
457 into real parentheses in case the type hint is too long to fit on a line
458 Examples:
459 def foo(a: int, b: float = 7): ...
461 ->
463 def foo(a: (int), b: (float) = 7): ...
464 """
465 if len(node.children) == 3 and maybe_make_parens_invisible_in_atom(
466 node.children[2], parent=node, mode=self.mode, features=self.features
467 ):
468 wrap_in_parentheses(node, node.children[2], visible=False)
470 yield from self.visit_default(node)
472 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
473 normalize_unicode_escape_sequences(leaf)
475 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
476 # We're ignoring docstrings with backslash newline escapes because changing
477 # indentation of those changes the AST representation of the code.
478 if self.mode.string_normalization:
479 docstring = normalize_string_prefix(leaf.value)
480 # We handle string normalization at the end of this method, but since
481 # what we do right now acts differently depending on quote style (ex.
482 # see padding logic below), there's a possibility for unstable
483 # formatting. To avoid a situation where this function formats a
484 # docstring differently on the second pass, normalize it early.
485 docstring = normalize_string_quotes(docstring)
486 else:
487 docstring = leaf.value
488 prefix = get_string_prefix(docstring)
489 docstring = docstring[len(prefix) :] # Remove the prefix
490 quote_char = docstring[0]
491 # A natural way to remove the outer quotes is to do:
492 # docstring = docstring.strip(quote_char)
493 # but that breaks on """""x""" (which is '""x').
494 # So we actually need to remove the first character and the next two
495 # characters but only if they are the same as the first.
496 quote_len = 1 if docstring[1] != quote_char else 3
497 docstring = docstring[quote_len:-quote_len]
498 docstring_started_empty = not docstring
499 indent = " " * 4 * self.current_line.depth
501 if is_multiline_string(leaf):
502 docstring = fix_multiline_docstring(docstring, indent)
503 else:
504 docstring = docstring.strip()
506 has_trailing_backslash = False
507 if docstring:
508 # Add some padding if the docstring starts / ends with a quote mark.
509 if docstring[0] == quote_char:
510 docstring = " " + docstring
511 if docstring[-1] == quote_char:
512 docstring += " "
513 if docstring[-1] == "\\":
514 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
515 if backslash_count % 2:
516 # Odd number of tailing backslashes, add some padding to
517 # avoid escaping the closing string quote.
518 docstring += " "
519 has_trailing_backslash = True
520 elif not docstring_started_empty:
521 docstring = " "
523 # We could enforce triple quotes at this point.
524 quote = quote_char * quote_len
526 # It's invalid to put closing single-character quotes on a new line.
527 if quote_len == 3:
528 # We need to find the length of the last line of the docstring
529 # to find if we can add the closing quotes to the line without
530 # exceeding the maximum line length.
531 # If docstring is one line, we don't put the closing quotes on a
532 # separate line because it looks ugly (#3320).
533 lines = docstring.splitlines()
534 last_line_length = len(lines[-1]) if docstring else 0
536 # If adding closing quotes would cause the last line to exceed
537 # the maximum line length, and the closing quote is not
538 # prefixed by a newline then put a line break before
539 # the closing quotes
540 if (
541 len(lines) > 1
542 and last_line_length + quote_len > self.mode.line_length
543 and len(indent) + quote_len <= self.mode.line_length
544 and not has_trailing_backslash
545 ):
546 if leaf.value[-1 - quote_len] == "\n":
547 leaf.value = prefix + quote + docstring + quote
548 else:
549 leaf.value = prefix + quote + docstring + "\n" + indent + quote
550 else:
551 leaf.value = prefix + quote + docstring + quote
552 else:
553 leaf.value = prefix + quote + docstring + quote
555 if self.mode.string_normalization and leaf.type == token.STRING:
556 leaf.value = normalize_string_prefix(leaf.value)
557 leaf.value = normalize_string_quotes(leaf.value)
558 yield from self.visit_default(leaf)
560 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
561 normalize_numeric_literal(leaf)
562 yield from self.visit_default(leaf)
564 def visit_atom(self, node: Node) -> Iterator[Line]:
565 """Visit any atom"""
566 if len(node.children) == 3:
567 first = node.children[0]
568 last = node.children[-1]
569 if (first.type == token.LSQB and last.type == token.RSQB) or (
570 first.type == token.LBRACE and last.type == token.RBRACE
571 ):
572 # Lists or sets of one item
573 maybe_make_parens_invisible_in_atom(
574 node.children[1],
575 parent=node,
576 mode=self.mode,
577 features=self.features,
578 )
580 yield from self.visit_default(node)
582 def visit_fstring(self, node: Node) -> Iterator[Line]:
583 # currently we don't want to format and split f-strings at all.
584 string_leaf = fstring_tstring_to_string(node)
585 node.replace(string_leaf)
586 if "\\" in string_leaf.value and any(
587 "\\" in str(child)
588 for child in node.children
589 if child.type == syms.fstring_replacement_field
590 ):
591 # string normalization doesn't account for nested quotes,
592 # causing breakages. skip normalization when nested quotes exist
593 yield from self.visit_default(string_leaf)
594 return
595 yield from self.visit_STRING(string_leaf)
597 def visit_tstring(self, node: Node) -> Iterator[Line]:
598 # currently we don't want to format and split t-strings at all.
599 string_leaf = fstring_tstring_to_string(node)
600 node.replace(string_leaf)
601 if "\\" in string_leaf.value and any(
602 "\\" in str(child)
603 for child in node.children
604 if child.type == syms.fstring_replacement_field
605 ):
606 # string normalization doesn't account for nested quotes,
607 # causing breakages. skip normalization when nested quotes exist
608 yield from self.visit_default(string_leaf)
609 return
610 yield from self.visit_STRING(string_leaf)
612 # TODO: Uncomment Implementation to format f-string children
613 # fstring_start = node.children[0]
614 # fstring_end = node.children[-1]
615 # assert isinstance(fstring_start, Leaf)
616 # assert isinstance(fstring_end, Leaf)
618 # quote_char = fstring_end.value[0]
619 # quote_idx = fstring_start.value.index(quote_char)
620 # prefix, quote = (
621 # fstring_start.value[:quote_idx],
622 # fstring_start.value[quote_idx:]
623 # )
625 # if not is_docstring(node, self.mode):
626 # prefix = normalize_string_prefix(prefix)
628 # assert quote == fstring_end.value
630 # is_raw_fstring = "r" in prefix or "R" in prefix
631 # middles = [
632 # leaf
633 # for leaf in node.leaves()
634 # if leaf.type == token.FSTRING_MIDDLE
635 # ]
637 # if self.mode.string_normalization:
638 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
640 # fstring_start.value = prefix + quote
641 # fstring_end.value = quote
643 # yield from self.visit_default(node)
645 def visit_comp_for(self, node: Node) -> Iterator[Line]:
646 if Preview.wrap_comprehension_in in self.mode:
647 normalize_invisible_parens(
648 node, parens_after={"in"}, mode=self.mode, features=self.features
649 )
650 yield from self.visit_default(node)
652 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
653 yield from self.visit_comp_for(node)
655 def __post_init__(self) -> None:
656 """You are in a twisty little maze of passages."""
657 self.current_line = Line(mode=self.mode)
659 v = self.visit_stmt
660 Ø: set[str] = set()
661 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
662 self.visit_if_stmt = partial(
663 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
664 )
665 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
666 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
667 self.visit_try_stmt = partial(
668 v, keywords={"try", "except", "else", "finally"}, parens=Ø
669 )
670 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
671 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
672 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
674 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
675 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
676 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
677 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
678 self.visit_async_funcdef = self.visit_async_stmt
679 self.visit_decorated = self.visit_decorators
681 # PEP 634
682 self.visit_match_stmt = self.visit_match_case
683 self.visit_case_block = self.visit_match_case
684 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
687def _hugging_power_ops_line_to_string(
688 line: Line,
689 features: Collection[Feature],
690 mode: Mode,
691) -> str | None:
692 try:
693 return line_to_string(next(hug_power_op(line, features, mode)))
694 except CannotTransform:
695 return None
698def transform_line(
699 line: Line, mode: Mode, features: Collection[Feature] = ()
700) -> Iterator[Line]:
701 """Transform a `line`, potentially splitting it into many lines.
703 They should fit in the allotted `line_length` but might not be able to.
705 `features` are syntactical features that may be used in the output.
706 """
707 if line.is_comment:
708 yield line
709 return
711 line_str = line_to_string(line)
713 # We need the line string when power operators are hugging to determine if we should
714 # split the line. Default to line_str, if no power operator are present on the line.
715 line_str_hugging_power_ops = (
716 _hugging_power_ops_line_to_string(line, features, mode) or line_str
717 )
719 ll = mode.line_length
720 sn = mode.string_normalization
721 string_merge = StringMerger(ll, sn)
722 string_paren_strip = StringParenStripper(ll, sn)
723 string_split = StringSplitter(ll, sn)
724 string_paren_wrap = StringParenWrapper(ll, sn)
726 transformers: list[Transformer]
727 if (
728 not line.contains_uncollapsable_type_comments()
729 and not line.should_split_rhs
730 and not line.magic_trailing_comma
731 and (
732 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
733 or line.contains_unsplittable_type_ignore()
734 )
735 and not (line.inside_brackets and line.contains_standalone_comments())
736 and not line.contains_implicit_multiline_string_with_comments()
737 ):
738 # Only apply basic string preprocessing, since lines shouldn't be split here.
739 if Preview.string_processing in mode:
740 transformers = [string_merge, string_paren_strip]
741 else:
742 transformers = []
743 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
744 transformers = [left_hand_split]
745 else:
747 def _rhs(
748 self: object, line: Line, features: Collection[Feature], mode: Mode
749 ) -> Iterator[Line]:
750 """Wraps calls to `right_hand_split`.
752 The calls increasingly `omit` right-hand trailers (bracket pairs with
753 content), meaning the trailers get glued together to split on another
754 bracket pair instead.
755 """
756 for omit in generate_trailers_to_omit(line, mode.line_length):
757 lines = list(right_hand_split(line, mode, features, omit=omit))
758 # Note: this check is only able to figure out if the first line of the
759 # *current* transformation fits in the line length. This is true only
760 # for simple cases. All others require running more transforms via
761 # `transform_line()`. This check doesn't know if those would succeed.
762 if is_line_short_enough(lines[0], mode=mode):
763 yield from lines
764 return
766 # All splits failed, best effort split with no omits.
767 # This mostly happens to multiline strings that are by definition
768 # reported as not fitting a single line, as well as lines that contain
769 # trailing commas (those have to be exploded).
770 yield from right_hand_split(line, mode, features=features)
772 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
773 # __name__ attribute which is needed in `run_transformer` further down.
774 # Unfortunately a nested class breaks mypyc too. So a class must be created
775 # via type ... https://github.com/mypyc/mypyc/issues/884
776 rhs = type("rhs", (), {"__call__": _rhs})()
778 if Preview.string_processing in mode:
779 if line.inside_brackets:
780 transformers = [
781 string_merge,
782 string_paren_strip,
783 string_split,
784 delimiter_split,
785 standalone_comment_split,
786 string_paren_wrap,
787 rhs,
788 ]
789 else:
790 transformers = [
791 string_merge,
792 string_paren_strip,
793 string_split,
794 string_paren_wrap,
795 rhs,
796 ]
797 else:
798 if line.inside_brackets:
799 transformers = [delimiter_split, standalone_comment_split, rhs]
800 else:
801 transformers = [rhs]
802 # It's always safe to attempt hugging of power operations and pretty much every line
803 # could match.
804 transformers.append(hug_power_op)
806 for transform in transformers:
807 # We are accumulating lines in `result` because we might want to abort
808 # mission and return the original line in the end, or attempt a different
809 # split altogether.
810 try:
811 result = run_transformer(line, transform, mode, features, line_str=line_str)
812 except CannotTransform:
813 continue
814 else:
815 yield from result
816 break
818 else:
819 yield line
822def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
823 """If a funcdef has a magic trailing comma in the return type, then we should first
824 split the line with rhs to respect the comma.
825 """
826 return_type_leaves: list[Leaf] = []
827 in_return_type = False
829 for leaf in line.leaves:
830 if leaf.type == token.COLON:
831 in_return_type = False
832 if in_return_type:
833 return_type_leaves.append(leaf)
834 if leaf.type == token.RARROW:
835 in_return_type = True
837 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
838 # couple lines from it.
839 result = Line(mode=line.mode, depth=line.depth)
840 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
841 for leaf in return_type_leaves:
842 result.append(
843 leaf,
844 preformatted=True,
845 track_bracket=id(leaf) in leaves_to_track,
846 )
848 # we could also return true if the line is too long, and the return type is longer
849 # than the param list. Or if `should_split_rhs` returns True.
850 return result.magic_trailing_comma is not None
853class _BracketSplitComponent(Enum):
854 head = auto()
855 body = auto()
856 tail = auto()
859def left_hand_split(
860 line: Line, _features: Collection[Feature], mode: Mode
861) -> Iterator[Line]:
862 """Split line into many lines, starting with the first matching bracket pair.
864 Note: this usually looks weird, only use this for function definitions.
865 Prefer RHS otherwise. This is why this function is not symmetrical with
866 :func:`right_hand_split` which also handles optional parentheses.
867 """
868 for leaf_type in [token.LPAR, token.LSQB]:
869 tail_leaves: list[Leaf] = []
870 body_leaves: list[Leaf] = []
871 head_leaves: list[Leaf] = []
872 current_leaves = head_leaves
873 matching_bracket: Leaf | None = None
874 depth = 0
875 for index, leaf in enumerate(line.leaves):
876 if index == 2 and leaf.type == token.LSQB:
877 # A [ at index 2 means this is a type param, so start
878 # tracking the depth
879 depth += 1
880 elif depth > 0:
881 if leaf.type == token.LSQB:
882 depth += 1
883 elif leaf.type == token.RSQB:
884 depth -= 1
885 if (
886 current_leaves is body_leaves
887 and leaf.type in CLOSING_BRACKETS
888 and leaf.opening_bracket is matching_bracket
889 and isinstance(matching_bracket, Leaf)
890 # If the code is still on LPAR and we are inside a type
891 # param, ignore the match since this is searching
892 # for the function arguments
893 and not (leaf_type == token.LPAR and depth > 0)
894 ):
895 ensure_visible(leaf)
896 ensure_visible(matching_bracket)
897 current_leaves = tail_leaves if body_leaves else head_leaves
898 current_leaves.append(leaf)
899 if current_leaves is head_leaves:
900 if leaf.type == leaf_type and (
901 Preview.fix_type_expansion_split not in mode
902 or not (leaf_type == token.LPAR and depth > 0)
903 ):
904 matching_bracket = leaf
905 current_leaves = body_leaves
906 if matching_bracket and tail_leaves:
907 break
908 if not matching_bracket or not tail_leaves:
909 raise CannotSplit("No brackets found")
911 head = bracket_split_build_line(
912 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
913 )
914 body = bracket_split_build_line(
915 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
916 )
917 tail = bracket_split_build_line(
918 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
919 )
920 bracket_split_succeeded_or_raise(head, body, tail)
921 for result in (head, body, tail):
922 if result:
923 yield result
926def right_hand_split(
927 line: Line,
928 mode: Mode,
929 features: Collection[Feature] = (),
930 omit: Collection[LeafID] = (),
931) -> Iterator[Line]:
932 """Split line into many lines, starting with the last matching bracket pair.
934 If the split was by optional parentheses, attempt splitting without them, too.
935 `omit` is a collection of closing bracket IDs that shouldn't be considered for
936 this split.
938 Note: running this function modifies `bracket_depth` on the leaves of `line`.
939 """
940 rhs_result = _first_right_hand_split(line, omit=omit)
941 yield from _maybe_split_omitting_optional_parens(
942 rhs_result, line, mode, features=features, omit=omit
943 )
946def _first_right_hand_split(
947 line: Line,
948 omit: Collection[LeafID] = (),
949) -> RHSResult:
950 """Split the line into head, body, tail starting with the last bracket pair.
952 Note: this function should not have side effects. It's relied upon by
953 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
954 splitting on the right side of an assignment statement.
955 """
956 tail_leaves: list[Leaf] = []
957 body_leaves: list[Leaf] = []
958 head_leaves: list[Leaf] = []
959 current_leaves = tail_leaves
960 opening_bracket: Leaf | None = None
961 closing_bracket: Leaf | None = None
962 for leaf in reversed(line.leaves):
963 if current_leaves is body_leaves:
964 if leaf is opening_bracket:
965 current_leaves = head_leaves if body_leaves else tail_leaves
966 current_leaves.append(leaf)
967 if current_leaves is tail_leaves:
968 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
969 opening_bracket = leaf.opening_bracket
970 closing_bracket = leaf
971 current_leaves = body_leaves
972 if not (opening_bracket and closing_bracket and head_leaves):
973 # If there is no opening or closing_bracket that means the split failed and
974 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
975 # the matching `opening_bracket` wasn't available on `line` anymore.
976 raise CannotSplit("No brackets found")
978 tail_leaves.reverse()
979 body_leaves.reverse()
980 head_leaves.reverse()
982 body: Line | None = None
983 if (
984 Preview.hug_parens_with_braces_and_square_brackets in line.mode
985 and tail_leaves[0].value
986 and tail_leaves[0].opening_bracket is head_leaves[-1]
987 ):
988 inner_body_leaves = list(body_leaves)
989 hugged_opening_leaves: list[Leaf] = []
990 hugged_closing_leaves: list[Leaf] = []
991 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
992 unpacking_offset: int = 1 if is_unpacking else 0
993 while (
994 len(inner_body_leaves) >= 2 + unpacking_offset
995 and inner_body_leaves[-1].type in CLOSING_BRACKETS
996 and inner_body_leaves[-1].opening_bracket
997 is inner_body_leaves[unpacking_offset]
998 ):
999 if unpacking_offset:
1000 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1001 unpacking_offset = 0
1002 hugged_opening_leaves.append(inner_body_leaves.pop(0))
1003 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
1005 if hugged_opening_leaves and inner_body_leaves:
1006 inner_body = bracket_split_build_line(
1007 inner_body_leaves,
1008 line,
1009 hugged_opening_leaves[-1],
1010 component=_BracketSplitComponent.body,
1011 )
1012 if (
1013 line.mode.magic_trailing_comma
1014 and inner_body_leaves[-1].type == token.COMMA
1015 ):
1016 should_hug = True
1017 else:
1018 line_length = line.mode.line_length - sum(
1019 len(str(leaf))
1020 for leaf in hugged_opening_leaves + hugged_closing_leaves
1021 )
1022 if is_line_short_enough(
1023 inner_body, mode=replace(line.mode, line_length=line_length)
1024 ):
1025 # Do not hug if it fits on a single line.
1026 should_hug = False
1027 else:
1028 should_hug = True
1029 if should_hug:
1030 body_leaves = inner_body_leaves
1031 head_leaves.extend(hugged_opening_leaves)
1032 tail_leaves = hugged_closing_leaves + tail_leaves
1033 body = inner_body # No need to re-calculate the body again later.
1035 head = bracket_split_build_line(
1036 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1037 )
1038 if body is None:
1039 body = bracket_split_build_line(
1040 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1041 )
1042 tail = bracket_split_build_line(
1043 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1044 )
1045 bracket_split_succeeded_or_raise(head, body, tail)
1046 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1049def _maybe_split_omitting_optional_parens(
1050 rhs: RHSResult,
1051 line: Line,
1052 mode: Mode,
1053 features: Collection[Feature] = (),
1054 omit: Collection[LeafID] = (),
1055) -> Iterator[Line]:
1056 if (
1057 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1058 # the opening bracket is an optional paren
1059 and rhs.opening_bracket.type == token.LPAR
1060 and not rhs.opening_bracket.value
1061 # the closing bracket is an optional paren
1062 and rhs.closing_bracket.type == token.RPAR
1063 and not rhs.closing_bracket.value
1064 # it's not an import (optional parens are the only thing we can split on
1065 # in this case; attempting a split without them is a waste of time)
1066 and not line.is_import
1067 # and we can actually remove the parens
1068 and can_omit_invisible_parens(rhs, mode.line_length)
1069 ):
1070 omit = {id(rhs.closing_bracket), *omit}
1071 try:
1072 # The RHSResult Omitting Optional Parens.
1073 rhs_oop = _first_right_hand_split(line, omit=omit)
1074 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1075 yield from _maybe_split_omitting_optional_parens(
1076 rhs_oop, line, mode, features=features, omit=omit
1077 )
1078 return
1080 except CannotSplit as e:
1081 # For chained assignments we want to use the previous successful split
1082 if line.is_chained_assignment:
1083 pass
1085 elif (
1086 not can_be_split(rhs.body)
1087 and not is_line_short_enough(rhs.body, mode=mode)
1088 and not (
1089 Preview.wrap_long_dict_values_in_parens
1090 and rhs.opening_bracket.parent
1091 and rhs.opening_bracket.parent.parent
1092 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1093 )
1094 ):
1095 raise CannotSplit(
1096 "Splitting failed, body is still too long and can't be split."
1097 ) from e
1099 elif (
1100 rhs.head.contains_multiline_strings()
1101 or rhs.tail.contains_multiline_strings()
1102 ):
1103 raise CannotSplit(
1104 "The current optional pair of parentheses is bound to fail to"
1105 " satisfy the splitting algorithm because the head or the tail"
1106 " contains multiline strings which by definition never fit one"
1107 " line."
1108 ) from e
1110 ensure_visible(rhs.opening_bracket)
1111 ensure_visible(rhs.closing_bracket)
1112 for result in (rhs.head, rhs.body, rhs.tail):
1113 if result:
1114 yield result
1117def _prefer_split_rhs_oop_over_rhs(
1118 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1119) -> bool:
1120 """
1121 Returns whether we should prefer the result from a split omitting optional parens
1122 (rhs_oop) over the original (rhs).
1123 """
1124 # contains unsplittable type ignore
1125 if (
1126 rhs_oop.head.contains_unsplittable_type_ignore()
1127 or rhs_oop.body.contains_unsplittable_type_ignore()
1128 or rhs_oop.tail.contains_unsplittable_type_ignore()
1129 ):
1130 return True
1132 # Retain optional parens around dictionary values
1133 if (
1134 Preview.wrap_long_dict_values_in_parens
1135 and rhs.opening_bracket.parent
1136 and rhs.opening_bracket.parent.parent
1137 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1138 and rhs.body.bracket_tracker.delimiters
1139 ):
1140 # Unless the split is inside the key
1141 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1143 # the split is right after `=`
1144 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1145 return True
1147 # the left side of assignment contains brackets
1148 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1149 return True
1151 # the left side of assignment is short enough (the -1 is for the ending optional
1152 # paren)
1153 if not is_line_short_enough(
1154 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1155 ):
1156 return True
1158 # the left side of assignment won't explode further because of magic trailing comma
1159 if rhs.head.magic_trailing_comma is not None:
1160 return True
1162 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1163 # the body
1164 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1165 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1166 token.EQUAL
1167 )
1168 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1169 return False
1171 has_closing_bracket_after_assign = False
1172 for leaf in reversed(rhs_oop.head.leaves):
1173 if leaf.type == token.EQUAL:
1174 break
1175 if leaf.type in CLOSING_BRACKETS:
1176 has_closing_bracket_after_assign = True
1177 break
1178 return (
1179 # contains matching brackets after the `=` (done by checking there is a
1180 # closing bracket)
1181 has_closing_bracket_after_assign
1182 or (
1183 # the split is actually from inside the optional parens (done by checking
1184 # the first line still contains the `=`)
1185 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1186 # the first line is short enough
1187 and is_line_short_enough(rhs_oop.head, mode=mode)
1188 )
1189 )
1192def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1193 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1195 Do nothing otherwise.
1197 A left- or right-hand split is based on a pair of brackets. Content before
1198 (and including) the opening bracket is left on one line, content inside the
1199 brackets is put on a separate line, and finally content starting with and
1200 following the closing bracket is put on a separate line.
1202 Those are called `head`, `body`, and `tail`, respectively. If the split
1203 produced the same line (all content in `head`) or ended up with an empty `body`
1204 and the `tail` is just the closing bracket, then it's considered failed.
1205 """
1206 tail_len = len(str(tail).strip())
1207 if not body:
1208 if tail_len == 0:
1209 raise CannotSplit("Splitting brackets produced the same line")
1211 elif tail_len < 3:
1212 raise CannotSplit(
1213 f"Splitting brackets on an empty body to save {tail_len} characters is"
1214 " not worth it"
1215 )
1218def _ensure_trailing_comma(
1219 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1220) -> bool:
1221 if not leaves:
1222 return False
1223 # Ensure a trailing comma for imports
1224 if original.is_import:
1225 return True
1226 # ...and standalone function arguments
1227 if not original.is_def:
1228 return False
1229 if opening_bracket.value != "(":
1230 return False
1231 # Don't add commas if we already have any commas
1232 if any(
1233 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1234 ):
1235 return False
1237 # Find a leaf with a parent (comments don't have parents)
1238 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1239 if leaf_with_parent is None:
1240 return True
1241 # Don't add commas inside parenthesized return annotations
1242 if get_annotation_type(leaf_with_parent) == "return":
1243 return False
1244 # Don't add commas inside PEP 604 unions
1245 if (
1246 leaf_with_parent.parent
1247 and leaf_with_parent.parent.next_sibling
1248 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1249 ):
1250 return False
1251 return True
1254def bracket_split_build_line(
1255 leaves: list[Leaf],
1256 original: Line,
1257 opening_bracket: Leaf,
1258 *,
1259 component: _BracketSplitComponent,
1260) -> Line:
1261 """Return a new line with given `leaves` and respective comments from `original`.
1263 If it's the head component, brackets will be tracked so trailing commas are
1264 respected.
1266 If it's the body component, the result line is one-indented inside brackets and as
1267 such has its first leaf's prefix normalized and a trailing comma added when
1268 expected.
1269 """
1270 result = Line(mode=original.mode, depth=original.depth)
1271 if component is _BracketSplitComponent.body:
1272 result.inside_brackets = True
1273 result.depth += 1
1274 if _ensure_trailing_comma(leaves, original, opening_bracket):
1275 for i in range(len(leaves) - 1, -1, -1):
1276 if leaves[i].type == STANDALONE_COMMENT:
1277 continue
1279 if leaves[i].type != token.COMMA:
1280 new_comma = Leaf(token.COMMA, ",")
1281 leaves.insert(i + 1, new_comma)
1282 break
1284 leaves_to_track: set[LeafID] = set()
1285 if component is _BracketSplitComponent.head:
1286 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1287 # Populate the line
1288 for leaf in leaves:
1289 result.append(
1290 leaf,
1291 preformatted=True,
1292 track_bracket=id(leaf) in leaves_to_track,
1293 )
1294 for comment_after in original.comments_after(leaf):
1295 result.append(comment_after, preformatted=True)
1296 if component is _BracketSplitComponent.body and should_split_line(
1297 result, opening_bracket
1298 ):
1299 result.should_split_rhs = True
1300 return result
1303def dont_increase_indentation(split_func: Transformer) -> Transformer:
1304 """Normalize prefix of the first leaf in every line returned by `split_func`.
1306 This is a decorator over relevant split functions.
1307 """
1309 @wraps(split_func)
1310 def split_wrapper(
1311 line: Line, features: Collection[Feature], mode: Mode
1312 ) -> Iterator[Line]:
1313 for split_line in split_func(line, features, mode):
1314 split_line.leaves[0].prefix = ""
1315 yield split_line
1317 return split_wrapper
1320def _get_last_non_comment_leaf(line: Line) -> int | None:
1321 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1322 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1323 return leaf_idx
1324 return None
1327def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1328 if is_vararg(leaf, within={syms.typedargslist}):
1329 return Feature.TRAILING_COMMA_IN_DEF in features
1330 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1331 return Feature.TRAILING_COMMA_IN_CALL in features
1332 return True
1335def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1336 if (
1337 safe
1338 and delimiter_priority == COMMA_PRIORITY
1339 and line.leaves[-1].type != token.COMMA
1340 and line.leaves[-1].type != STANDALONE_COMMENT
1341 ):
1342 new_comma = Leaf(token.COMMA, ",")
1343 line.append(new_comma)
1344 return line
1347MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1350@dont_increase_indentation
1351def delimiter_split(
1352 line: Line, features: Collection[Feature], mode: Mode
1353) -> Iterator[Line]:
1354 """Split according to delimiters of the highest priority.
1356 If the appropriate Features are given, the split will add trailing commas
1357 also in function signatures and calls that contain `*` and `**`.
1358 """
1359 if len(line.leaves) == 0:
1360 raise CannotSplit("Line empty") from None
1361 last_leaf = line.leaves[-1]
1363 bt = line.bracket_tracker
1364 try:
1365 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1366 except ValueError:
1367 raise CannotSplit("No delimiters found") from None
1369 if (
1370 delimiter_priority == DOT_PRIORITY
1371 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1372 ):
1373 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1375 current_line = Line(
1376 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1377 )
1378 lowest_depth = sys.maxsize
1379 trailing_comma_safe = True
1381 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1382 """Append `leaf` to current line or to new line if appending impossible."""
1383 nonlocal current_line
1384 try:
1385 current_line.append_safe(leaf, preformatted=True)
1386 except ValueError:
1387 yield current_line
1389 current_line = Line(
1390 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1391 )
1392 current_line.append(leaf)
1394 def append_comments(leaf: Leaf) -> Iterator[Line]:
1395 for comment_after in line.comments_after(leaf):
1396 yield from append_to_line(comment_after)
1398 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1399 for leaf_idx, leaf in enumerate(line.leaves):
1400 yield from append_to_line(leaf)
1402 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1403 id(line.leaves[leaf_idx - 1])
1404 )
1405 if (
1406 previous_priority != delimiter_priority
1407 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1408 ):
1409 yield from append_comments(leaf)
1411 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1412 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1413 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1415 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1416 current_line = _safe_add_trailing_comma(
1417 trailing_comma_safe, delimiter_priority, current_line
1418 )
1420 leaf_priority = bt.delimiters.get(id(leaf))
1421 if leaf_priority == delimiter_priority:
1422 if (
1423 leaf_idx + 1 < len(line.leaves)
1424 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1425 ):
1426 yield from append_comments(line.leaves[leaf_idx + 1])
1428 yield current_line
1429 current_line = Line(
1430 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1431 )
1433 if current_line:
1434 current_line = _safe_add_trailing_comma(
1435 trailing_comma_safe, delimiter_priority, current_line
1436 )
1437 yield current_line
1440@dont_increase_indentation
1441def standalone_comment_split(
1442 line: Line, features: Collection[Feature], mode: Mode
1443) -> Iterator[Line]:
1444 """Split standalone comments from the rest of the line."""
1445 if not line.contains_standalone_comments():
1446 raise CannotSplit("Line does not have any standalone comments")
1448 current_line = Line(
1449 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1450 )
1452 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1453 """Append `leaf` to current line or to new line if appending impossible."""
1454 nonlocal current_line
1455 try:
1456 current_line.append_safe(leaf, preformatted=True)
1457 except ValueError:
1458 yield current_line
1460 current_line = Line(
1461 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1462 )
1463 current_line.append(leaf)
1465 for leaf in line.leaves:
1466 yield from append_to_line(leaf)
1468 for comment_after in line.comments_after(leaf):
1469 yield from append_to_line(comment_after)
1471 if current_line:
1472 yield current_line
1475def normalize_invisible_parens( # noqa: C901
1476 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1477) -> None:
1478 """Make existing optional parentheses invisible or create new ones.
1480 `parens_after` is a set of string leaf values immediately after which parens
1481 should be put.
1483 Standardizes on visible parentheses for single-element tuples, and keeps
1484 existing visible parentheses for other tuples and generator expressions.
1485 """
1486 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1487 if pc.value in FMT_OFF:
1488 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1489 return
1491 # The multiple context managers grammar has a different pattern, thus this is
1492 # separate from the for-loop below. This possibly wraps them in invisible parens,
1493 # and later will be removed in remove_with_parens when needed.
1494 if node.type == syms.with_stmt:
1495 _maybe_wrap_cms_in_parens(node, mode, features)
1497 check_lpar = False
1498 for index, child in enumerate(list(node.children)):
1499 # Fixes a bug where invisible parens are not properly stripped from
1500 # assignment statements that contain type annotations.
1501 if isinstance(child, Node) and child.type == syms.annassign:
1502 normalize_invisible_parens(
1503 child, parens_after=parens_after, mode=mode, features=features
1504 )
1506 # Fixes a bug where invisible parens are not properly wrapped around
1507 # case blocks.
1508 if isinstance(child, Node) and child.type == syms.case_block:
1509 normalize_invisible_parens(
1510 child, parens_after={"case"}, mode=mode, features=features
1511 )
1513 # Add parentheses around if guards in case blocks
1514 if isinstance(child, Node) and child.type == syms.guard:
1515 normalize_invisible_parens(
1516 child, parens_after={"if"}, mode=mode, features=features
1517 )
1519 # Add parentheses around long tuple unpacking in assignments.
1520 if (
1521 index == 0
1522 and isinstance(child, Node)
1523 and child.type == syms.testlist_star_expr
1524 ):
1525 check_lpar = True
1527 if check_lpar:
1528 if (
1529 child.type == syms.atom
1530 and node.type == syms.for_stmt
1531 and isinstance(child.prev_sibling, Leaf)
1532 and child.prev_sibling.type == token.NAME
1533 and child.prev_sibling.value == "for"
1534 ):
1535 if maybe_make_parens_invisible_in_atom(
1536 child,
1537 parent=node,
1538 mode=mode,
1539 features=features,
1540 remove_brackets_around_comma=True,
1541 ):
1542 wrap_in_parentheses(node, child, visible=False)
1543 elif isinstance(child, Node) and node.type == syms.with_stmt:
1544 remove_with_parens(child, node, mode=mode, features=features)
1545 elif child.type == syms.atom and not (
1546 "in" in parens_after
1547 and len(child.children) == 3
1548 and is_lpar_token(child.children[0])
1549 and is_rpar_token(child.children[-1])
1550 and child.children[1].type == syms.test
1551 ):
1552 if maybe_make_parens_invisible_in_atom(
1553 child, parent=node, mode=mode, features=features
1554 ):
1555 wrap_in_parentheses(node, child, visible=False)
1556 elif is_one_tuple(child):
1557 wrap_in_parentheses(node, child, visible=True)
1558 elif node.type == syms.import_from:
1559 _normalize_import_from(node, child, index)
1560 break
1561 elif (
1562 index == 1
1563 and child.type == token.STAR
1564 and node.type == syms.except_clause
1565 ):
1566 # In except* (PEP 654), the star is actually part of
1567 # of the keyword. So we need to skip the insertion of
1568 # invisible parentheses to work more precisely.
1569 continue
1571 elif (
1572 isinstance(child, Leaf)
1573 and child.next_sibling is not None
1574 and child.next_sibling.type == token.COLON
1575 and child.value == "case"
1576 ):
1577 # A special patch for "case case:" scenario, the second occurrence
1578 # of case will be not parsed as a Python keyword.
1579 break
1581 elif not is_multiline_string(child):
1582 wrap_in_parentheses(node, child, visible=False)
1584 comma_check = child.type == token.COMMA
1586 check_lpar = isinstance(child, Leaf) and (
1587 child.value in parens_after or comma_check
1588 )
1591def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1592 # "import from" nodes store parentheses directly as part of
1593 # the statement
1594 if is_lpar_token(child):
1595 assert is_rpar_token(parent.children[-1])
1596 # make parentheses invisible
1597 child.value = ""
1598 parent.children[-1].value = ""
1599 elif child.type != token.STAR:
1600 # insert invisible parentheses
1601 parent.insert_child(index, Leaf(token.LPAR, ""))
1602 parent.append_child(Leaf(token.RPAR, ""))
1605def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1606 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1607 if (
1608 node.children[1].type == syms.atom
1609 and node.children[1].children[0].type == token.LPAR
1610 ):
1611 if maybe_make_parens_invisible_in_atom(
1612 node.children[1],
1613 parent=node,
1614 mode=mode,
1615 features=features,
1616 remove_brackets_around_comma=True,
1617 ):
1618 wrap_in_parentheses(node, node.children[1], visible=False)
1620 # Since await is an expression we shouldn't remove
1621 # brackets in cases where this would change
1622 # the AST due to operator precedence.
1623 # Therefore we only aim to remove brackets around
1624 # power nodes that aren't also await expressions themselves.
1625 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1626 # N.B. We've still removed any redundant nested brackets though :)
1627 opening_bracket = cast(Leaf, node.children[1].children[0])
1628 closing_bracket = cast(Leaf, node.children[1].children[-1])
1629 bracket_contents = node.children[1].children[1]
1630 if isinstance(bracket_contents, Node) and (
1631 bracket_contents.type != syms.power
1632 or bracket_contents.children[0].type == token.AWAIT
1633 or any(
1634 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1635 for child in bracket_contents.children
1636 )
1637 ):
1638 ensure_visible(opening_bracket)
1639 ensure_visible(closing_bracket)
1642def _maybe_wrap_cms_in_parens(
1643 node: Node, mode: Mode, features: Collection[Feature]
1644) -> None:
1645 """When enabled and safe, wrap the multiple context managers in invisible parens.
1647 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1648 """
1649 if (
1650 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1651 or len(node.children) <= 2
1652 # If it's an atom, it's already wrapped in parens.
1653 or node.children[1].type == syms.atom
1654 ):
1655 return
1656 colon_index: int | None = None
1657 for i in range(2, len(node.children)):
1658 if node.children[i].type == token.COLON:
1659 colon_index = i
1660 break
1661 if colon_index is not None:
1662 lpar = Leaf(token.LPAR, "")
1663 rpar = Leaf(token.RPAR, "")
1664 context_managers = node.children[1:colon_index]
1665 for child in context_managers:
1666 child.remove()
1667 # After wrapping, the with_stmt will look like this:
1668 # with_stmt
1669 # NAME 'with'
1670 # atom
1671 # LPAR ''
1672 # testlist_gexp
1673 # ... <-- context_managers
1674 # /testlist_gexp
1675 # RPAR ''
1676 # /atom
1677 # COLON ':'
1678 new_child = Node(
1679 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1680 )
1681 node.insert_child(1, new_child)
1684def remove_with_parens(
1685 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1686) -> None:
1687 """Recursively hide optional parens in `with` statements."""
1688 # Removing all unnecessary parentheses in with statements in one pass is a tad
1689 # complex as different variations of bracketed statements result in pretty
1690 # different parse trees:
1691 #
1692 # with (open("file")) as f: # this is an asexpr_test
1693 # ...
1694 #
1695 # with (open("file") as f): # this is an atom containing an
1696 # ... # asexpr_test
1697 #
1698 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1699 # ... # asexpr_test
1700 #
1701 # with (open("file") as f, open("file") as f): # an atom containing a
1702 # ... # testlist_gexp which then
1703 # # contains multiple asexpr_test(s)
1704 if node.type == syms.atom:
1705 if maybe_make_parens_invisible_in_atom(
1706 node,
1707 parent=parent,
1708 mode=mode,
1709 features=features,
1710 remove_brackets_around_comma=True,
1711 ):
1712 wrap_in_parentheses(parent, node, visible=False)
1713 if isinstance(node.children[1], Node):
1714 remove_with_parens(node.children[1], node, mode=mode, features=features)
1715 elif node.type == syms.testlist_gexp:
1716 for child in node.children:
1717 if isinstance(child, Node):
1718 remove_with_parens(child, node, mode=mode, features=features)
1719 elif node.type == syms.asexpr_test and not any(
1720 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1721 ):
1722 if maybe_make_parens_invisible_in_atom(
1723 node.children[0],
1724 parent=node,
1725 mode=mode,
1726 features=features,
1727 remove_brackets_around_comma=True,
1728 ):
1729 wrap_in_parentheses(node, node.children[0], visible=False)
1732def maybe_make_parens_invisible_in_atom(
1733 node: LN,
1734 parent: LN,
1735 mode: Mode,
1736 features: Collection[Feature],
1737 remove_brackets_around_comma: bool = False,
1738) -> bool:
1739 """If it's safe, make the parens in the atom `node` invisible, recursively.
1740 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1741 as they are redundant.
1743 Returns whether the node should itself be wrapped in invisible parentheses.
1744 """
1745 if (
1746 node.type not in (syms.atom, syms.expr)
1747 or is_empty_tuple(node)
1748 or is_one_tuple(node)
1749 or (is_tuple(node) and parent.type == syms.asexpr_test)
1750 or (
1751 is_tuple(node)
1752 and parent.type == syms.with_stmt
1753 and has_sibling_with_type(node, token.COMMA)
1754 )
1755 or (is_yield(node) and parent.type != syms.expr_stmt)
1756 or (
1757 # This condition tries to prevent removing non-optional brackets
1758 # around a tuple, however, can be a bit overzealous so we provide
1759 # and option to skip this check for `for` and `with` statements.
1760 not remove_brackets_around_comma
1761 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1762 # Skip this check in Preview mode in order to
1763 # Remove parentheses around multiple exception types in except and
1764 # except* without as. See PEP 758 for details.
1765 and not (
1766 Preview.remove_parens_around_except_types in mode
1767 and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1768 # is a tuple
1769 and is_tuple(node)
1770 # has a parent node
1771 and node.parent is not None
1772 # parent is an except clause
1773 and node.parent.type == syms.except_clause
1774 # is not immediately followed by as clause
1775 and not (
1776 node.next_sibling is not None
1777 and is_name_token(node.next_sibling)
1778 and node.next_sibling.value == "as"
1779 )
1780 )
1781 )
1782 or is_tuple_containing_walrus(node)
1783 or is_tuple_containing_star(node)
1784 or is_generator(node)
1785 ):
1786 return False
1788 if is_walrus_assignment(node):
1789 if parent.type in [
1790 syms.annassign,
1791 syms.expr_stmt,
1792 syms.assert_stmt,
1793 syms.return_stmt,
1794 syms.except_clause,
1795 syms.funcdef,
1796 syms.with_stmt,
1797 syms.testlist_gexp,
1798 syms.tname,
1799 # these ones aren't useful to end users, but they do please fuzzers
1800 syms.for_stmt,
1801 syms.del_stmt,
1802 syms.for_stmt,
1803 ]:
1804 return False
1806 first = node.children[0]
1807 last = node.children[-1]
1808 if is_lpar_token(first) and is_rpar_token(last):
1809 middle = node.children[1]
1810 # make parentheses invisible
1811 if (
1812 # If the prefix of `middle` includes a type comment with
1813 # ignore annotation, then we do not remove the parentheses
1814 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
1815 ):
1816 first.value = ""
1817 last.value = ""
1818 maybe_make_parens_invisible_in_atom(
1819 middle,
1820 parent=parent,
1821 mode=mode,
1822 features=features,
1823 remove_brackets_around_comma=remove_brackets_around_comma,
1824 )
1826 if is_atom_with_invisible_parens(middle):
1827 # Strip the invisible parens from `middle` by replacing
1828 # it with the child in-between the invisible parens
1829 middle.replace(middle.children[1])
1831 if middle.children[0].prefix.strip():
1832 # Preserve comments before first paren
1833 middle.children[1].prefix = (
1834 middle.children[0].prefix + middle.children[1].prefix
1835 )
1837 if middle.children[-1].prefix.strip():
1838 # Preserve comments before last paren
1839 last.prefix = middle.children[-1].prefix + last.prefix
1841 return False
1843 return True
1846def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1847 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1849 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1850 return False
1852 # We're essentially checking if the body is delimited by commas and there's more
1853 # than one of them (we're excluding the trailing comma and if the delimiter priority
1854 # is still commas, that means there's more).
1855 exclude = set()
1856 trailing_comma = False
1857 try:
1858 last_leaf = line.leaves[-1]
1859 if last_leaf.type == token.COMMA:
1860 trailing_comma = True
1861 exclude.add(id(last_leaf))
1862 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1863 except (IndexError, ValueError):
1864 return False
1866 return max_priority == COMMA_PRIORITY and (
1867 (line.mode.magic_trailing_comma and trailing_comma)
1868 # always explode imports
1869 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1870 )
1873def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1874 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1876 Brackets can be omitted if the entire trailer up to and including
1877 a preceding closing bracket fits in one line.
1879 Yielded sets are cumulative (contain results of previous yields, too). First
1880 set is empty, unless the line should explode, in which case bracket pairs until
1881 the one that needs to explode are omitted.
1882 """
1884 omit: set[LeafID] = set()
1885 if not line.magic_trailing_comma:
1886 yield omit
1888 length = 4 * line.depth
1889 opening_bracket: Leaf | None = None
1890 closing_bracket: Leaf | None = None
1891 inner_brackets: set[LeafID] = set()
1892 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1893 length += leaf_length
1894 if length > line_length:
1895 break
1897 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1898 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1899 break
1901 if opening_bracket:
1902 if leaf is opening_bracket:
1903 opening_bracket = None
1904 elif leaf.type in CLOSING_BRACKETS:
1905 prev = line.leaves[index - 1] if index > 0 else None
1906 if (
1907 prev
1908 and prev.type == token.COMMA
1909 and leaf.opening_bracket is not None
1910 and not is_one_sequence_between(
1911 leaf.opening_bracket, leaf, line.leaves
1912 )
1913 ):
1914 # Never omit bracket pairs with trailing commas.
1915 # We need to explode on those.
1916 break
1918 inner_brackets.add(id(leaf))
1919 elif leaf.type in CLOSING_BRACKETS:
1920 prev = line.leaves[index - 1] if index > 0 else None
1921 if prev and prev.type in OPENING_BRACKETS:
1922 # Empty brackets would fail a split so treat them as "inner"
1923 # brackets (e.g. only add them to the `omit` set if another
1924 # pair of brackets was good enough.
1925 inner_brackets.add(id(leaf))
1926 continue
1928 if closing_bracket:
1929 omit.add(id(closing_bracket))
1930 omit.update(inner_brackets)
1931 inner_brackets.clear()
1932 yield omit
1934 if (
1935 prev
1936 and prev.type == token.COMMA
1937 and leaf.opening_bracket is not None
1938 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1939 ):
1940 # Never omit bracket pairs with trailing commas.
1941 # We need to explode on those.
1942 break
1944 if leaf.value:
1945 opening_bracket = leaf.opening_bracket
1946 closing_bracket = leaf
1949def run_transformer(
1950 line: Line,
1951 transform: Transformer,
1952 mode: Mode,
1953 features: Collection[Feature],
1954 *,
1955 line_str: str = "",
1956) -> list[Line]:
1957 if not line_str:
1958 line_str = line_to_string(line)
1959 result: list[Line] = []
1960 for transformed_line in transform(line, features, mode):
1961 if str(transformed_line).strip("\n") == line_str:
1962 raise CannotTransform("Line transformer returned an unchanged result")
1964 result.extend(transform_line(transformed_line, mode=mode, features=features))
1966 features_set = set(features)
1967 if (
1968 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1969 or transform.__class__.__name__ != "rhs"
1970 or not line.bracket_tracker.invisible
1971 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1972 or line.contains_multiline_strings()
1973 or result[0].contains_uncollapsable_type_comments()
1974 or result[0].contains_unsplittable_type_ignore()
1975 or is_line_short_enough(result[0], mode=mode)
1976 # If any leaves have no parents (which _can_ occur since
1977 # `transform(line)` potentially destroys the line's underlying node
1978 # structure), then we can't proceed. Doing so would cause the below
1979 # call to `append_leaves()` to fail.
1980 or any(leaf.parent is None for leaf in line.leaves)
1981 ):
1982 return result
1984 line_copy = line.clone()
1985 append_leaves(line_copy, line, line.leaves)
1986 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1987 second_opinion = run_transformer(
1988 line_copy, transform, mode, features_fop, line_str=line_str
1989 )
1990 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1991 result = second_opinion
1992 return result