Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Optional, Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import FMT_OFF, FMT_ON, generate_comments, list_comments
21from black.lines import (
22 Line,
23 RHSResult,
24 append_leaves,
25 can_be_split,
26 can_omit_invisible_parens,
27 is_line_short_enough,
28 line_to_string,
29)
30from black.mode import Feature, Mode, Preview
31from black.nodes import (
32 ASSIGNMENTS,
33 BRACKETS,
34 CLOSING_BRACKETS,
35 OPENING_BRACKETS,
36 STANDALONE_COMMENT,
37 STATEMENT,
38 WHITESPACE,
39 Visitor,
40 ensure_visible,
41 fstring_to_string,
42 get_annotation_type,
43 has_sibling_with_type,
44 is_arith_like,
45 is_async_stmt_or_funcdef,
46 is_atom_with_invisible_parens,
47 is_docstring,
48 is_empty_tuple,
49 is_generator,
50 is_lpar_token,
51 is_multiline_string,
52 is_name_token,
53 is_one_sequence_between,
54 is_one_tuple,
55 is_parent_function_or_class,
56 is_part_of_annotation,
57 is_rpar_token,
58 is_stub_body,
59 is_stub_suite,
60 is_tuple,
61 is_tuple_containing_star,
62 is_tuple_containing_walrus,
63 is_type_ignore_comment_string,
64 is_vararg,
65 is_walrus_assignment,
66 is_yield,
67 syms,
68 wrap_in_parentheses,
69)
70from black.numerics import normalize_numeric_literal
71from black.strings import (
72 fix_multiline_docstring,
73 get_string_prefix,
74 normalize_string_prefix,
75 normalize_string_quotes,
76 normalize_unicode_escape_sequences,
77)
78from black.trans import (
79 CannotTransform,
80 StringMerger,
81 StringParenStripper,
82 StringParenWrapper,
83 StringSplitter,
84 Transformer,
85 hug_power_op,
86)
87from blib2to3.pgen2 import token
88from blib2to3.pytree import Leaf, Node
90# types
91LeafID = int
92LN = Union[Leaf, Node]
95class CannotSplit(CannotTransform):
96 """A readable split that fits the allotted line length is impossible."""
99# This isn't a dataclass because @dataclass + Generic breaks mypyc.
100# See also https://github.com/mypyc/mypyc/issues/827.
101class LineGenerator(Visitor[Line]):
102 """Generates reformatted Line objects. Empty lines are not emitted.
104 Note: destroys the tree it's visiting by mutating prefixes of its leaves
105 in ways that will no longer stringify to valid Python code on the tree.
106 """
108 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
109 self.mode = mode
110 self.features = features
111 self.current_line: Line
112 self.__post_init__()
114 def line(self, indent: int = 0) -> Iterator[Line]:
115 """Generate a line.
117 If the line is empty, only emit if it makes sense.
118 If the line is too long, split it first and then generate.
120 If any lines were generated, set up a new current_line.
121 """
122 if not self.current_line:
123 self.current_line.depth += indent
124 return # Line is empty, don't emit. Creating a new one unnecessary.
126 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
127 self.current_line.leaves[0]
128 ):
129 # Special case for async def/for/with statements. `visit_async_stmt`
130 # adds an `ASYNC` leaf then visits the child def/for/with statement
131 # nodes. Line yields from those nodes shouldn't treat the former
132 # `ASYNC` leaf as a complete line.
133 return
135 complete_line = self.current_line
136 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
137 yield complete_line
139 def visit_default(self, node: LN) -> Iterator[Line]:
140 """Default `visit_*()` implementation. Recurses to children of `node`."""
141 if isinstance(node, Leaf):
142 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
143 for comment in generate_comments(node, mode=self.mode):
144 if any_open_brackets:
145 # any comment within brackets is subject to splitting
146 self.current_line.append(comment)
147 elif comment.type == token.COMMENT:
148 # regular trailing comment
149 self.current_line.append(comment)
150 yield from self.line()
152 else:
153 # regular standalone comment
154 yield from self.line()
156 self.current_line.append(comment)
157 yield from self.line()
159 if any_open_brackets:
160 node.prefix = ""
161 if node.type not in WHITESPACE:
162 self.current_line.append(node)
163 yield from super().visit_default(node)
165 def visit_test(self, node: Node) -> Iterator[Line]:
166 """Visit an `x if y else z` test"""
168 already_parenthesized = (
169 node.prev_sibling and node.prev_sibling.type == token.LPAR
170 )
172 if not already_parenthesized:
173 # Similar to logic in wrap_in_parentheses
174 lpar = Leaf(token.LPAR, "")
175 rpar = Leaf(token.RPAR, "")
176 prefix = node.prefix
177 node.prefix = ""
178 lpar.prefix = prefix
179 node.insert_child(0, lpar)
180 node.append_child(rpar)
182 yield from self.visit_default(node)
184 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
185 """Increase indentation level, maybe yield a line."""
186 # In blib2to3 INDENT never holds comments.
187 yield from self.line(+1)
188 yield from self.visit_default(node)
190 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
191 """Decrease indentation level, maybe yield a line."""
192 # The current line might still wait for trailing comments. At DEDENT time
193 # there won't be any (they would be prefixes on the preceding NEWLINE).
194 # Emit the line then.
195 yield from self.line()
197 # While DEDENT has no value, its prefix may contain standalone comments
198 # that belong to the current indentation level. Get 'em.
199 yield from self.visit_default(node)
201 # Finally, emit the dedent.
202 yield from self.line(-1)
204 def visit_stmt(
205 self, node: Node, keywords: set[str], parens: set[str]
206 ) -> Iterator[Line]:
207 """Visit a statement.
209 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
210 `def`, `with`, `class`, `assert`, and assignments.
212 The relevant Python language `keywords` for a given statement will be
213 NAME leaves within it. This methods puts those on a separate line.
215 `parens` holds a set of string leaf values immediately after which
216 invisible parens should be put.
217 """
218 normalize_invisible_parens(
219 node, parens_after=parens, mode=self.mode, features=self.features
220 )
221 for child in node.children:
222 if is_name_token(child) and child.value in keywords:
223 yield from self.line()
225 yield from self.visit(child)
227 def visit_typeparams(self, node: Node) -> Iterator[Line]:
228 yield from self.visit_default(node)
229 node.children[0].prefix = ""
231 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
232 yield from self.visit_default(node)
233 node.children[1].prefix = ""
235 def visit_paramspec(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[1].prefix = ""
239 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
240 if Preview.wrap_long_dict_values_in_parens in self.mode:
241 for i, child in enumerate(node.children):
242 if i == 0:
243 continue
244 if node.children[i - 1].type == token.COLON:
245 if (
246 child.type == syms.atom
247 and child.children[0].type in OPENING_BRACKETS
248 and not is_walrus_assignment(child)
249 ):
250 maybe_make_parens_invisible_in_atom(
251 child,
252 parent=node,
253 mode=self.mode,
254 features=self.features,
255 remove_brackets_around_comma=False,
256 )
257 else:
258 wrap_in_parentheses(node, child, visible=False)
259 yield from self.visit_default(node)
261 def visit_funcdef(self, node: Node) -> Iterator[Line]:
262 """Visit function definition."""
263 yield from self.line()
265 # Remove redundant brackets around return type annotation.
266 is_return_annotation = False
267 for child in node.children:
268 if child.type == token.RARROW:
269 is_return_annotation = True
270 elif is_return_annotation:
271 if child.type == syms.atom and child.children[0].type == token.LPAR:
272 if maybe_make_parens_invisible_in_atom(
273 child,
274 parent=node,
275 mode=self.mode,
276 features=self.features,
277 remove_brackets_around_comma=False,
278 ):
279 wrap_in_parentheses(node, child, visible=False)
280 else:
281 wrap_in_parentheses(node, child, visible=False)
282 is_return_annotation = False
284 for child in node.children:
285 yield from self.visit(child)
287 def visit_match_case(self, node: Node) -> Iterator[Line]:
288 """Visit either a match or case statement."""
289 normalize_invisible_parens(
290 node, parens_after=set(), mode=self.mode, features=self.features
291 )
293 yield from self.line()
294 for child in node.children:
295 yield from self.visit(child)
297 def visit_suite(self, node: Node) -> Iterator[Line]:
298 """Visit a suite."""
299 if is_stub_suite(node):
300 yield from self.visit(node.children[2])
301 else:
302 yield from self.visit_default(node)
304 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
305 """Visit a statement without nested statements."""
306 prev_type: Optional[int] = None
307 for child in node.children:
308 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
309 wrap_in_parentheses(node, child, visible=False)
310 prev_type = child.type
312 if node.parent and node.parent.type in STATEMENT:
313 if is_parent_function_or_class(node) and is_stub_body(node):
314 yield from self.visit_default(node)
315 else:
316 yield from self.line(+1)
317 yield from self.visit_default(node)
318 yield from self.line(-1)
320 else:
321 if node.parent and is_stub_suite(node.parent):
322 node.prefix = ""
323 yield from self.visit_default(node)
324 return
325 yield from self.line()
326 yield from self.visit_default(node)
328 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
329 """Visit `async def`, `async for`, `async with`."""
330 yield from self.line()
332 children = iter(node.children)
333 for child in children:
334 yield from self.visit(child)
336 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
337 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
338 # line.
339 break
341 internal_stmt = next(children)
342 yield from self.visit(internal_stmt)
344 def visit_decorators(self, node: Node) -> Iterator[Line]:
345 """Visit decorators."""
346 for child in node.children:
347 yield from self.line()
348 yield from self.visit(child)
350 def visit_power(self, node: Node) -> Iterator[Line]:
351 for idx, leaf in enumerate(node.children[:-1]):
352 next_leaf = node.children[idx + 1]
354 if not isinstance(leaf, Leaf):
355 continue
357 value = leaf.value.lower()
358 if (
359 leaf.type == token.NUMBER
360 and next_leaf.type == syms.trailer
361 # Ensure that we are in an attribute trailer
362 and next_leaf.children[0].type == token.DOT
363 # It shouldn't wrap hexadecimal, binary and octal literals
364 and not value.startswith(("0x", "0b", "0o"))
365 # It shouldn't wrap complex literals
366 and "j" not in value
367 ):
368 wrap_in_parentheses(node, leaf)
370 remove_await_parens(node, mode=self.mode, features=self.features)
372 yield from self.visit_default(node)
374 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
375 """Remove a semicolon and put the other statement on a separate line."""
376 yield from self.line()
378 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
379 """End of file. Process outstanding comments and end with a newline."""
380 yield from self.visit_default(leaf)
381 yield from self.line()
383 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
384 if not self.current_line.bracket_tracker.any_open_brackets():
385 yield from self.line()
386 # STANDALONE_COMMENT nodes created by our special handling in
387 # normalize_fmt_off for comment-only blocks have fmt:off as the first
388 # line and fmt:on as the last line (each directive on its own line,
389 # not embedded in other text). These should be appended directly
390 # without calling visit_default, which would process their prefix and
391 # lose indentation. Normal STANDALONE_COMMENT nodes go through
392 # visit_default.
393 value = leaf.value
394 lines = value.splitlines()
395 if len(lines) >= 2:
396 # Check if first line (after stripping whitespace) is exactly a
397 # fmt:off directive
398 first_line = lines[0].lstrip()
399 first_is_fmt_off = first_line in FMT_OFF
400 # Check if last line (after stripping whitespace) is exactly a
401 # fmt:on directive
402 last_line = lines[-1].lstrip()
403 last_is_fmt_on = last_line in FMT_ON
404 is_fmt_off_block = first_is_fmt_off and last_is_fmt_on
405 else:
406 is_fmt_off_block = False
407 if is_fmt_off_block:
408 # This is a fmt:off/on block from normalize_fmt_off - append directly
409 self.current_line.append(leaf)
410 yield from self.line()
411 else:
412 # Normal standalone comment - process through visit_default
413 yield from self.visit_default(leaf)
415 def visit_factor(self, node: Node) -> Iterator[Line]:
416 """Force parentheses between a unary op and a binary power:
418 -2 ** 8 -> -(2 ** 8)
419 """
420 _operator, operand = node.children
421 if (
422 operand.type == syms.power
423 and len(operand.children) == 3
424 and operand.children[1].type == token.DOUBLESTAR
425 ):
426 lpar = Leaf(token.LPAR, "(")
427 rpar = Leaf(token.RPAR, ")")
428 index = operand.remove() or 0
429 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
430 yield from self.visit_default(node)
432 def visit_tname(self, node: Node) -> Iterator[Line]:
433 """
434 Add potential parentheses around types in function parameter lists to be made
435 into real parentheses in case the type hint is too long to fit on a line
436 Examples:
437 def foo(a: int, b: float = 7): ...
439 ->
441 def foo(a: (int), b: (float) = 7): ...
442 """
443 assert len(node.children) == 3
444 if maybe_make_parens_invisible_in_atom(
445 node.children[2], parent=node, mode=self.mode, features=self.features
446 ):
447 wrap_in_parentheses(node, node.children[2], visible=False)
449 yield from self.visit_default(node)
451 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
452 normalize_unicode_escape_sequences(leaf)
454 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
455 # We're ignoring docstrings with backslash newline escapes because changing
456 # indentation of those changes the AST representation of the code.
457 if self.mode.string_normalization:
458 docstring = normalize_string_prefix(leaf.value)
459 # We handle string normalization at the end of this method, but since
460 # what we do right now acts differently depending on quote style (ex.
461 # see padding logic below), there's a possibility for unstable
462 # formatting. To avoid a situation where this function formats a
463 # docstring differently on the second pass, normalize it early.
464 docstring = normalize_string_quotes(docstring)
465 else:
466 docstring = leaf.value
467 prefix = get_string_prefix(docstring)
468 docstring = docstring[len(prefix) :] # Remove the prefix
469 quote_char = docstring[0]
470 # A natural way to remove the outer quotes is to do:
471 # docstring = docstring.strip(quote_char)
472 # but that breaks on """""x""" (which is '""x').
473 # So we actually need to remove the first character and the next two
474 # characters but only if they are the same as the first.
475 quote_len = 1 if docstring[1] != quote_char else 3
476 docstring = docstring[quote_len:-quote_len]
477 docstring_started_empty = not docstring
478 indent = " " * 4 * self.current_line.depth
480 if is_multiline_string(leaf):
481 docstring = fix_multiline_docstring(docstring, indent)
482 else:
483 docstring = docstring.strip()
485 has_trailing_backslash = False
486 if docstring:
487 # Add some padding if the docstring starts / ends with a quote mark.
488 if docstring[0] == quote_char:
489 docstring = " " + docstring
490 if docstring[-1] == quote_char:
491 docstring += " "
492 if docstring[-1] == "\\":
493 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
494 if backslash_count % 2:
495 # Odd number of tailing backslashes, add some padding to
496 # avoid escaping the closing string quote.
497 docstring += " "
498 has_trailing_backslash = True
499 elif not docstring_started_empty:
500 docstring = " "
502 # We could enforce triple quotes at this point.
503 quote = quote_char * quote_len
505 # It's invalid to put closing single-character quotes on a new line.
506 if quote_len == 3:
507 # We need to find the length of the last line of the docstring
508 # to find if we can add the closing quotes to the line without
509 # exceeding the maximum line length.
510 # If docstring is one line, we don't put the closing quotes on a
511 # separate line because it looks ugly (#3320).
512 lines = docstring.splitlines()
513 last_line_length = len(lines[-1]) if docstring else 0
515 # If adding closing quotes would cause the last line to exceed
516 # the maximum line length, and the closing quote is not
517 # prefixed by a newline then put a line break before
518 # the closing quotes
519 if (
520 len(lines) > 1
521 and last_line_length + quote_len > self.mode.line_length
522 and len(indent) + quote_len <= self.mode.line_length
523 and not has_trailing_backslash
524 ):
525 if leaf.value[-1 - quote_len] == "\n":
526 leaf.value = prefix + quote + docstring + quote
527 else:
528 leaf.value = prefix + quote + docstring + "\n" + indent + quote
529 else:
530 leaf.value = prefix + quote + docstring + quote
531 else:
532 leaf.value = prefix + quote + docstring + quote
534 if self.mode.string_normalization and leaf.type == token.STRING:
535 leaf.value = normalize_string_prefix(leaf.value)
536 leaf.value = normalize_string_quotes(leaf.value)
537 yield from self.visit_default(leaf)
539 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
540 normalize_numeric_literal(leaf)
541 yield from self.visit_default(leaf)
543 def visit_atom(self, node: Node) -> Iterator[Line]:
544 """Visit any atom"""
545 if len(node.children) == 3:
546 first = node.children[0]
547 last = node.children[-1]
548 if (first.type == token.LSQB and last.type == token.RSQB) or (
549 first.type == token.LBRACE and last.type == token.RBRACE
550 ):
551 # Lists or sets of one item
552 maybe_make_parens_invisible_in_atom(
553 node.children[1],
554 parent=node,
555 mode=self.mode,
556 features=self.features,
557 )
559 yield from self.visit_default(node)
561 def visit_fstring(self, node: Node) -> Iterator[Line]:
562 # currently we don't want to format and split f-strings at all.
563 string_leaf = fstring_to_string(node)
564 node.replace(string_leaf)
565 if "\\" in string_leaf.value and any(
566 "\\" in str(child)
567 for child in node.children
568 if child.type == syms.fstring_replacement_field
569 ):
570 # string normalization doesn't account for nested quotes,
571 # causing breakages. skip normalization when nested quotes exist
572 yield from self.visit_default(string_leaf)
573 return
574 yield from self.visit_STRING(string_leaf)
576 # TODO: Uncomment Implementation to format f-string children
577 # fstring_start = node.children[0]
578 # fstring_end = node.children[-1]
579 # assert isinstance(fstring_start, Leaf)
580 # assert isinstance(fstring_end, Leaf)
582 # quote_char = fstring_end.value[0]
583 # quote_idx = fstring_start.value.index(quote_char)
584 # prefix, quote = (
585 # fstring_start.value[:quote_idx],
586 # fstring_start.value[quote_idx:]
587 # )
589 # if not is_docstring(node, self.mode):
590 # prefix = normalize_string_prefix(prefix)
592 # assert quote == fstring_end.value
594 # is_raw_fstring = "r" in prefix or "R" in prefix
595 # middles = [
596 # leaf
597 # for leaf in node.leaves()
598 # if leaf.type == token.FSTRING_MIDDLE
599 # ]
601 # if self.mode.string_normalization:
602 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
604 # fstring_start.value = prefix + quote
605 # fstring_end.value = quote
607 # yield from self.visit_default(node)
609 def visit_comp_for(self, node: Node) -> Iterator[Line]:
610 if Preview.wrap_comprehension_in in self.mode:
611 normalize_invisible_parens(
612 node, parens_after={"in"}, mode=self.mode, features=self.features
613 )
614 yield from self.visit_default(node)
616 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
617 yield from self.visit_comp_for(node)
619 def __post_init__(self) -> None:
620 """You are in a twisty little maze of passages."""
621 self.current_line = Line(mode=self.mode)
623 v = self.visit_stmt
624 Ø: set[str] = set()
625 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
626 self.visit_if_stmt = partial(
627 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
628 )
629 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
630 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
631 self.visit_try_stmt = partial(
632 v, keywords={"try", "except", "else", "finally"}, parens=Ø
633 )
634 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
635 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
636 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
638 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
639 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
640 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
641 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
642 self.visit_async_funcdef = self.visit_async_stmt
643 self.visit_decorated = self.visit_decorators
645 # PEP 634
646 self.visit_match_stmt = self.visit_match_case
647 self.visit_case_block = self.visit_match_case
648 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
651def _hugging_power_ops_line_to_string(
652 line: Line,
653 features: Collection[Feature],
654 mode: Mode,
655) -> Optional[str]:
656 try:
657 return line_to_string(next(hug_power_op(line, features, mode)))
658 except CannotTransform:
659 return None
662def transform_line(
663 line: Line, mode: Mode, features: Collection[Feature] = ()
664) -> Iterator[Line]:
665 """Transform a `line`, potentially splitting it into many lines.
667 They should fit in the allotted `line_length` but might not be able to.
669 `features` are syntactical features that may be used in the output.
670 """
671 if line.is_comment:
672 yield line
673 return
675 line_str = line_to_string(line)
677 # We need the line string when power operators are hugging to determine if we should
678 # split the line. Default to line_str, if no power operator are present on the line.
679 line_str_hugging_power_ops = (
680 _hugging_power_ops_line_to_string(line, features, mode) or line_str
681 )
683 ll = mode.line_length
684 sn = mode.string_normalization
685 string_merge = StringMerger(ll, sn)
686 string_paren_strip = StringParenStripper(ll, sn)
687 string_split = StringSplitter(ll, sn)
688 string_paren_wrap = StringParenWrapper(ll, sn)
690 transformers: list[Transformer]
691 if (
692 not line.contains_uncollapsable_type_comments()
693 and not line.should_split_rhs
694 and not line.magic_trailing_comma
695 and (
696 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
697 or line.contains_unsplittable_type_ignore()
698 )
699 and not (line.inside_brackets and line.contains_standalone_comments())
700 and not line.contains_implicit_multiline_string_with_comments()
701 ):
702 # Only apply basic string preprocessing, since lines shouldn't be split here.
703 if Preview.string_processing in mode:
704 transformers = [string_merge, string_paren_strip]
705 else:
706 transformers = []
707 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
708 transformers = [left_hand_split]
709 else:
711 def _rhs(
712 self: object, line: Line, features: Collection[Feature], mode: Mode
713 ) -> Iterator[Line]:
714 """Wraps calls to `right_hand_split`.
716 The calls increasingly `omit` right-hand trailers (bracket pairs with
717 content), meaning the trailers get glued together to split on another
718 bracket pair instead.
719 """
720 for omit in generate_trailers_to_omit(line, mode.line_length):
721 lines = list(right_hand_split(line, mode, features, omit=omit))
722 # Note: this check is only able to figure out if the first line of the
723 # *current* transformation fits in the line length. This is true only
724 # for simple cases. All others require running more transforms via
725 # `transform_line()`. This check doesn't know if those would succeed.
726 if is_line_short_enough(lines[0], mode=mode):
727 yield from lines
728 return
730 # All splits failed, best effort split with no omits.
731 # This mostly happens to multiline strings that are by definition
732 # reported as not fitting a single line, as well as lines that contain
733 # trailing commas (those have to be exploded).
734 yield from right_hand_split(line, mode, features=features)
736 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
737 # __name__ attribute which is needed in `run_transformer` further down.
738 # Unfortunately a nested class breaks mypyc too. So a class must be created
739 # via type ... https://github.com/mypyc/mypyc/issues/884
740 rhs = type("rhs", (), {"__call__": _rhs})()
742 if Preview.string_processing in mode:
743 if line.inside_brackets:
744 transformers = [
745 string_merge,
746 string_paren_strip,
747 string_split,
748 delimiter_split,
749 standalone_comment_split,
750 string_paren_wrap,
751 rhs,
752 ]
753 else:
754 transformers = [
755 string_merge,
756 string_paren_strip,
757 string_split,
758 string_paren_wrap,
759 rhs,
760 ]
761 else:
762 if line.inside_brackets:
763 transformers = [delimiter_split, standalone_comment_split, rhs]
764 else:
765 transformers = [rhs]
766 # It's always safe to attempt hugging of power operations and pretty much every line
767 # could match.
768 transformers.append(hug_power_op)
770 for transform in transformers:
771 # We are accumulating lines in `result` because we might want to abort
772 # mission and return the original line in the end, or attempt a different
773 # split altogether.
774 try:
775 result = run_transformer(line, transform, mode, features, line_str=line_str)
776 except CannotTransform:
777 continue
778 else:
779 yield from result
780 break
782 else:
783 yield line
786def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
787 """If a funcdef has a magic trailing comma in the return type, then we should first
788 split the line with rhs to respect the comma.
789 """
790 return_type_leaves: list[Leaf] = []
791 in_return_type = False
793 for leaf in line.leaves:
794 if leaf.type == token.COLON:
795 in_return_type = False
796 if in_return_type:
797 return_type_leaves.append(leaf)
798 if leaf.type == token.RARROW:
799 in_return_type = True
801 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
802 # couple lines from it.
803 result = Line(mode=line.mode, depth=line.depth)
804 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
805 for leaf in return_type_leaves:
806 result.append(
807 leaf,
808 preformatted=True,
809 track_bracket=id(leaf) in leaves_to_track,
810 )
812 # we could also return true if the line is too long, and the return type is longer
813 # than the param list. Or if `should_split_rhs` returns True.
814 return result.magic_trailing_comma is not None
817class _BracketSplitComponent(Enum):
818 head = auto()
819 body = auto()
820 tail = auto()
823def left_hand_split(
824 line: Line, _features: Collection[Feature], mode: Mode
825) -> Iterator[Line]:
826 """Split line into many lines, starting with the first matching bracket pair.
828 Note: this usually looks weird, only use this for function definitions.
829 Prefer RHS otherwise. This is why this function is not symmetrical with
830 :func:`right_hand_split` which also handles optional parentheses.
831 """
832 for leaf_type in [token.LPAR, token.LSQB]:
833 tail_leaves: list[Leaf] = []
834 body_leaves: list[Leaf] = []
835 head_leaves: list[Leaf] = []
836 current_leaves = head_leaves
837 matching_bracket: Optional[Leaf] = None
838 depth = 0
839 for index, leaf in enumerate(line.leaves):
840 if index == 2 and leaf.type == token.LSQB:
841 # A [ at index 2 means this is a type param, so start
842 # tracking the depth
843 depth += 1
844 elif depth > 0:
845 if leaf.type == token.LSQB:
846 depth += 1
847 elif leaf.type == token.RSQB:
848 depth -= 1
849 if (
850 current_leaves is body_leaves
851 and leaf.type in CLOSING_BRACKETS
852 and leaf.opening_bracket is matching_bracket
853 and isinstance(matching_bracket, Leaf)
854 # If the code is still on LPAR and we are inside a type
855 # param, ignore the match since this is searching
856 # for the function arguments
857 and not (leaf_type == token.LPAR and depth > 0)
858 ):
859 ensure_visible(leaf)
860 ensure_visible(matching_bracket)
861 current_leaves = tail_leaves if body_leaves else head_leaves
862 current_leaves.append(leaf)
863 if current_leaves is head_leaves:
864 if leaf.type == leaf_type and (
865 Preview.fix_type_expansion_split not in mode
866 or not (leaf_type == token.LPAR and depth > 0)
867 ):
868 matching_bracket = leaf
869 current_leaves = body_leaves
870 if matching_bracket and tail_leaves:
871 break
872 if not matching_bracket or not tail_leaves:
873 raise CannotSplit("No brackets found")
875 head = bracket_split_build_line(
876 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
877 )
878 body = bracket_split_build_line(
879 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
880 )
881 tail = bracket_split_build_line(
882 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
883 )
884 bracket_split_succeeded_or_raise(head, body, tail)
885 for result in (head, body, tail):
886 if result:
887 yield result
890def right_hand_split(
891 line: Line,
892 mode: Mode,
893 features: Collection[Feature] = (),
894 omit: Collection[LeafID] = (),
895) -> Iterator[Line]:
896 """Split line into many lines, starting with the last matching bracket pair.
898 If the split was by optional parentheses, attempt splitting without them, too.
899 `omit` is a collection of closing bracket IDs that shouldn't be considered for
900 this split.
902 Note: running this function modifies `bracket_depth` on the leaves of `line`.
903 """
904 rhs_result = _first_right_hand_split(line, omit=omit)
905 yield from _maybe_split_omitting_optional_parens(
906 rhs_result, line, mode, features=features, omit=omit
907 )
910def _first_right_hand_split(
911 line: Line,
912 omit: Collection[LeafID] = (),
913) -> RHSResult:
914 """Split the line into head, body, tail starting with the last bracket pair.
916 Note: this function should not have side effects. It's relied upon by
917 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
918 splitting on the right side of an assignment statement.
919 """
920 tail_leaves: list[Leaf] = []
921 body_leaves: list[Leaf] = []
922 head_leaves: list[Leaf] = []
923 current_leaves = tail_leaves
924 opening_bracket: Optional[Leaf] = None
925 closing_bracket: Optional[Leaf] = None
926 for leaf in reversed(line.leaves):
927 if current_leaves is body_leaves:
928 if leaf is opening_bracket:
929 current_leaves = head_leaves if body_leaves else tail_leaves
930 current_leaves.append(leaf)
931 if current_leaves is tail_leaves:
932 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
933 opening_bracket = leaf.opening_bracket
934 closing_bracket = leaf
935 current_leaves = body_leaves
936 if not (opening_bracket and closing_bracket and head_leaves):
937 # If there is no opening or closing_bracket that means the split failed and
938 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
939 # the matching `opening_bracket` wasn't available on `line` anymore.
940 raise CannotSplit("No brackets found")
942 tail_leaves.reverse()
943 body_leaves.reverse()
944 head_leaves.reverse()
946 body: Optional[Line] = None
947 if (
948 Preview.hug_parens_with_braces_and_square_brackets in line.mode
949 and tail_leaves[0].value
950 and tail_leaves[0].opening_bracket is head_leaves[-1]
951 ):
952 inner_body_leaves = list(body_leaves)
953 hugged_opening_leaves: list[Leaf] = []
954 hugged_closing_leaves: list[Leaf] = []
955 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
956 unpacking_offset: int = 1 if is_unpacking else 0
957 while (
958 len(inner_body_leaves) >= 2 + unpacking_offset
959 and inner_body_leaves[-1].type in CLOSING_BRACKETS
960 and inner_body_leaves[-1].opening_bracket
961 is inner_body_leaves[unpacking_offset]
962 ):
963 if unpacking_offset:
964 hugged_opening_leaves.append(inner_body_leaves.pop(0))
965 unpacking_offset = 0
966 hugged_opening_leaves.append(inner_body_leaves.pop(0))
967 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
969 if hugged_opening_leaves and inner_body_leaves:
970 inner_body = bracket_split_build_line(
971 inner_body_leaves,
972 line,
973 hugged_opening_leaves[-1],
974 component=_BracketSplitComponent.body,
975 )
976 if (
977 line.mode.magic_trailing_comma
978 and inner_body_leaves[-1].type == token.COMMA
979 ):
980 should_hug = True
981 else:
982 line_length = line.mode.line_length - sum(
983 len(str(leaf))
984 for leaf in hugged_opening_leaves + hugged_closing_leaves
985 )
986 if is_line_short_enough(
987 inner_body, mode=replace(line.mode, line_length=line_length)
988 ):
989 # Do not hug if it fits on a single line.
990 should_hug = False
991 else:
992 should_hug = True
993 if should_hug:
994 body_leaves = inner_body_leaves
995 head_leaves.extend(hugged_opening_leaves)
996 tail_leaves = hugged_closing_leaves + tail_leaves
997 body = inner_body # No need to re-calculate the body again later.
999 head = bracket_split_build_line(
1000 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1001 )
1002 if body is None:
1003 body = bracket_split_build_line(
1004 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1005 )
1006 tail = bracket_split_build_line(
1007 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1008 )
1009 bracket_split_succeeded_or_raise(head, body, tail)
1010 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1013def _maybe_split_omitting_optional_parens(
1014 rhs: RHSResult,
1015 line: Line,
1016 mode: Mode,
1017 features: Collection[Feature] = (),
1018 omit: Collection[LeafID] = (),
1019) -> Iterator[Line]:
1020 if (
1021 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1022 # the opening bracket is an optional paren
1023 and rhs.opening_bracket.type == token.LPAR
1024 and not rhs.opening_bracket.value
1025 # the closing bracket is an optional paren
1026 and rhs.closing_bracket.type == token.RPAR
1027 and not rhs.closing_bracket.value
1028 # it's not an import (optional parens are the only thing we can split on
1029 # in this case; attempting a split without them is a waste of time)
1030 and not line.is_import
1031 # and we can actually remove the parens
1032 and can_omit_invisible_parens(rhs, mode.line_length)
1033 ):
1034 omit = {id(rhs.closing_bracket), *omit}
1035 try:
1036 # The RHSResult Omitting Optional Parens.
1037 rhs_oop = _first_right_hand_split(line, omit=omit)
1038 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1039 yield from _maybe_split_omitting_optional_parens(
1040 rhs_oop, line, mode, features=features, omit=omit
1041 )
1042 return
1044 except CannotSplit as e:
1045 # For chained assignments we want to use the previous successful split
1046 if line.is_chained_assignment:
1047 pass
1049 elif (
1050 not can_be_split(rhs.body)
1051 and not is_line_short_enough(rhs.body, mode=mode)
1052 and not (
1053 Preview.wrap_long_dict_values_in_parens
1054 and rhs.opening_bracket.parent
1055 and rhs.opening_bracket.parent.parent
1056 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1057 )
1058 ):
1059 raise CannotSplit(
1060 "Splitting failed, body is still too long and can't be split."
1061 ) from e
1063 elif (
1064 rhs.head.contains_multiline_strings()
1065 or rhs.tail.contains_multiline_strings()
1066 ):
1067 raise CannotSplit(
1068 "The current optional pair of parentheses is bound to fail to"
1069 " satisfy the splitting algorithm because the head or the tail"
1070 " contains multiline strings which by definition never fit one"
1071 " line."
1072 ) from e
1074 ensure_visible(rhs.opening_bracket)
1075 ensure_visible(rhs.closing_bracket)
1076 for result in (rhs.head, rhs.body, rhs.tail):
1077 if result:
1078 yield result
1081def _prefer_split_rhs_oop_over_rhs(
1082 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1083) -> bool:
1084 """
1085 Returns whether we should prefer the result from a split omitting optional parens
1086 (rhs_oop) over the original (rhs).
1087 """
1088 # contains unsplittable type ignore
1089 if (
1090 rhs_oop.head.contains_unsplittable_type_ignore()
1091 or rhs_oop.body.contains_unsplittable_type_ignore()
1092 or rhs_oop.tail.contains_unsplittable_type_ignore()
1093 ):
1094 return True
1096 # Retain optional parens around dictionary values
1097 if (
1098 Preview.wrap_long_dict_values_in_parens
1099 and rhs.opening_bracket.parent
1100 and rhs.opening_bracket.parent.parent
1101 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1102 and rhs.body.bracket_tracker.delimiters
1103 ):
1104 # Unless the split is inside the key
1105 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1107 # the split is right after `=`
1108 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1109 return True
1111 # the left side of assignment contains brackets
1112 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1113 return True
1115 # the left side of assignment is short enough (the -1 is for the ending optional
1116 # paren)
1117 if not is_line_short_enough(
1118 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1119 ):
1120 return True
1122 # the left side of assignment won't explode further because of magic trailing comma
1123 if rhs.head.magic_trailing_comma is not None:
1124 return True
1126 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1127 # the body
1128 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1129 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1130 token.EQUAL
1131 )
1132 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1133 return False
1135 has_closing_bracket_after_assign = False
1136 for leaf in reversed(rhs_oop.head.leaves):
1137 if leaf.type == token.EQUAL:
1138 break
1139 if leaf.type in CLOSING_BRACKETS:
1140 has_closing_bracket_after_assign = True
1141 break
1142 return (
1143 # contains matching brackets after the `=` (done by checking there is a
1144 # closing bracket)
1145 has_closing_bracket_after_assign
1146 or (
1147 # the split is actually from inside the optional parens (done by checking
1148 # the first line still contains the `=`)
1149 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1150 # the first line is short enough
1151 and is_line_short_enough(rhs_oop.head, mode=mode)
1152 )
1153 )
1156def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1157 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1159 Do nothing otherwise.
1161 A left- or right-hand split is based on a pair of brackets. Content before
1162 (and including) the opening bracket is left on one line, content inside the
1163 brackets is put on a separate line, and finally content starting with and
1164 following the closing bracket is put on a separate line.
1166 Those are called `head`, `body`, and `tail`, respectively. If the split
1167 produced the same line (all content in `head`) or ended up with an empty `body`
1168 and the `tail` is just the closing bracket, then it's considered failed.
1169 """
1170 tail_len = len(str(tail).strip())
1171 if not body:
1172 if tail_len == 0:
1173 raise CannotSplit("Splitting brackets produced the same line")
1175 elif tail_len < 3:
1176 raise CannotSplit(
1177 f"Splitting brackets on an empty body to save {tail_len} characters is"
1178 " not worth it"
1179 )
1182def _ensure_trailing_comma(
1183 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1184) -> bool:
1185 if not leaves:
1186 return False
1187 # Ensure a trailing comma for imports
1188 if original.is_import:
1189 return True
1190 # ...and standalone function arguments
1191 if not original.is_def:
1192 return False
1193 if opening_bracket.value != "(":
1194 return False
1195 # Don't add commas if we already have any commas
1196 if any(
1197 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1198 ):
1199 return False
1201 # Find a leaf with a parent (comments don't have parents)
1202 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1203 if leaf_with_parent is None:
1204 return True
1205 # Don't add commas inside parenthesized return annotations
1206 if get_annotation_type(leaf_with_parent) == "return":
1207 return False
1208 # Don't add commas inside PEP 604 unions
1209 if (
1210 leaf_with_parent.parent
1211 and leaf_with_parent.parent.next_sibling
1212 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1213 ):
1214 return False
1215 return True
1218def bracket_split_build_line(
1219 leaves: list[Leaf],
1220 original: Line,
1221 opening_bracket: Leaf,
1222 *,
1223 component: _BracketSplitComponent,
1224) -> Line:
1225 """Return a new line with given `leaves` and respective comments from `original`.
1227 If it's the head component, brackets will be tracked so trailing commas are
1228 respected.
1230 If it's the body component, the result line is one-indented inside brackets and as
1231 such has its first leaf's prefix normalized and a trailing comma added when
1232 expected.
1233 """
1234 result = Line(mode=original.mode, depth=original.depth)
1235 if component is _BracketSplitComponent.body:
1236 result.inside_brackets = True
1237 result.depth += 1
1238 if _ensure_trailing_comma(leaves, original, opening_bracket):
1239 for i in range(len(leaves) - 1, -1, -1):
1240 if leaves[i].type == STANDALONE_COMMENT:
1241 continue
1243 if leaves[i].type != token.COMMA:
1244 new_comma = Leaf(token.COMMA, ",")
1245 leaves.insert(i + 1, new_comma)
1246 break
1248 leaves_to_track: set[LeafID] = set()
1249 if component is _BracketSplitComponent.head:
1250 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1251 # Populate the line
1252 for leaf in leaves:
1253 result.append(
1254 leaf,
1255 preformatted=True,
1256 track_bracket=id(leaf) in leaves_to_track,
1257 )
1258 for comment_after in original.comments_after(leaf):
1259 result.append(comment_after, preformatted=True)
1260 if component is _BracketSplitComponent.body and should_split_line(
1261 result, opening_bracket
1262 ):
1263 result.should_split_rhs = True
1264 return result
1267def dont_increase_indentation(split_func: Transformer) -> Transformer:
1268 """Normalize prefix of the first leaf in every line returned by `split_func`.
1270 This is a decorator over relevant split functions.
1271 """
1273 @wraps(split_func)
1274 def split_wrapper(
1275 line: Line, features: Collection[Feature], mode: Mode
1276 ) -> Iterator[Line]:
1277 for split_line in split_func(line, features, mode):
1278 split_line.leaves[0].prefix = ""
1279 yield split_line
1281 return split_wrapper
1284def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1285 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1286 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1287 return leaf_idx
1288 return None
1291def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1292 if is_vararg(leaf, within={syms.typedargslist}):
1293 return Feature.TRAILING_COMMA_IN_DEF in features
1294 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1295 return Feature.TRAILING_COMMA_IN_CALL in features
1296 return True
1299def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1300 if (
1301 safe
1302 and delimiter_priority == COMMA_PRIORITY
1303 and line.leaves[-1].type != token.COMMA
1304 and line.leaves[-1].type != STANDALONE_COMMENT
1305 ):
1306 new_comma = Leaf(token.COMMA, ",")
1307 line.append(new_comma)
1308 return line
1311MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1314@dont_increase_indentation
1315def delimiter_split(
1316 line: Line, features: Collection[Feature], mode: Mode
1317) -> Iterator[Line]:
1318 """Split according to delimiters of the highest priority.
1320 If the appropriate Features are given, the split will add trailing commas
1321 also in function signatures and calls that contain `*` and `**`.
1322 """
1323 if len(line.leaves) == 0:
1324 raise CannotSplit("Line empty") from None
1325 last_leaf = line.leaves[-1]
1327 bt = line.bracket_tracker
1328 try:
1329 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1330 except ValueError:
1331 raise CannotSplit("No delimiters found") from None
1333 if (
1334 delimiter_priority == DOT_PRIORITY
1335 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1336 ):
1337 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1339 current_line = Line(
1340 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1341 )
1342 lowest_depth = sys.maxsize
1343 trailing_comma_safe = True
1345 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1346 """Append `leaf` to current line or to new line if appending impossible."""
1347 nonlocal current_line
1348 try:
1349 current_line.append_safe(leaf, preformatted=True)
1350 except ValueError:
1351 yield current_line
1353 current_line = Line(
1354 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1355 )
1356 current_line.append(leaf)
1358 def append_comments(leaf: Leaf) -> Iterator[Line]:
1359 for comment_after in line.comments_after(leaf):
1360 yield from append_to_line(comment_after)
1362 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1363 for leaf_idx, leaf in enumerate(line.leaves):
1364 yield from append_to_line(leaf)
1366 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1367 id(line.leaves[leaf_idx - 1])
1368 )
1369 if (
1370 previous_priority != delimiter_priority
1371 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1372 ):
1373 yield from append_comments(leaf)
1375 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1376 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1377 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1379 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1380 current_line = _safe_add_trailing_comma(
1381 trailing_comma_safe, delimiter_priority, current_line
1382 )
1384 leaf_priority = bt.delimiters.get(id(leaf))
1385 if leaf_priority == delimiter_priority:
1386 if (
1387 leaf_idx + 1 < len(line.leaves)
1388 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1389 ):
1390 yield from append_comments(line.leaves[leaf_idx + 1])
1392 yield current_line
1393 current_line = Line(
1394 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1395 )
1397 if current_line:
1398 current_line = _safe_add_trailing_comma(
1399 trailing_comma_safe, delimiter_priority, current_line
1400 )
1401 yield current_line
1404@dont_increase_indentation
1405def standalone_comment_split(
1406 line: Line, features: Collection[Feature], mode: Mode
1407) -> Iterator[Line]:
1408 """Split standalone comments from the rest of the line."""
1409 if not line.contains_standalone_comments():
1410 raise CannotSplit("Line does not have any standalone comments")
1412 current_line = Line(
1413 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1414 )
1416 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1417 """Append `leaf` to current line or to new line if appending impossible."""
1418 nonlocal current_line
1419 try:
1420 current_line.append_safe(leaf, preformatted=True)
1421 except ValueError:
1422 yield current_line
1424 current_line = Line(
1425 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1426 )
1427 current_line.append(leaf)
1429 for leaf in line.leaves:
1430 yield from append_to_line(leaf)
1432 for comment_after in line.comments_after(leaf):
1433 yield from append_to_line(comment_after)
1435 if current_line:
1436 yield current_line
1439def normalize_invisible_parens( # noqa: C901
1440 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1441) -> None:
1442 """Make existing optional parentheses invisible or create new ones.
1444 `parens_after` is a set of string leaf values immediately after which parens
1445 should be put.
1447 Standardizes on visible parentheses for single-element tuples, and keeps
1448 existing visible parentheses for other tuples and generator expressions.
1449 """
1450 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1451 if pc.value in FMT_OFF:
1452 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1453 return
1455 # The multiple context managers grammar has a different pattern, thus this is
1456 # separate from the for-loop below. This possibly wraps them in invisible parens,
1457 # and later will be removed in remove_with_parens when needed.
1458 if node.type == syms.with_stmt:
1459 _maybe_wrap_cms_in_parens(node, mode, features)
1461 check_lpar = False
1462 for index, child in enumerate(list(node.children)):
1463 # Fixes a bug where invisible parens are not properly stripped from
1464 # assignment statements that contain type annotations.
1465 if isinstance(child, Node) and child.type == syms.annassign:
1466 normalize_invisible_parens(
1467 child, parens_after=parens_after, mode=mode, features=features
1468 )
1470 # Fixes a bug where invisible parens are not properly wrapped around
1471 # case blocks.
1472 if isinstance(child, Node) and child.type == syms.case_block:
1473 normalize_invisible_parens(
1474 child, parens_after={"case"}, mode=mode, features=features
1475 )
1477 # Add parentheses around if guards in case blocks
1478 if isinstance(child, Node) and child.type == syms.guard:
1479 normalize_invisible_parens(
1480 child, parens_after={"if"}, mode=mode, features=features
1481 )
1483 # Add parentheses around long tuple unpacking in assignments.
1484 if (
1485 index == 0
1486 and isinstance(child, Node)
1487 and child.type == syms.testlist_star_expr
1488 ):
1489 check_lpar = True
1491 if check_lpar:
1492 if (
1493 child.type == syms.atom
1494 and node.type == syms.for_stmt
1495 and isinstance(child.prev_sibling, Leaf)
1496 and child.prev_sibling.type == token.NAME
1497 and child.prev_sibling.value == "for"
1498 ):
1499 if maybe_make_parens_invisible_in_atom(
1500 child,
1501 parent=node,
1502 mode=mode,
1503 features=features,
1504 remove_brackets_around_comma=True,
1505 ):
1506 wrap_in_parentheses(node, child, visible=False)
1507 elif isinstance(child, Node) and node.type == syms.with_stmt:
1508 remove_with_parens(child, node, mode=mode, features=features)
1509 elif child.type == syms.atom and not (
1510 "in" in parens_after
1511 and len(child.children) == 3
1512 and is_lpar_token(child.children[0])
1513 and is_rpar_token(child.children[-1])
1514 and child.children[1].type == syms.test
1515 ):
1516 if maybe_make_parens_invisible_in_atom(
1517 child, parent=node, mode=mode, features=features
1518 ):
1519 wrap_in_parentheses(node, child, visible=False)
1520 elif is_one_tuple(child):
1521 wrap_in_parentheses(node, child, visible=True)
1522 elif node.type == syms.import_from:
1523 _normalize_import_from(node, child, index)
1524 break
1525 elif (
1526 index == 1
1527 and child.type == token.STAR
1528 and node.type == syms.except_clause
1529 ):
1530 # In except* (PEP 654), the star is actually part of
1531 # of the keyword. So we need to skip the insertion of
1532 # invisible parentheses to work more precisely.
1533 continue
1535 elif (
1536 isinstance(child, Leaf)
1537 and child.next_sibling is not None
1538 and child.next_sibling.type == token.COLON
1539 and child.value == "case"
1540 ):
1541 # A special patch for "case case:" scenario, the second occurrence
1542 # of case will be not parsed as a Python keyword.
1543 break
1545 elif not is_multiline_string(child):
1546 wrap_in_parentheses(node, child, visible=False)
1548 comma_check = child.type == token.COMMA
1550 check_lpar = isinstance(child, Leaf) and (
1551 child.value in parens_after or comma_check
1552 )
1555def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1556 # "import from" nodes store parentheses directly as part of
1557 # the statement
1558 if is_lpar_token(child):
1559 assert is_rpar_token(parent.children[-1])
1560 # make parentheses invisible
1561 child.value = ""
1562 parent.children[-1].value = ""
1563 elif child.type != token.STAR:
1564 # insert invisible parentheses
1565 parent.insert_child(index, Leaf(token.LPAR, ""))
1566 parent.append_child(Leaf(token.RPAR, ""))
1569def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1570 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1571 if (
1572 node.children[1].type == syms.atom
1573 and node.children[1].children[0].type == token.LPAR
1574 ):
1575 if maybe_make_parens_invisible_in_atom(
1576 node.children[1],
1577 parent=node,
1578 mode=mode,
1579 features=features,
1580 remove_brackets_around_comma=True,
1581 ):
1582 wrap_in_parentheses(node, node.children[1], visible=False)
1584 # Since await is an expression we shouldn't remove
1585 # brackets in cases where this would change
1586 # the AST due to operator precedence.
1587 # Therefore we only aim to remove brackets around
1588 # power nodes that aren't also await expressions themselves.
1589 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1590 # N.B. We've still removed any redundant nested brackets though :)
1591 opening_bracket = cast(Leaf, node.children[1].children[0])
1592 closing_bracket = cast(Leaf, node.children[1].children[-1])
1593 bracket_contents = node.children[1].children[1]
1594 if isinstance(bracket_contents, Node) and (
1595 bracket_contents.type != syms.power
1596 or bracket_contents.children[0].type == token.AWAIT
1597 or any(
1598 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1599 for child in bracket_contents.children
1600 )
1601 ):
1602 ensure_visible(opening_bracket)
1603 ensure_visible(closing_bracket)
1606def _maybe_wrap_cms_in_parens(
1607 node: Node, mode: Mode, features: Collection[Feature]
1608) -> None:
1609 """When enabled and safe, wrap the multiple context managers in invisible parens.
1611 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1612 """
1613 if (
1614 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1615 or len(node.children) <= 2
1616 # If it's an atom, it's already wrapped in parens.
1617 or node.children[1].type == syms.atom
1618 ):
1619 return
1620 colon_index: Optional[int] = None
1621 for i in range(2, len(node.children)):
1622 if node.children[i].type == token.COLON:
1623 colon_index = i
1624 break
1625 if colon_index is not None:
1626 lpar = Leaf(token.LPAR, "")
1627 rpar = Leaf(token.RPAR, "")
1628 context_managers = node.children[1:colon_index]
1629 for child in context_managers:
1630 child.remove()
1631 # After wrapping, the with_stmt will look like this:
1632 # with_stmt
1633 # NAME 'with'
1634 # atom
1635 # LPAR ''
1636 # testlist_gexp
1637 # ... <-- context_managers
1638 # /testlist_gexp
1639 # RPAR ''
1640 # /atom
1641 # COLON ':'
1642 new_child = Node(
1643 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1644 )
1645 node.insert_child(1, new_child)
1648def remove_with_parens(
1649 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1650) -> None:
1651 """Recursively hide optional parens in `with` statements."""
1652 # Removing all unnecessary parentheses in with statements in one pass is a tad
1653 # complex as different variations of bracketed statements result in pretty
1654 # different parse trees:
1655 #
1656 # with (open("file")) as f: # this is an asexpr_test
1657 # ...
1658 #
1659 # with (open("file") as f): # this is an atom containing an
1660 # ... # asexpr_test
1661 #
1662 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1663 # ... # asexpr_test
1664 #
1665 # with (open("file") as f, open("file") as f): # an atom containing a
1666 # ... # testlist_gexp which then
1667 # # contains multiple asexpr_test(s)
1668 if node.type == syms.atom:
1669 if maybe_make_parens_invisible_in_atom(
1670 node,
1671 parent=parent,
1672 mode=mode,
1673 features=features,
1674 remove_brackets_around_comma=True,
1675 ):
1676 wrap_in_parentheses(parent, node, visible=False)
1677 if isinstance(node.children[1], Node):
1678 remove_with_parens(node.children[1], node, mode=mode, features=features)
1679 elif node.type == syms.testlist_gexp:
1680 for child in node.children:
1681 if isinstance(child, Node):
1682 remove_with_parens(child, node, mode=mode, features=features)
1683 elif node.type == syms.asexpr_test and not any(
1684 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1685 ):
1686 if maybe_make_parens_invisible_in_atom(
1687 node.children[0],
1688 parent=node,
1689 mode=mode,
1690 features=features,
1691 remove_brackets_around_comma=True,
1692 ):
1693 wrap_in_parentheses(node, node.children[0], visible=False)
1696def maybe_make_parens_invisible_in_atom(
1697 node: LN,
1698 parent: LN,
1699 mode: Mode,
1700 features: Collection[Feature],
1701 remove_brackets_around_comma: bool = False,
1702) -> bool:
1703 """If it's safe, make the parens in the atom `node` invisible, recursively.
1704 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1705 as they are redundant.
1707 Returns whether the node should itself be wrapped in invisible parentheses.
1708 """
1709 if (
1710 node.type not in (syms.atom, syms.expr)
1711 or is_empty_tuple(node)
1712 or is_one_tuple(node)
1713 or (is_tuple(node) and parent.type == syms.asexpr_test)
1714 or (
1715 is_tuple(node)
1716 and parent.type == syms.with_stmt
1717 and has_sibling_with_type(node, token.COMMA)
1718 )
1719 or (is_yield(node) and parent.type != syms.expr_stmt)
1720 or (
1721 # This condition tries to prevent removing non-optional brackets
1722 # around a tuple, however, can be a bit overzealous so we provide
1723 # and option to skip this check for `for` and `with` statements.
1724 not remove_brackets_around_comma
1725 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1726 # Skip this check in Preview mode in order to
1727 # Remove parentheses around multiple exception types in except and
1728 # except* without as. See PEP 758 for details.
1729 and not (
1730 Preview.remove_parens_around_except_types in mode
1731 and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1732 # is a tuple
1733 and is_tuple(node)
1734 # has a parent node
1735 and node.parent is not None
1736 # parent is an except clause
1737 and node.parent.type == syms.except_clause
1738 # is not immediately followed by as clause
1739 and not (
1740 node.next_sibling is not None
1741 and is_name_token(node.next_sibling)
1742 and node.next_sibling.value == "as"
1743 )
1744 )
1745 )
1746 or is_tuple_containing_walrus(node)
1747 or is_tuple_containing_star(node)
1748 or is_generator(node)
1749 ):
1750 return False
1752 if is_walrus_assignment(node):
1753 if parent.type in [
1754 syms.annassign,
1755 syms.expr_stmt,
1756 syms.assert_stmt,
1757 syms.return_stmt,
1758 syms.except_clause,
1759 syms.funcdef,
1760 syms.with_stmt,
1761 syms.testlist_gexp,
1762 syms.tname,
1763 # these ones aren't useful to end users, but they do please fuzzers
1764 syms.for_stmt,
1765 syms.del_stmt,
1766 syms.for_stmt,
1767 ]:
1768 return False
1770 first = node.children[0]
1771 last = node.children[-1]
1772 if is_lpar_token(first) and is_rpar_token(last):
1773 middle = node.children[1]
1774 # make parentheses invisible
1775 if (
1776 # If the prefix of `middle` includes a type comment with
1777 # ignore annotation, then we do not remove the parentheses
1778 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
1779 ):
1780 first.value = ""
1781 last.value = ""
1782 maybe_make_parens_invisible_in_atom(
1783 middle,
1784 parent=parent,
1785 mode=mode,
1786 features=features,
1787 remove_brackets_around_comma=remove_brackets_around_comma,
1788 )
1790 if is_atom_with_invisible_parens(middle):
1791 # Strip the invisible parens from `middle` by replacing
1792 # it with the child in-between the invisible parens
1793 middle.replace(middle.children[1])
1795 if middle.children[0].prefix.strip():
1796 # Preserve comments before first paren
1797 middle.children[1].prefix = (
1798 middle.children[0].prefix + middle.children[1].prefix
1799 )
1801 if middle.children[-1].prefix.strip():
1802 # Preserve comments before last paren
1803 last.prefix = middle.children[-1].prefix + last.prefix
1805 return False
1807 return True
1810def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1811 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1813 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1814 return False
1816 # We're essentially checking if the body is delimited by commas and there's more
1817 # than one of them (we're excluding the trailing comma and if the delimiter priority
1818 # is still commas, that means there's more).
1819 exclude = set()
1820 trailing_comma = False
1821 try:
1822 last_leaf = line.leaves[-1]
1823 if last_leaf.type == token.COMMA:
1824 trailing_comma = True
1825 exclude.add(id(last_leaf))
1826 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1827 except (IndexError, ValueError):
1828 return False
1830 return max_priority == COMMA_PRIORITY and (
1831 (line.mode.magic_trailing_comma and trailing_comma)
1832 # always explode imports
1833 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1834 )
1837def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1838 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1840 Brackets can be omitted if the entire trailer up to and including
1841 a preceding closing bracket fits in one line.
1843 Yielded sets are cumulative (contain results of previous yields, too). First
1844 set is empty, unless the line should explode, in which case bracket pairs until
1845 the one that needs to explode are omitted.
1846 """
1848 omit: set[LeafID] = set()
1849 if not line.magic_trailing_comma:
1850 yield omit
1852 length = 4 * line.depth
1853 opening_bracket: Optional[Leaf] = None
1854 closing_bracket: Optional[Leaf] = None
1855 inner_brackets: set[LeafID] = set()
1856 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1857 length += leaf_length
1858 if length > line_length:
1859 break
1861 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1862 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1863 break
1865 if opening_bracket:
1866 if leaf is opening_bracket:
1867 opening_bracket = None
1868 elif leaf.type in CLOSING_BRACKETS:
1869 prev = line.leaves[index - 1] if index > 0 else None
1870 if (
1871 prev
1872 and prev.type == token.COMMA
1873 and leaf.opening_bracket is not None
1874 and not is_one_sequence_between(
1875 leaf.opening_bracket, leaf, line.leaves
1876 )
1877 ):
1878 # Never omit bracket pairs with trailing commas.
1879 # We need to explode on those.
1880 break
1882 inner_brackets.add(id(leaf))
1883 elif leaf.type in CLOSING_BRACKETS:
1884 prev = line.leaves[index - 1] if index > 0 else None
1885 if prev and prev.type in OPENING_BRACKETS:
1886 # Empty brackets would fail a split so treat them as "inner"
1887 # brackets (e.g. only add them to the `omit` set if another
1888 # pair of brackets was good enough.
1889 inner_brackets.add(id(leaf))
1890 continue
1892 if closing_bracket:
1893 omit.add(id(closing_bracket))
1894 omit.update(inner_brackets)
1895 inner_brackets.clear()
1896 yield omit
1898 if (
1899 prev
1900 and prev.type == token.COMMA
1901 and leaf.opening_bracket is not None
1902 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1903 ):
1904 # Never omit bracket pairs with trailing commas.
1905 # We need to explode on those.
1906 break
1908 if leaf.value:
1909 opening_bracket = leaf.opening_bracket
1910 closing_bracket = leaf
1913def run_transformer(
1914 line: Line,
1915 transform: Transformer,
1916 mode: Mode,
1917 features: Collection[Feature],
1918 *,
1919 line_str: str = "",
1920) -> list[Line]:
1921 if not line_str:
1922 line_str = line_to_string(line)
1923 result: list[Line] = []
1924 for transformed_line in transform(line, features, mode):
1925 if str(transformed_line).strip("\n") == line_str:
1926 raise CannotTransform("Line transformer returned an unchanged result")
1928 result.extend(transform_line(transformed_line, mode=mode, features=features))
1930 features_set = set(features)
1931 if (
1932 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1933 or transform.__class__.__name__ != "rhs"
1934 or not line.bracket_tracker.invisible
1935 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1936 or line.contains_multiline_strings()
1937 or result[0].contains_uncollapsable_type_comments()
1938 or result[0].contains_unsplittable_type_ignore()
1939 or is_line_short_enough(result[0], mode=mode)
1940 # If any leaves have no parents (which _can_ occur since
1941 # `transform(line)` potentially destroys the line's underlying node
1942 # structure), then we can't proceed. Doing so would cause the below
1943 # call to `append_leaves()` to fail.
1944 or any(leaf.parent is None for leaf in line.leaves)
1945 ):
1946 return result
1948 line_copy = line.clone()
1949 append_leaves(line_copy, line, line.leaves)
1950 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1951 second_opinion = run_transformer(
1952 line_copy, transform, mode, features_fop, line_str=line_str
1953 )
1954 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1955 result = second_opinion
1956 return result