Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/linegen.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Generating lines of code.
3"""
5import re
6import sys
7from collections.abc import Collection, Iterator
8from dataclasses import replace
9from enum import Enum, auto
10from functools import partial, wraps
11from typing import Optional, Union, cast
13from black.brackets import (
14 COMMA_PRIORITY,
15 DOT_PRIORITY,
16 STRING_PRIORITY,
17 get_leaves_inside_matching_brackets,
18 max_delimiter_priority_in_atom,
19)
20from black.comments import FMT_OFF, FMT_ON, generate_comments, list_comments
21from black.lines import (
22 Line,
23 RHSResult,
24 append_leaves,
25 can_be_split,
26 can_omit_invisible_parens,
27 is_line_short_enough,
28 line_to_string,
29)
30from black.mode import Feature, Mode, Preview
31from black.nodes import (
32 ASSIGNMENTS,
33 BRACKETS,
34 CLOSING_BRACKETS,
35 OPENING_BRACKETS,
36 STANDALONE_COMMENT,
37 STATEMENT,
38 WHITESPACE,
39 Visitor,
40 ensure_visible,
41 fstring_tstring_to_string,
42 get_annotation_type,
43 has_sibling_with_type,
44 is_arith_like,
45 is_async_stmt_or_funcdef,
46 is_atom_with_invisible_parens,
47 is_docstring,
48 is_empty_tuple,
49 is_generator,
50 is_lpar_token,
51 is_multiline_string,
52 is_name_token,
53 is_one_sequence_between,
54 is_one_tuple,
55 is_parent_function_or_class,
56 is_part_of_annotation,
57 is_rpar_token,
58 is_stub_body,
59 is_stub_suite,
60 is_tuple,
61 is_tuple_containing_star,
62 is_tuple_containing_walrus,
63 is_type_ignore_comment_string,
64 is_vararg,
65 is_walrus_assignment,
66 is_yield,
67 syms,
68 wrap_in_parentheses,
69)
70from black.numerics import normalize_numeric_literal
71from black.strings import (
72 fix_multiline_docstring,
73 get_string_prefix,
74 normalize_string_prefix,
75 normalize_string_quotes,
76 normalize_unicode_escape_sequences,
77)
78from black.trans import (
79 CannotTransform,
80 StringMerger,
81 StringParenStripper,
82 StringParenWrapper,
83 StringSplitter,
84 Transformer,
85 hug_power_op,
86)
87from blib2to3.pgen2 import token
88from blib2to3.pytree import Leaf, Node
90# types
91LeafID = int
92LN = Union[Leaf, Node]
95class CannotSplit(CannotTransform):
96 """A readable split that fits the allotted line length is impossible."""
99# This isn't a dataclass because @dataclass + Generic breaks mypyc.
100# See also https://github.com/mypyc/mypyc/issues/827.
101class LineGenerator(Visitor[Line]):
102 """Generates reformatted Line objects. Empty lines are not emitted.
104 Note: destroys the tree it's visiting by mutating prefixes of its leaves
105 in ways that will no longer stringify to valid Python code on the tree.
106 """
108 def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
109 self.mode = mode
110 self.features = features
111 self.current_line: Line
112 self.__post_init__()
114 def line(self, indent: int = 0) -> Iterator[Line]:
115 """Generate a line.
117 If the line is empty, only emit if it makes sense.
118 If the line is too long, split it first and then generate.
120 If any lines were generated, set up a new current_line.
121 """
122 if not self.current_line:
123 self.current_line.depth += indent
124 return # Line is empty, don't emit. Creating a new one unnecessary.
126 if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(
127 self.current_line.leaves[0]
128 ):
129 # Special case for async def/for/with statements. `visit_async_stmt`
130 # adds an `ASYNC` leaf then visits the child def/for/with statement
131 # nodes. Line yields from those nodes shouldn't treat the former
132 # `ASYNC` leaf as a complete line.
133 return
135 complete_line = self.current_line
136 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
137 yield complete_line
139 def visit_default(self, node: LN) -> Iterator[Line]:
140 """Default `visit_*()` implementation. Recurses to children of `node`."""
141 if isinstance(node, Leaf):
142 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
143 for comment in generate_comments(node, mode=self.mode):
144 if any_open_brackets:
145 # any comment within brackets is subject to splitting
146 self.current_line.append(comment)
147 elif comment.type == token.COMMENT:
148 # regular trailing comment
149 self.current_line.append(comment)
150 yield from self.line()
152 else:
153 # regular standalone comment
154 yield from self.line()
156 self.current_line.append(comment)
157 yield from self.line()
159 if any_open_brackets:
160 node.prefix = ""
161 if node.type not in WHITESPACE:
162 self.current_line.append(node)
163 yield from super().visit_default(node)
165 def visit_test(self, node: Node) -> Iterator[Line]:
166 """Visit an `x if y else z` test"""
168 already_parenthesized = (
169 node.prev_sibling and node.prev_sibling.type == token.LPAR
170 )
172 if not already_parenthesized:
173 # Similar to logic in wrap_in_parentheses
174 lpar = Leaf(token.LPAR, "")
175 rpar = Leaf(token.RPAR, "")
176 prefix = node.prefix
177 node.prefix = ""
178 lpar.prefix = prefix
179 node.insert_child(0, lpar)
180 node.append_child(rpar)
182 yield from self.visit_default(node)
184 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
185 """Increase indentation level, maybe yield a line."""
186 # In blib2to3 INDENT never holds comments.
187 yield from self.line(+1)
188 yield from self.visit_default(node)
190 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
191 """Decrease indentation level, maybe yield a line."""
192 # The current line might still wait for trailing comments. At DEDENT time
193 # there won't be any (they would be prefixes on the preceding NEWLINE).
194 # Emit the line then.
195 yield from self.line()
197 # While DEDENT has no value, its prefix may contain standalone comments
198 # that belong to the current indentation level. Get 'em.
199 yield from self.visit_default(node)
201 # Finally, emit the dedent.
202 yield from self.line(-1)
204 def visit_stmt(
205 self, node: Node, keywords: set[str], parens: set[str]
206 ) -> Iterator[Line]:
207 """Visit a statement.
209 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
210 `def`, `with`, `class`, `assert`, and assignments.
212 The relevant Python language `keywords` for a given statement will be
213 NAME leaves within it. This methods puts those on a separate line.
215 `parens` holds a set of string leaf values immediately after which
216 invisible parens should be put.
217 """
218 normalize_invisible_parens(
219 node, parens_after=parens, mode=self.mode, features=self.features
220 )
221 for child in node.children:
222 if is_name_token(child) and child.value in keywords:
223 yield from self.line()
225 yield from self.visit(child)
227 def visit_typeparams(self, node: Node) -> Iterator[Line]:
228 yield from self.visit_default(node)
229 node.children[0].prefix = ""
231 def visit_typevartuple(self, node: Node) -> Iterator[Line]:
232 yield from self.visit_default(node)
233 node.children[1].prefix = ""
235 def visit_paramspec(self, node: Node) -> Iterator[Line]:
236 yield from self.visit_default(node)
237 node.children[1].prefix = ""
239 def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
240 if Preview.wrap_long_dict_values_in_parens in self.mode:
241 for i, child in enumerate(node.children):
242 if i == 0:
243 continue
244 if node.children[i - 1].type == token.COLON:
245 if (
246 child.type == syms.atom
247 and child.children[0].type in OPENING_BRACKETS
248 and not is_walrus_assignment(child)
249 ):
250 maybe_make_parens_invisible_in_atom(
251 child,
252 parent=node,
253 mode=self.mode,
254 features=self.features,
255 remove_brackets_around_comma=False,
256 )
257 else:
258 wrap_in_parentheses(node, child, visible=False)
259 yield from self.visit_default(node)
261 def visit_funcdef(self, node: Node) -> Iterator[Line]:
262 """Visit function definition."""
263 yield from self.line()
265 # Remove redundant brackets around return type annotation.
266 is_return_annotation = False
267 for child in node.children:
268 if child.type == token.RARROW:
269 is_return_annotation = True
270 elif is_return_annotation:
271 if child.type == syms.atom and child.children[0].type == token.LPAR:
272 if maybe_make_parens_invisible_in_atom(
273 child,
274 parent=node,
275 mode=self.mode,
276 features=self.features,
277 remove_brackets_around_comma=False,
278 ):
279 wrap_in_parentheses(node, child, visible=False)
280 else:
281 wrap_in_parentheses(node, child, visible=False)
282 is_return_annotation = False
284 for child in node.children:
285 yield from self.visit(child)
287 def visit_match_case(self, node: Node) -> Iterator[Line]:
288 """Visit either a match or case statement."""
289 normalize_invisible_parens(
290 node, parens_after=set(), mode=self.mode, features=self.features
291 )
293 yield from self.line()
294 for child in node.children:
295 yield from self.visit(child)
297 def visit_suite(self, node: Node) -> Iterator[Line]:
298 """Visit a suite."""
299 if is_stub_suite(node):
300 yield from self.visit(node.children[2])
301 else:
302 yield from self.visit_default(node)
304 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
305 """Visit a statement without nested statements."""
306 prev_type: Optional[int] = None
307 for child in node.children:
308 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
309 wrap_in_parentheses(node, child, visible=False)
310 prev_type = child.type
312 if node.parent and node.parent.type in STATEMENT:
313 if is_parent_function_or_class(node) and is_stub_body(node):
314 yield from self.visit_default(node)
315 else:
316 yield from self.line(+1)
317 yield from self.visit_default(node)
318 yield from self.line(-1)
320 else:
321 if node.parent and is_stub_suite(node.parent):
322 node.prefix = ""
323 yield from self.visit_default(node)
324 return
325 yield from self.line()
326 yield from self.visit_default(node)
328 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
329 """Visit `async def`, `async for`, `async with`."""
330 yield from self.line()
332 children = iter(node.children)
333 for child in children:
334 yield from self.visit(child)
336 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
337 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
338 # line.
339 break
341 internal_stmt = next(children)
342 yield from self.visit(internal_stmt)
344 def visit_decorators(self, node: Node) -> Iterator[Line]:
345 """Visit decorators."""
346 for child in node.children:
347 yield from self.line()
348 yield from self.visit(child)
350 def visit_power(self, node: Node) -> Iterator[Line]:
351 for idx, leaf in enumerate(node.children[:-1]):
352 next_leaf = node.children[idx + 1]
354 if not isinstance(leaf, Leaf):
355 continue
357 value = leaf.value.lower()
358 if (
359 leaf.type == token.NUMBER
360 and next_leaf.type == syms.trailer
361 # Ensure that we are in an attribute trailer
362 and next_leaf.children[0].type == token.DOT
363 # It shouldn't wrap hexadecimal, binary and octal literals
364 and not value.startswith(("0x", "0b", "0o"))
365 # It shouldn't wrap complex literals
366 and "j" not in value
367 ):
368 wrap_in_parentheses(node, leaf)
370 remove_await_parens(node, mode=self.mode, features=self.features)
372 yield from self.visit_default(node)
374 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
375 """Remove a semicolon and put the other statement on a separate line."""
376 yield from self.line()
378 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
379 """End of file. Process outstanding comments and end with a newline."""
380 yield from self.visit_default(leaf)
381 yield from self.line()
383 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
384 if not self.current_line.bracket_tracker.any_open_brackets():
385 yield from self.line()
386 # STANDALONE_COMMENT nodes created by our special handling in
387 # normalize_fmt_off for comment-only blocks have fmt:off as the first
388 # line and fmt:on as the last line (each directive on its own line,
389 # not embedded in other text). These should be appended directly
390 # without calling visit_default, which would process their prefix and
391 # lose indentation. Normal STANDALONE_COMMENT nodes go through
392 # visit_default.
393 value = leaf.value
394 lines = value.splitlines()
395 if len(lines) >= 2:
396 # Check if first line (after stripping whitespace) is exactly a
397 # fmt:off directive
398 first_line = lines[0].lstrip()
399 first_is_fmt_off = first_line in FMT_OFF
400 # Check if last line (after stripping whitespace) is exactly a
401 # fmt:on directive
402 last_line = lines[-1].lstrip()
403 last_is_fmt_on = last_line in FMT_ON
404 is_fmt_off_block = first_is_fmt_off and last_is_fmt_on
405 else:
406 is_fmt_off_block = False
407 if is_fmt_off_block:
408 # This is a fmt:off/on block from normalize_fmt_off - append directly
409 self.current_line.append(leaf)
410 yield from self.line()
411 else:
412 # Normal standalone comment - process through visit_default
413 yield from self.visit_default(leaf)
415 def visit_factor(self, node: Node) -> Iterator[Line]:
416 """Force parentheses between a unary op and a binary power:
418 -2 ** 8 -> -(2 ** 8)
419 """
420 _operator, operand = node.children
421 if (
422 operand.type == syms.power
423 and len(operand.children) == 3
424 and operand.children[1].type == token.DOUBLESTAR
425 ):
426 lpar = Leaf(token.LPAR, "(")
427 rpar = Leaf(token.RPAR, ")")
428 index = operand.remove() or 0
429 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
430 yield from self.visit_default(node)
432 def visit_tname(self, node: Node) -> Iterator[Line]:
433 """
434 Add potential parentheses around types in function parameter lists to be made
435 into real parentheses in case the type hint is too long to fit on a line
436 Examples:
437 def foo(a: int, b: float = 7): ...
439 ->
441 def foo(a: (int), b: (float) = 7): ...
442 """
443 assert len(node.children) == 3
444 if maybe_make_parens_invisible_in_atom(
445 node.children[2], parent=node, mode=self.mode, features=self.features
446 ):
447 wrap_in_parentheses(node, node.children[2], visible=False)
449 yield from self.visit_default(node)
451 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
452 normalize_unicode_escape_sequences(leaf)
454 if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value):
455 # We're ignoring docstrings with backslash newline escapes because changing
456 # indentation of those changes the AST representation of the code.
457 if self.mode.string_normalization:
458 docstring = normalize_string_prefix(leaf.value)
459 # We handle string normalization at the end of this method, but since
460 # what we do right now acts differently depending on quote style (ex.
461 # see padding logic below), there's a possibility for unstable
462 # formatting. To avoid a situation where this function formats a
463 # docstring differently on the second pass, normalize it early.
464 docstring = normalize_string_quotes(docstring)
465 else:
466 docstring = leaf.value
467 prefix = get_string_prefix(docstring)
468 docstring = docstring[len(prefix) :] # Remove the prefix
469 quote_char = docstring[0]
470 # A natural way to remove the outer quotes is to do:
471 # docstring = docstring.strip(quote_char)
472 # but that breaks on """""x""" (which is '""x').
473 # So we actually need to remove the first character and the next two
474 # characters but only if they are the same as the first.
475 quote_len = 1 if docstring[1] != quote_char else 3
476 docstring = docstring[quote_len:-quote_len]
477 docstring_started_empty = not docstring
478 indent = " " * 4 * self.current_line.depth
480 if is_multiline_string(leaf):
481 docstring = fix_multiline_docstring(docstring, indent)
482 else:
483 docstring = docstring.strip()
485 has_trailing_backslash = False
486 if docstring:
487 # Add some padding if the docstring starts / ends with a quote mark.
488 if docstring[0] == quote_char:
489 docstring = " " + docstring
490 if docstring[-1] == quote_char:
491 docstring += " "
492 if docstring[-1] == "\\":
493 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
494 if backslash_count % 2:
495 # Odd number of tailing backslashes, add some padding to
496 # avoid escaping the closing string quote.
497 docstring += " "
498 has_trailing_backslash = True
499 elif not docstring_started_empty:
500 docstring = " "
502 # We could enforce triple quotes at this point.
503 quote = quote_char * quote_len
505 # It's invalid to put closing single-character quotes on a new line.
506 if quote_len == 3:
507 # We need to find the length of the last line of the docstring
508 # to find if we can add the closing quotes to the line without
509 # exceeding the maximum line length.
510 # If docstring is one line, we don't put the closing quotes on a
511 # separate line because it looks ugly (#3320).
512 lines = docstring.splitlines()
513 last_line_length = len(lines[-1]) if docstring else 0
515 # If adding closing quotes would cause the last line to exceed
516 # the maximum line length, and the closing quote is not
517 # prefixed by a newline then put a line break before
518 # the closing quotes
519 if (
520 len(lines) > 1
521 and last_line_length + quote_len > self.mode.line_length
522 and len(indent) + quote_len <= self.mode.line_length
523 and not has_trailing_backslash
524 ):
525 if leaf.value[-1 - quote_len] == "\n":
526 leaf.value = prefix + quote + docstring + quote
527 else:
528 leaf.value = prefix + quote + docstring + "\n" + indent + quote
529 else:
530 leaf.value = prefix + quote + docstring + quote
531 else:
532 leaf.value = prefix + quote + docstring + quote
534 if self.mode.string_normalization and leaf.type == token.STRING:
535 leaf.value = normalize_string_prefix(leaf.value)
536 leaf.value = normalize_string_quotes(leaf.value)
537 yield from self.visit_default(leaf)
539 def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]:
540 normalize_numeric_literal(leaf)
541 yield from self.visit_default(leaf)
543 def visit_atom(self, node: Node) -> Iterator[Line]:
544 """Visit any atom"""
545 if len(node.children) == 3:
546 first = node.children[0]
547 last = node.children[-1]
548 if (first.type == token.LSQB and last.type == token.RSQB) or (
549 first.type == token.LBRACE and last.type == token.RBRACE
550 ):
551 # Lists or sets of one item
552 maybe_make_parens_invisible_in_atom(
553 node.children[1],
554 parent=node,
555 mode=self.mode,
556 features=self.features,
557 )
559 yield from self.visit_default(node)
561 def visit_fstring(self, node: Node) -> Iterator[Line]:
562 # currently we don't want to format and split f-strings at all.
563 string_leaf = fstring_tstring_to_string(node)
564 node.replace(string_leaf)
565 if "\\" in string_leaf.value and any(
566 "\\" in str(child)
567 for child in node.children
568 if child.type == syms.fstring_replacement_field
569 ):
570 # string normalization doesn't account for nested quotes,
571 # causing breakages. skip normalization when nested quotes exist
572 yield from self.visit_default(string_leaf)
573 return
574 yield from self.visit_STRING(string_leaf)
576 def visit_tstring(self, node: Node) -> Iterator[Line]:
577 # currently we don't want to format and split t-strings at all.
578 string_leaf = fstring_tstring_to_string(node)
579 node.replace(string_leaf)
580 if "\\" in string_leaf.value and any(
581 "\\" in str(child)
582 for child in node.children
583 if child.type == syms.fstring_replacement_field
584 ):
585 # string normalization doesn't account for nested quotes,
586 # causing breakages. skip normalization when nested quotes exist
587 yield from self.visit_default(string_leaf)
588 return
589 yield from self.visit_STRING(string_leaf)
591 # TODO: Uncomment Implementation to format f-string children
592 # fstring_start = node.children[0]
593 # fstring_end = node.children[-1]
594 # assert isinstance(fstring_start, Leaf)
595 # assert isinstance(fstring_end, Leaf)
597 # quote_char = fstring_end.value[0]
598 # quote_idx = fstring_start.value.index(quote_char)
599 # prefix, quote = (
600 # fstring_start.value[:quote_idx],
601 # fstring_start.value[quote_idx:]
602 # )
604 # if not is_docstring(node, self.mode):
605 # prefix = normalize_string_prefix(prefix)
607 # assert quote == fstring_end.value
609 # is_raw_fstring = "r" in prefix or "R" in prefix
610 # middles = [
611 # leaf
612 # for leaf in node.leaves()
613 # if leaf.type == token.FSTRING_MIDDLE
614 # ]
616 # if self.mode.string_normalization:
617 # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring)
619 # fstring_start.value = prefix + quote
620 # fstring_end.value = quote
622 # yield from self.visit_default(node)
624 def visit_comp_for(self, node: Node) -> Iterator[Line]:
625 if Preview.wrap_comprehension_in in self.mode:
626 normalize_invisible_parens(
627 node, parens_after={"in"}, mode=self.mode, features=self.features
628 )
629 yield from self.visit_default(node)
631 def visit_old_comp_for(self, node: Node) -> Iterator[Line]:
632 yield from self.visit_comp_for(node)
634 def __post_init__(self) -> None:
635 """You are in a twisty little maze of passages."""
636 self.current_line = Line(mode=self.mode)
638 v = self.visit_stmt
639 Ø: set[str] = set()
640 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
641 self.visit_if_stmt = partial(
642 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
643 )
644 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
645 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
646 self.visit_try_stmt = partial(
647 v, keywords={"try", "except", "else", "finally"}, parens=Ø
648 )
649 self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
650 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
651 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
653 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
654 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
655 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
656 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
657 self.visit_async_funcdef = self.visit_async_stmt
658 self.visit_decorated = self.visit_decorators
660 # PEP 634
661 self.visit_match_stmt = self.visit_match_case
662 self.visit_case_block = self.visit_match_case
663 self.visit_guard = partial(v, keywords=Ø, parens={"if"})
666def _hugging_power_ops_line_to_string(
667 line: Line,
668 features: Collection[Feature],
669 mode: Mode,
670) -> Optional[str]:
671 try:
672 return line_to_string(next(hug_power_op(line, features, mode)))
673 except CannotTransform:
674 return None
677def transform_line(
678 line: Line, mode: Mode, features: Collection[Feature] = ()
679) -> Iterator[Line]:
680 """Transform a `line`, potentially splitting it into many lines.
682 They should fit in the allotted `line_length` but might not be able to.
684 `features` are syntactical features that may be used in the output.
685 """
686 if line.is_comment:
687 yield line
688 return
690 line_str = line_to_string(line)
692 # We need the line string when power operators are hugging to determine if we should
693 # split the line. Default to line_str, if no power operator are present on the line.
694 line_str_hugging_power_ops = (
695 _hugging_power_ops_line_to_string(line, features, mode) or line_str
696 )
698 ll = mode.line_length
699 sn = mode.string_normalization
700 string_merge = StringMerger(ll, sn)
701 string_paren_strip = StringParenStripper(ll, sn)
702 string_split = StringSplitter(ll, sn)
703 string_paren_wrap = StringParenWrapper(ll, sn)
705 transformers: list[Transformer]
706 if (
707 not line.contains_uncollapsable_type_comments()
708 and not line.should_split_rhs
709 and not line.magic_trailing_comma
710 and (
711 is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
712 or line.contains_unsplittable_type_ignore()
713 )
714 and not (line.inside_brackets and line.contains_standalone_comments())
715 and not line.contains_implicit_multiline_string_with_comments()
716 ):
717 # Only apply basic string preprocessing, since lines shouldn't be split here.
718 if Preview.string_processing in mode:
719 transformers = [string_merge, string_paren_strip]
720 else:
721 transformers = []
722 elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
723 transformers = [left_hand_split]
724 else:
726 def _rhs(
727 self: object, line: Line, features: Collection[Feature], mode: Mode
728 ) -> Iterator[Line]:
729 """Wraps calls to `right_hand_split`.
731 The calls increasingly `omit` right-hand trailers (bracket pairs with
732 content), meaning the trailers get glued together to split on another
733 bracket pair instead.
734 """
735 for omit in generate_trailers_to_omit(line, mode.line_length):
736 lines = list(right_hand_split(line, mode, features, omit=omit))
737 # Note: this check is only able to figure out if the first line of the
738 # *current* transformation fits in the line length. This is true only
739 # for simple cases. All others require running more transforms via
740 # `transform_line()`. This check doesn't know if those would succeed.
741 if is_line_short_enough(lines[0], mode=mode):
742 yield from lines
743 return
745 # All splits failed, best effort split with no omits.
746 # This mostly happens to multiline strings that are by definition
747 # reported as not fitting a single line, as well as lines that contain
748 # trailing commas (those have to be exploded).
749 yield from right_hand_split(line, mode, features=features)
751 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
752 # __name__ attribute which is needed in `run_transformer` further down.
753 # Unfortunately a nested class breaks mypyc too. So a class must be created
754 # via type ... https://github.com/mypyc/mypyc/issues/884
755 rhs = type("rhs", (), {"__call__": _rhs})()
757 if Preview.string_processing in mode:
758 if line.inside_brackets:
759 transformers = [
760 string_merge,
761 string_paren_strip,
762 string_split,
763 delimiter_split,
764 standalone_comment_split,
765 string_paren_wrap,
766 rhs,
767 ]
768 else:
769 transformers = [
770 string_merge,
771 string_paren_strip,
772 string_split,
773 string_paren_wrap,
774 rhs,
775 ]
776 else:
777 if line.inside_brackets:
778 transformers = [delimiter_split, standalone_comment_split, rhs]
779 else:
780 transformers = [rhs]
781 # It's always safe to attempt hugging of power operations and pretty much every line
782 # could match.
783 transformers.append(hug_power_op)
785 for transform in transformers:
786 # We are accumulating lines in `result` because we might want to abort
787 # mission and return the original line in the end, or attempt a different
788 # split altogether.
789 try:
790 result = run_transformer(line, transform, mode, features, line_str=line_str)
791 except CannotTransform:
792 continue
793 else:
794 yield from result
795 break
797 else:
798 yield line
801def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
802 """If a funcdef has a magic trailing comma in the return type, then we should first
803 split the line with rhs to respect the comma.
804 """
805 return_type_leaves: list[Leaf] = []
806 in_return_type = False
808 for leaf in line.leaves:
809 if leaf.type == token.COLON:
810 in_return_type = False
811 if in_return_type:
812 return_type_leaves.append(leaf)
813 if leaf.type == token.RARROW:
814 in_return_type = True
816 # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
817 # couple lines from it.
818 result = Line(mode=line.mode, depth=line.depth)
819 leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
820 for leaf in return_type_leaves:
821 result.append(
822 leaf,
823 preformatted=True,
824 track_bracket=id(leaf) in leaves_to_track,
825 )
827 # we could also return true if the line is too long, and the return type is longer
828 # than the param list. Or if `should_split_rhs` returns True.
829 return result.magic_trailing_comma is not None
832class _BracketSplitComponent(Enum):
833 head = auto()
834 body = auto()
835 tail = auto()
838def left_hand_split(
839 line: Line, _features: Collection[Feature], mode: Mode
840) -> Iterator[Line]:
841 """Split line into many lines, starting with the first matching bracket pair.
843 Note: this usually looks weird, only use this for function definitions.
844 Prefer RHS otherwise. This is why this function is not symmetrical with
845 :func:`right_hand_split` which also handles optional parentheses.
846 """
847 for leaf_type in [token.LPAR, token.LSQB]:
848 tail_leaves: list[Leaf] = []
849 body_leaves: list[Leaf] = []
850 head_leaves: list[Leaf] = []
851 current_leaves = head_leaves
852 matching_bracket: Optional[Leaf] = None
853 depth = 0
854 for index, leaf in enumerate(line.leaves):
855 if index == 2 and leaf.type == token.LSQB:
856 # A [ at index 2 means this is a type param, so start
857 # tracking the depth
858 depth += 1
859 elif depth > 0:
860 if leaf.type == token.LSQB:
861 depth += 1
862 elif leaf.type == token.RSQB:
863 depth -= 1
864 if (
865 current_leaves is body_leaves
866 and leaf.type in CLOSING_BRACKETS
867 and leaf.opening_bracket is matching_bracket
868 and isinstance(matching_bracket, Leaf)
869 # If the code is still on LPAR and we are inside a type
870 # param, ignore the match since this is searching
871 # for the function arguments
872 and not (leaf_type == token.LPAR and depth > 0)
873 ):
874 ensure_visible(leaf)
875 ensure_visible(matching_bracket)
876 current_leaves = tail_leaves if body_leaves else head_leaves
877 current_leaves.append(leaf)
878 if current_leaves is head_leaves:
879 if leaf.type == leaf_type and (
880 Preview.fix_type_expansion_split not in mode
881 or not (leaf_type == token.LPAR and depth > 0)
882 ):
883 matching_bracket = leaf
884 current_leaves = body_leaves
885 if matching_bracket and tail_leaves:
886 break
887 if not matching_bracket or not tail_leaves:
888 raise CannotSplit("No brackets found")
890 head = bracket_split_build_line(
891 head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
892 )
893 body = bracket_split_build_line(
894 body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
895 )
896 tail = bracket_split_build_line(
897 tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
898 )
899 bracket_split_succeeded_or_raise(head, body, tail)
900 for result in (head, body, tail):
901 if result:
902 yield result
905def right_hand_split(
906 line: Line,
907 mode: Mode,
908 features: Collection[Feature] = (),
909 omit: Collection[LeafID] = (),
910) -> Iterator[Line]:
911 """Split line into many lines, starting with the last matching bracket pair.
913 If the split was by optional parentheses, attempt splitting without them, too.
914 `omit` is a collection of closing bracket IDs that shouldn't be considered for
915 this split.
917 Note: running this function modifies `bracket_depth` on the leaves of `line`.
918 """
919 rhs_result = _first_right_hand_split(line, omit=omit)
920 yield from _maybe_split_omitting_optional_parens(
921 rhs_result, line, mode, features=features, omit=omit
922 )
925def _first_right_hand_split(
926 line: Line,
927 omit: Collection[LeafID] = (),
928) -> RHSResult:
929 """Split the line into head, body, tail starting with the last bracket pair.
931 Note: this function should not have side effects. It's relied upon by
932 _maybe_split_omitting_optional_parens to get an opinion whether to prefer
933 splitting on the right side of an assignment statement.
934 """
935 tail_leaves: list[Leaf] = []
936 body_leaves: list[Leaf] = []
937 head_leaves: list[Leaf] = []
938 current_leaves = tail_leaves
939 opening_bracket: Optional[Leaf] = None
940 closing_bracket: Optional[Leaf] = None
941 for leaf in reversed(line.leaves):
942 if current_leaves is body_leaves:
943 if leaf is opening_bracket:
944 current_leaves = head_leaves if body_leaves else tail_leaves
945 current_leaves.append(leaf)
946 if current_leaves is tail_leaves:
947 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
948 opening_bracket = leaf.opening_bracket
949 closing_bracket = leaf
950 current_leaves = body_leaves
951 if not (opening_bracket and closing_bracket and head_leaves):
952 # If there is no opening or closing_bracket that means the split failed and
953 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
954 # the matching `opening_bracket` wasn't available on `line` anymore.
955 raise CannotSplit("No brackets found")
957 tail_leaves.reverse()
958 body_leaves.reverse()
959 head_leaves.reverse()
961 body: Optional[Line] = None
962 if (
963 Preview.hug_parens_with_braces_and_square_brackets in line.mode
964 and tail_leaves[0].value
965 and tail_leaves[0].opening_bracket is head_leaves[-1]
966 ):
967 inner_body_leaves = list(body_leaves)
968 hugged_opening_leaves: list[Leaf] = []
969 hugged_closing_leaves: list[Leaf] = []
970 is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR]
971 unpacking_offset: int = 1 if is_unpacking else 0
972 while (
973 len(inner_body_leaves) >= 2 + unpacking_offset
974 and inner_body_leaves[-1].type in CLOSING_BRACKETS
975 and inner_body_leaves[-1].opening_bracket
976 is inner_body_leaves[unpacking_offset]
977 ):
978 if unpacking_offset:
979 hugged_opening_leaves.append(inner_body_leaves.pop(0))
980 unpacking_offset = 0
981 hugged_opening_leaves.append(inner_body_leaves.pop(0))
982 hugged_closing_leaves.insert(0, inner_body_leaves.pop())
984 if hugged_opening_leaves and inner_body_leaves:
985 inner_body = bracket_split_build_line(
986 inner_body_leaves,
987 line,
988 hugged_opening_leaves[-1],
989 component=_BracketSplitComponent.body,
990 )
991 if (
992 line.mode.magic_trailing_comma
993 and inner_body_leaves[-1].type == token.COMMA
994 ):
995 should_hug = True
996 else:
997 line_length = line.mode.line_length - sum(
998 len(str(leaf))
999 for leaf in hugged_opening_leaves + hugged_closing_leaves
1000 )
1001 if is_line_short_enough(
1002 inner_body, mode=replace(line.mode, line_length=line_length)
1003 ):
1004 # Do not hug if it fits on a single line.
1005 should_hug = False
1006 else:
1007 should_hug = True
1008 if should_hug:
1009 body_leaves = inner_body_leaves
1010 head_leaves.extend(hugged_opening_leaves)
1011 tail_leaves = hugged_closing_leaves + tail_leaves
1012 body = inner_body # No need to re-calculate the body again later.
1014 head = bracket_split_build_line(
1015 head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
1016 )
1017 if body is None:
1018 body = bracket_split_build_line(
1019 body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
1020 )
1021 tail = bracket_split_build_line(
1022 tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
1023 )
1024 bracket_split_succeeded_or_raise(head, body, tail)
1025 return RHSResult(head, body, tail, opening_bracket, closing_bracket)
1028def _maybe_split_omitting_optional_parens(
1029 rhs: RHSResult,
1030 line: Line,
1031 mode: Mode,
1032 features: Collection[Feature] = (),
1033 omit: Collection[LeafID] = (),
1034) -> Iterator[Line]:
1035 if (
1036 Feature.FORCE_OPTIONAL_PARENTHESES not in features
1037 # the opening bracket is an optional paren
1038 and rhs.opening_bracket.type == token.LPAR
1039 and not rhs.opening_bracket.value
1040 # the closing bracket is an optional paren
1041 and rhs.closing_bracket.type == token.RPAR
1042 and not rhs.closing_bracket.value
1043 # it's not an import (optional parens are the only thing we can split on
1044 # in this case; attempting a split without them is a waste of time)
1045 and not line.is_import
1046 # and we can actually remove the parens
1047 and can_omit_invisible_parens(rhs, mode.line_length)
1048 ):
1049 omit = {id(rhs.closing_bracket), *omit}
1050 try:
1051 # The RHSResult Omitting Optional Parens.
1052 rhs_oop = _first_right_hand_split(line, omit=omit)
1053 if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode):
1054 yield from _maybe_split_omitting_optional_parens(
1055 rhs_oop, line, mode, features=features, omit=omit
1056 )
1057 return
1059 except CannotSplit as e:
1060 # For chained assignments we want to use the previous successful split
1061 if line.is_chained_assignment:
1062 pass
1064 elif (
1065 not can_be_split(rhs.body)
1066 and not is_line_short_enough(rhs.body, mode=mode)
1067 and not (
1068 Preview.wrap_long_dict_values_in_parens
1069 and rhs.opening_bracket.parent
1070 and rhs.opening_bracket.parent.parent
1071 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1072 )
1073 ):
1074 raise CannotSplit(
1075 "Splitting failed, body is still too long and can't be split."
1076 ) from e
1078 elif (
1079 rhs.head.contains_multiline_strings()
1080 or rhs.tail.contains_multiline_strings()
1081 ):
1082 raise CannotSplit(
1083 "The current optional pair of parentheses is bound to fail to"
1084 " satisfy the splitting algorithm because the head or the tail"
1085 " contains multiline strings which by definition never fit one"
1086 " line."
1087 ) from e
1089 ensure_visible(rhs.opening_bracket)
1090 ensure_visible(rhs.closing_bracket)
1091 for result in (rhs.head, rhs.body, rhs.tail):
1092 if result:
1093 yield result
1096def _prefer_split_rhs_oop_over_rhs(
1097 rhs_oop: RHSResult, rhs: RHSResult, mode: Mode
1098) -> bool:
1099 """
1100 Returns whether we should prefer the result from a split omitting optional parens
1101 (rhs_oop) over the original (rhs).
1102 """
1103 # contains unsplittable type ignore
1104 if (
1105 rhs_oop.head.contains_unsplittable_type_ignore()
1106 or rhs_oop.body.contains_unsplittable_type_ignore()
1107 or rhs_oop.tail.contains_unsplittable_type_ignore()
1108 ):
1109 return True
1111 # Retain optional parens around dictionary values
1112 if (
1113 Preview.wrap_long_dict_values_in_parens
1114 and rhs.opening_bracket.parent
1115 and rhs.opening_bracket.parent.parent
1116 and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker
1117 and rhs.body.bracket_tracker.delimiters
1118 ):
1119 # Unless the split is inside the key
1120 return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves)
1122 # the split is right after `=`
1123 if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL):
1124 return True
1126 # the left side of assignment contains brackets
1127 if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]):
1128 return True
1130 # the left side of assignment is short enough (the -1 is for the ending optional
1131 # paren)
1132 if not is_line_short_enough(
1133 rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
1134 ):
1135 return True
1137 # the left side of assignment won't explode further because of magic trailing comma
1138 if rhs.head.magic_trailing_comma is not None:
1139 return True
1141 # If we have multiple targets, we prefer more `=`s on the head vs pushing them to
1142 # the body
1143 rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL)
1144 rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count(
1145 token.EQUAL
1146 )
1147 if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count:
1148 return False
1150 has_closing_bracket_after_assign = False
1151 for leaf in reversed(rhs_oop.head.leaves):
1152 if leaf.type == token.EQUAL:
1153 break
1154 if leaf.type in CLOSING_BRACKETS:
1155 has_closing_bracket_after_assign = True
1156 break
1157 return (
1158 # contains matching brackets after the `=` (done by checking there is a
1159 # closing bracket)
1160 has_closing_bracket_after_assign
1161 or (
1162 # the split is actually from inside the optional parens (done by checking
1163 # the first line still contains the `=`)
1164 any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
1165 # the first line is short enough
1166 and is_line_short_enough(rhs_oop.head, mode=mode)
1167 )
1168 )
1171def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
1172 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
1174 Do nothing otherwise.
1176 A left- or right-hand split is based on a pair of brackets. Content before
1177 (and including) the opening bracket is left on one line, content inside the
1178 brackets is put on a separate line, and finally content starting with and
1179 following the closing bracket is put on a separate line.
1181 Those are called `head`, `body`, and `tail`, respectively. If the split
1182 produced the same line (all content in `head`) or ended up with an empty `body`
1183 and the `tail` is just the closing bracket, then it's considered failed.
1184 """
1185 tail_len = len(str(tail).strip())
1186 if not body:
1187 if tail_len == 0:
1188 raise CannotSplit("Splitting brackets produced the same line")
1190 elif tail_len < 3:
1191 raise CannotSplit(
1192 f"Splitting brackets on an empty body to save {tail_len} characters is"
1193 " not worth it"
1194 )
1197def _ensure_trailing_comma(
1198 leaves: list[Leaf], original: Line, opening_bracket: Leaf
1199) -> bool:
1200 if not leaves:
1201 return False
1202 # Ensure a trailing comma for imports
1203 if original.is_import:
1204 return True
1205 # ...and standalone function arguments
1206 if not original.is_def:
1207 return False
1208 if opening_bracket.value != "(":
1209 return False
1210 # Don't add commas if we already have any commas
1211 if any(
1212 leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves
1213 ):
1214 return False
1216 # Find a leaf with a parent (comments don't have parents)
1217 leaf_with_parent = next((leaf for leaf in leaves if leaf.parent), None)
1218 if leaf_with_parent is None:
1219 return True
1220 # Don't add commas inside parenthesized return annotations
1221 if get_annotation_type(leaf_with_parent) == "return":
1222 return False
1223 # Don't add commas inside PEP 604 unions
1224 if (
1225 leaf_with_parent.parent
1226 and leaf_with_parent.parent.next_sibling
1227 and leaf_with_parent.parent.next_sibling.type == token.VBAR
1228 ):
1229 return False
1230 return True
1233def bracket_split_build_line(
1234 leaves: list[Leaf],
1235 original: Line,
1236 opening_bracket: Leaf,
1237 *,
1238 component: _BracketSplitComponent,
1239) -> Line:
1240 """Return a new line with given `leaves` and respective comments from `original`.
1242 If it's the head component, brackets will be tracked so trailing commas are
1243 respected.
1245 If it's the body component, the result line is one-indented inside brackets and as
1246 such has its first leaf's prefix normalized and a trailing comma added when
1247 expected.
1248 """
1249 result = Line(mode=original.mode, depth=original.depth)
1250 if component is _BracketSplitComponent.body:
1251 result.inside_brackets = True
1252 result.depth += 1
1253 if _ensure_trailing_comma(leaves, original, opening_bracket):
1254 for i in range(len(leaves) - 1, -1, -1):
1255 if leaves[i].type == STANDALONE_COMMENT:
1256 continue
1258 if leaves[i].type != token.COMMA:
1259 new_comma = Leaf(token.COMMA, ",")
1260 leaves.insert(i + 1, new_comma)
1261 break
1263 leaves_to_track: set[LeafID] = set()
1264 if component is _BracketSplitComponent.head:
1265 leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1266 # Populate the line
1267 for leaf in leaves:
1268 result.append(
1269 leaf,
1270 preformatted=True,
1271 track_bracket=id(leaf) in leaves_to_track,
1272 )
1273 for comment_after in original.comments_after(leaf):
1274 result.append(comment_after, preformatted=True)
1275 if component is _BracketSplitComponent.body and should_split_line(
1276 result, opening_bracket
1277 ):
1278 result.should_split_rhs = True
1279 return result
1282def dont_increase_indentation(split_func: Transformer) -> Transformer:
1283 """Normalize prefix of the first leaf in every line returned by `split_func`.
1285 This is a decorator over relevant split functions.
1286 """
1288 @wraps(split_func)
1289 def split_wrapper(
1290 line: Line, features: Collection[Feature], mode: Mode
1291 ) -> Iterator[Line]:
1292 for split_line in split_func(line, features, mode):
1293 split_line.leaves[0].prefix = ""
1294 yield split_line
1296 return split_wrapper
1299def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1300 for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1301 if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1302 return leaf_idx
1303 return None
1306def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool:
1307 if is_vararg(leaf, within={syms.typedargslist}):
1308 return Feature.TRAILING_COMMA_IN_DEF in features
1309 if is_vararg(leaf, within={syms.arglist, syms.argument}):
1310 return Feature.TRAILING_COMMA_IN_CALL in features
1311 return True
1314def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1315 if (
1316 safe
1317 and delimiter_priority == COMMA_PRIORITY
1318 and line.leaves[-1].type != token.COMMA
1319 and line.leaves[-1].type != STANDALONE_COMMENT
1320 ):
1321 new_comma = Leaf(token.COMMA, ",")
1322 line.append(new_comma)
1323 return line
1326MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY}
1329@dont_increase_indentation
1330def delimiter_split(
1331 line: Line, features: Collection[Feature], mode: Mode
1332) -> Iterator[Line]:
1333 """Split according to delimiters of the highest priority.
1335 If the appropriate Features are given, the split will add trailing commas
1336 also in function signatures and calls that contain `*` and `**`.
1337 """
1338 if len(line.leaves) == 0:
1339 raise CannotSplit("Line empty") from None
1340 last_leaf = line.leaves[-1]
1342 bt = line.bracket_tracker
1343 try:
1344 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1345 except ValueError:
1346 raise CannotSplit("No delimiters found") from None
1348 if (
1349 delimiter_priority == DOT_PRIORITY
1350 and bt.delimiter_count_with_priority(delimiter_priority) == 1
1351 ):
1352 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1354 current_line = Line(
1355 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1356 )
1357 lowest_depth = sys.maxsize
1358 trailing_comma_safe = True
1360 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1361 """Append `leaf` to current line or to new line if appending impossible."""
1362 nonlocal current_line
1363 try:
1364 current_line.append_safe(leaf, preformatted=True)
1365 except ValueError:
1366 yield current_line
1368 current_line = Line(
1369 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1370 )
1371 current_line.append(leaf)
1373 def append_comments(leaf: Leaf) -> Iterator[Line]:
1374 for comment_after in line.comments_after(leaf):
1375 yield from append_to_line(comment_after)
1377 last_non_comment_leaf = _get_last_non_comment_leaf(line)
1378 for leaf_idx, leaf in enumerate(line.leaves):
1379 yield from append_to_line(leaf)
1381 previous_priority = leaf_idx > 0 and bt.delimiters.get(
1382 id(line.leaves[leaf_idx - 1])
1383 )
1384 if (
1385 previous_priority != delimiter_priority
1386 or delimiter_priority in MIGRATE_COMMENT_DELIMITERS
1387 ):
1388 yield from append_comments(leaf)
1390 lowest_depth = min(lowest_depth, leaf.bracket_depth)
1391 if trailing_comma_safe and leaf.bracket_depth == lowest_depth:
1392 trailing_comma_safe = _can_add_trailing_comma(leaf, features)
1394 if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:
1395 current_line = _safe_add_trailing_comma(
1396 trailing_comma_safe, delimiter_priority, current_line
1397 )
1399 leaf_priority = bt.delimiters.get(id(leaf))
1400 if leaf_priority == delimiter_priority:
1401 if (
1402 leaf_idx + 1 < len(line.leaves)
1403 and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS
1404 ):
1405 yield from append_comments(line.leaves[leaf_idx + 1])
1407 yield current_line
1408 current_line = Line(
1409 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1410 )
1412 if current_line:
1413 current_line = _safe_add_trailing_comma(
1414 trailing_comma_safe, delimiter_priority, current_line
1415 )
1416 yield current_line
1419@dont_increase_indentation
1420def standalone_comment_split(
1421 line: Line, features: Collection[Feature], mode: Mode
1422) -> Iterator[Line]:
1423 """Split standalone comments from the rest of the line."""
1424 if not line.contains_standalone_comments():
1425 raise CannotSplit("Line does not have any standalone comments")
1427 current_line = Line(
1428 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1429 )
1431 def append_to_line(leaf: Leaf) -> Iterator[Line]:
1432 """Append `leaf` to current line or to new line if appending impossible."""
1433 nonlocal current_line
1434 try:
1435 current_line.append_safe(leaf, preformatted=True)
1436 except ValueError:
1437 yield current_line
1439 current_line = Line(
1440 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1441 )
1442 current_line.append(leaf)
1444 for leaf in line.leaves:
1445 yield from append_to_line(leaf)
1447 for comment_after in line.comments_after(leaf):
1448 yield from append_to_line(comment_after)
1450 if current_line:
1451 yield current_line
1454def normalize_invisible_parens( # noqa: C901
1455 node: Node, parens_after: set[str], *, mode: Mode, features: Collection[Feature]
1456) -> None:
1457 """Make existing optional parentheses invisible or create new ones.
1459 `parens_after` is a set of string leaf values immediately after which parens
1460 should be put.
1462 Standardizes on visible parentheses for single-element tuples, and keeps
1463 existing visible parentheses for other tuples and generator expressions.
1464 """
1465 for pc in list_comments(node.prefix, is_endmarker=False, mode=mode):
1466 if pc.value in FMT_OFF:
1467 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1468 return
1470 # The multiple context managers grammar has a different pattern, thus this is
1471 # separate from the for-loop below. This possibly wraps them in invisible parens,
1472 # and later will be removed in remove_with_parens when needed.
1473 if node.type == syms.with_stmt:
1474 _maybe_wrap_cms_in_parens(node, mode, features)
1476 check_lpar = False
1477 for index, child in enumerate(list(node.children)):
1478 # Fixes a bug where invisible parens are not properly stripped from
1479 # assignment statements that contain type annotations.
1480 if isinstance(child, Node) and child.type == syms.annassign:
1481 normalize_invisible_parens(
1482 child, parens_after=parens_after, mode=mode, features=features
1483 )
1485 # Fixes a bug where invisible parens are not properly wrapped around
1486 # case blocks.
1487 if isinstance(child, Node) and child.type == syms.case_block:
1488 normalize_invisible_parens(
1489 child, parens_after={"case"}, mode=mode, features=features
1490 )
1492 # Add parentheses around if guards in case blocks
1493 if isinstance(child, Node) and child.type == syms.guard:
1494 normalize_invisible_parens(
1495 child, parens_after={"if"}, mode=mode, features=features
1496 )
1498 # Add parentheses around long tuple unpacking in assignments.
1499 if (
1500 index == 0
1501 and isinstance(child, Node)
1502 and child.type == syms.testlist_star_expr
1503 ):
1504 check_lpar = True
1506 if check_lpar:
1507 if (
1508 child.type == syms.atom
1509 and node.type == syms.for_stmt
1510 and isinstance(child.prev_sibling, Leaf)
1511 and child.prev_sibling.type == token.NAME
1512 and child.prev_sibling.value == "for"
1513 ):
1514 if maybe_make_parens_invisible_in_atom(
1515 child,
1516 parent=node,
1517 mode=mode,
1518 features=features,
1519 remove_brackets_around_comma=True,
1520 ):
1521 wrap_in_parentheses(node, child, visible=False)
1522 elif isinstance(child, Node) and node.type == syms.with_stmt:
1523 remove_with_parens(child, node, mode=mode, features=features)
1524 elif child.type == syms.atom and not (
1525 "in" in parens_after
1526 and len(child.children) == 3
1527 and is_lpar_token(child.children[0])
1528 and is_rpar_token(child.children[-1])
1529 and child.children[1].type == syms.test
1530 ):
1531 if maybe_make_parens_invisible_in_atom(
1532 child, parent=node, mode=mode, features=features
1533 ):
1534 wrap_in_parentheses(node, child, visible=False)
1535 elif is_one_tuple(child):
1536 wrap_in_parentheses(node, child, visible=True)
1537 elif node.type == syms.import_from:
1538 _normalize_import_from(node, child, index)
1539 break
1540 elif (
1541 index == 1
1542 and child.type == token.STAR
1543 and node.type == syms.except_clause
1544 ):
1545 # In except* (PEP 654), the star is actually part of
1546 # of the keyword. So we need to skip the insertion of
1547 # invisible parentheses to work more precisely.
1548 continue
1550 elif (
1551 isinstance(child, Leaf)
1552 and child.next_sibling is not None
1553 and child.next_sibling.type == token.COLON
1554 and child.value == "case"
1555 ):
1556 # A special patch for "case case:" scenario, the second occurrence
1557 # of case will be not parsed as a Python keyword.
1558 break
1560 elif not is_multiline_string(child):
1561 wrap_in_parentheses(node, child, visible=False)
1563 comma_check = child.type == token.COMMA
1565 check_lpar = isinstance(child, Leaf) and (
1566 child.value in parens_after or comma_check
1567 )
1570def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1571 # "import from" nodes store parentheses directly as part of
1572 # the statement
1573 if is_lpar_token(child):
1574 assert is_rpar_token(parent.children[-1])
1575 # make parentheses invisible
1576 child.value = ""
1577 parent.children[-1].value = ""
1578 elif child.type != token.STAR:
1579 # insert invisible parentheses
1580 parent.insert_child(index, Leaf(token.LPAR, ""))
1581 parent.append_child(Leaf(token.RPAR, ""))
1584def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None:
1585 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1586 if (
1587 node.children[1].type == syms.atom
1588 and node.children[1].children[0].type == token.LPAR
1589 ):
1590 if maybe_make_parens_invisible_in_atom(
1591 node.children[1],
1592 parent=node,
1593 mode=mode,
1594 features=features,
1595 remove_brackets_around_comma=True,
1596 ):
1597 wrap_in_parentheses(node, node.children[1], visible=False)
1599 # Since await is an expression we shouldn't remove
1600 # brackets in cases where this would change
1601 # the AST due to operator precedence.
1602 # Therefore we only aim to remove brackets around
1603 # power nodes that aren't also await expressions themselves.
1604 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1605 # N.B. We've still removed any redundant nested brackets though :)
1606 opening_bracket = cast(Leaf, node.children[1].children[0])
1607 closing_bracket = cast(Leaf, node.children[1].children[-1])
1608 bracket_contents = node.children[1].children[1]
1609 if isinstance(bracket_contents, Node) and (
1610 bracket_contents.type != syms.power
1611 or bracket_contents.children[0].type == token.AWAIT
1612 or any(
1613 isinstance(child, Leaf) and child.type == token.DOUBLESTAR
1614 for child in bracket_contents.children
1615 )
1616 ):
1617 ensure_visible(opening_bracket)
1618 ensure_visible(closing_bracket)
1621def _maybe_wrap_cms_in_parens(
1622 node: Node, mode: Mode, features: Collection[Feature]
1623) -> None:
1624 """When enabled and safe, wrap the multiple context managers in invisible parens.
1626 It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1627 """
1628 if (
1629 Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1630 or len(node.children) <= 2
1631 # If it's an atom, it's already wrapped in parens.
1632 or node.children[1].type == syms.atom
1633 ):
1634 return
1635 colon_index: Optional[int] = None
1636 for i in range(2, len(node.children)):
1637 if node.children[i].type == token.COLON:
1638 colon_index = i
1639 break
1640 if colon_index is not None:
1641 lpar = Leaf(token.LPAR, "")
1642 rpar = Leaf(token.RPAR, "")
1643 context_managers = node.children[1:colon_index]
1644 for child in context_managers:
1645 child.remove()
1646 # After wrapping, the with_stmt will look like this:
1647 # with_stmt
1648 # NAME 'with'
1649 # atom
1650 # LPAR ''
1651 # testlist_gexp
1652 # ... <-- context_managers
1653 # /testlist_gexp
1654 # RPAR ''
1655 # /atom
1656 # COLON ':'
1657 new_child = Node(
1658 syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1659 )
1660 node.insert_child(1, new_child)
1663def remove_with_parens(
1664 node: Node, parent: Node, mode: Mode, features: Collection[Feature]
1665) -> None:
1666 """Recursively hide optional parens in `with` statements."""
1667 # Removing all unnecessary parentheses in with statements in one pass is a tad
1668 # complex as different variations of bracketed statements result in pretty
1669 # different parse trees:
1670 #
1671 # with (open("file")) as f: # this is an asexpr_test
1672 # ...
1673 #
1674 # with (open("file") as f): # this is an atom containing an
1675 # ... # asexpr_test
1676 #
1677 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1678 # ... # asexpr_test
1679 #
1680 # with (open("file") as f, open("file") as f): # an atom containing a
1681 # ... # testlist_gexp which then
1682 # # contains multiple asexpr_test(s)
1683 if node.type == syms.atom:
1684 if maybe_make_parens_invisible_in_atom(
1685 node,
1686 parent=parent,
1687 mode=mode,
1688 features=features,
1689 remove_brackets_around_comma=True,
1690 ):
1691 wrap_in_parentheses(parent, node, visible=False)
1692 if isinstance(node.children[1], Node):
1693 remove_with_parens(node.children[1], node, mode=mode, features=features)
1694 elif node.type == syms.testlist_gexp:
1695 for child in node.children:
1696 if isinstance(child, Node):
1697 remove_with_parens(child, node, mode=mode, features=features)
1698 elif node.type == syms.asexpr_test and not any(
1699 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1700 ):
1701 if maybe_make_parens_invisible_in_atom(
1702 node.children[0],
1703 parent=node,
1704 mode=mode,
1705 features=features,
1706 remove_brackets_around_comma=True,
1707 ):
1708 wrap_in_parentheses(node, node.children[0], visible=False)
1711def maybe_make_parens_invisible_in_atom(
1712 node: LN,
1713 parent: LN,
1714 mode: Mode,
1715 features: Collection[Feature],
1716 remove_brackets_around_comma: bool = False,
1717) -> bool:
1718 """If it's safe, make the parens in the atom `node` invisible, recursively.
1719 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1720 as they are redundant.
1722 Returns whether the node should itself be wrapped in invisible parentheses.
1723 """
1724 if (
1725 node.type not in (syms.atom, syms.expr)
1726 or is_empty_tuple(node)
1727 or is_one_tuple(node)
1728 or (is_tuple(node) and parent.type == syms.asexpr_test)
1729 or (
1730 is_tuple(node)
1731 and parent.type == syms.with_stmt
1732 and has_sibling_with_type(node, token.COMMA)
1733 )
1734 or (is_yield(node) and parent.type != syms.expr_stmt)
1735 or (
1736 # This condition tries to prevent removing non-optional brackets
1737 # around a tuple, however, can be a bit overzealous so we provide
1738 # and option to skip this check for `for` and `with` statements.
1739 not remove_brackets_around_comma
1740 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1741 # Skip this check in Preview mode in order to
1742 # Remove parentheses around multiple exception types in except and
1743 # except* without as. See PEP 758 for details.
1744 and not (
1745 Preview.remove_parens_around_except_types in mode
1746 and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features
1747 # is a tuple
1748 and is_tuple(node)
1749 # has a parent node
1750 and node.parent is not None
1751 # parent is an except clause
1752 and node.parent.type == syms.except_clause
1753 # is not immediately followed by as clause
1754 and not (
1755 node.next_sibling is not None
1756 and is_name_token(node.next_sibling)
1757 and node.next_sibling.value == "as"
1758 )
1759 )
1760 )
1761 or is_tuple_containing_walrus(node)
1762 or is_tuple_containing_star(node)
1763 or is_generator(node)
1764 ):
1765 return False
1767 if is_walrus_assignment(node):
1768 if parent.type in [
1769 syms.annassign,
1770 syms.expr_stmt,
1771 syms.assert_stmt,
1772 syms.return_stmt,
1773 syms.except_clause,
1774 syms.funcdef,
1775 syms.with_stmt,
1776 syms.testlist_gexp,
1777 syms.tname,
1778 # these ones aren't useful to end users, but they do please fuzzers
1779 syms.for_stmt,
1780 syms.del_stmt,
1781 syms.for_stmt,
1782 ]:
1783 return False
1785 first = node.children[0]
1786 last = node.children[-1]
1787 if is_lpar_token(first) and is_rpar_token(last):
1788 middle = node.children[1]
1789 # make parentheses invisible
1790 if (
1791 # If the prefix of `middle` includes a type comment with
1792 # ignore annotation, then we do not remove the parentheses
1793 not is_type_ignore_comment_string(middle.prefix.strip(), mode=mode)
1794 ):
1795 first.value = ""
1796 last.value = ""
1797 maybe_make_parens_invisible_in_atom(
1798 middle,
1799 parent=parent,
1800 mode=mode,
1801 features=features,
1802 remove_brackets_around_comma=remove_brackets_around_comma,
1803 )
1805 if is_atom_with_invisible_parens(middle):
1806 # Strip the invisible parens from `middle` by replacing
1807 # it with the child in-between the invisible parens
1808 middle.replace(middle.children[1])
1810 if middle.children[0].prefix.strip():
1811 # Preserve comments before first paren
1812 middle.children[1].prefix = (
1813 middle.children[0].prefix + middle.children[1].prefix
1814 )
1816 if middle.children[-1].prefix.strip():
1817 # Preserve comments before last paren
1818 last.prefix = middle.children[-1].prefix + last.prefix
1820 return False
1822 return True
1825def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1826 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1828 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1829 return False
1831 # We're essentially checking if the body is delimited by commas and there's more
1832 # than one of them (we're excluding the trailing comma and if the delimiter priority
1833 # is still commas, that means there's more).
1834 exclude = set()
1835 trailing_comma = False
1836 try:
1837 last_leaf = line.leaves[-1]
1838 if last_leaf.type == token.COMMA:
1839 trailing_comma = True
1840 exclude.add(id(last_leaf))
1841 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1842 except (IndexError, ValueError):
1843 return False
1845 return max_priority == COMMA_PRIORITY and (
1846 (line.mode.magic_trailing_comma and trailing_comma)
1847 # always explode imports
1848 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1849 )
1852def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[set[LeafID]]:
1853 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1855 Brackets can be omitted if the entire trailer up to and including
1856 a preceding closing bracket fits in one line.
1858 Yielded sets are cumulative (contain results of previous yields, too). First
1859 set is empty, unless the line should explode, in which case bracket pairs until
1860 the one that needs to explode are omitted.
1861 """
1863 omit: set[LeafID] = set()
1864 if not line.magic_trailing_comma:
1865 yield omit
1867 length = 4 * line.depth
1868 opening_bracket: Optional[Leaf] = None
1869 closing_bracket: Optional[Leaf] = None
1870 inner_brackets: set[LeafID] = set()
1871 for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):
1872 length += leaf_length
1873 if length > line_length:
1874 break
1876 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1877 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1878 break
1880 if opening_bracket:
1881 if leaf is opening_bracket:
1882 opening_bracket = None
1883 elif leaf.type in CLOSING_BRACKETS:
1884 prev = line.leaves[index - 1] if index > 0 else None
1885 if (
1886 prev
1887 and prev.type == token.COMMA
1888 and leaf.opening_bracket is not None
1889 and not is_one_sequence_between(
1890 leaf.opening_bracket, leaf, line.leaves
1891 )
1892 ):
1893 # Never omit bracket pairs with trailing commas.
1894 # We need to explode on those.
1895 break
1897 inner_brackets.add(id(leaf))
1898 elif leaf.type in CLOSING_BRACKETS:
1899 prev = line.leaves[index - 1] if index > 0 else None
1900 if prev and prev.type in OPENING_BRACKETS:
1901 # Empty brackets would fail a split so treat them as "inner"
1902 # brackets (e.g. only add them to the `omit` set if another
1903 # pair of brackets was good enough.
1904 inner_brackets.add(id(leaf))
1905 continue
1907 if closing_bracket:
1908 omit.add(id(closing_bracket))
1909 omit.update(inner_brackets)
1910 inner_brackets.clear()
1911 yield omit
1913 if (
1914 prev
1915 and prev.type == token.COMMA
1916 and leaf.opening_bracket is not None
1917 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1918 ):
1919 # Never omit bracket pairs with trailing commas.
1920 # We need to explode on those.
1921 break
1923 if leaf.value:
1924 opening_bracket = leaf.opening_bracket
1925 closing_bracket = leaf
1928def run_transformer(
1929 line: Line,
1930 transform: Transformer,
1931 mode: Mode,
1932 features: Collection[Feature],
1933 *,
1934 line_str: str = "",
1935) -> list[Line]:
1936 if not line_str:
1937 line_str = line_to_string(line)
1938 result: list[Line] = []
1939 for transformed_line in transform(line, features, mode):
1940 if str(transformed_line).strip("\n") == line_str:
1941 raise CannotTransform("Line transformer returned an unchanged result")
1943 result.extend(transform_line(transformed_line, mode=mode, features=features))
1945 features_set = set(features)
1946 if (
1947 Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1948 or transform.__class__.__name__ != "rhs"
1949 or not line.bracket_tracker.invisible
1950 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1951 or line.contains_multiline_strings()
1952 or result[0].contains_uncollapsable_type_comments()
1953 or result[0].contains_unsplittable_type_ignore()
1954 or is_line_short_enough(result[0], mode=mode)
1955 # If any leaves have no parents (which _can_ occur since
1956 # `transform(line)` potentially destroys the line's underlying node
1957 # structure), then we can't proceed. Doing so would cause the below
1958 # call to `append_leaves()` to fail.
1959 or any(leaf.parent is None for leaf in line.leaves)
1960 ):
1961 return result
1963 line_copy = line.clone()
1964 append_leaves(line_copy, line, line.leaves)
1965 features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1966 second_opinion = run_transformer(
1967 line_copy, transform, mode, features_fop, line_str=line_str
1968 )
1969 if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1970 result = second_opinion
1971 return result