Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/trans.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2String transformers that can split and merge strings.
3"""
5import re
6from abc import ABC, abstractmethod
7from collections import defaultdict
8from collections.abc import Callable, Collection, Iterable, Iterator, Sequence
9from dataclasses import dataclass
10from typing import Any, ClassVar, Final, Literal, TypeVar, Union
12from mypy_extensions import trait
14from black.comments import contains_pragma_comment
15from black.lines import Line, append_leaves
16from black.mode import Feature, Mode
17from black.nodes import (
18 CLOSING_BRACKETS,
19 OPENING_BRACKETS,
20 STANDALONE_COMMENT,
21 is_empty_lpar,
22 is_empty_par,
23 is_empty_rpar,
24 is_part_of_annotation,
25 parent_type,
26 replace_child,
27 syms,
28)
29from black.rusty import Err, Ok, Result
30from black.strings import (
31 assert_is_leaf_string,
32 count_chars_in_width,
33 get_string_prefix,
34 has_triple_quotes,
35 normalize_string_quotes,
36 str_width,
37)
38from blib2to3.pgen2 import token
39from blib2to3.pytree import Leaf, Node
42class CannotTransform(Exception):
43 """Base class for errors raised by Transformers."""
46# types
47T = TypeVar("T")
48LN = Union[Leaf, Node]
49Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]
50Index = int
51NodeType = int
52ParserState = int
53StringID = int
54TResult = Result[T, CannotTransform] # (T)ransform Result
55TMatchResult = TResult[list[Index]]
57SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops
60def TErr(err_msg: str) -> Err[CannotTransform]:
61 """(T)ransform Err
63 Convenience function used when working with the TResult type.
64 """
65 cant_transform = CannotTransform(err_msg)
66 return Err(cant_transform)
69# Remove when `simplify_power_operator_hugging` becomes stable.
70def hug_power_op(
71 line: Line, features: Collection[Feature], mode: Mode
72) -> Iterator[Line]:
73 """A transformer which normalizes spacing around power operators."""
75 # Performance optimization to avoid unnecessary Leaf clones and other ops.
76 for leaf in line.leaves:
77 if leaf.type == token.DOUBLESTAR:
78 break
79 else:
80 raise CannotTransform("No doublestar token was found in the line.")
82 def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool:
83 # Brackets and parentheses indicate calls, subscripts, etc. ...
84 # basically stuff that doesn't count as "simple". Only a NAME lookup
85 # or dotted lookup (eg. NAME.NAME) is OK.
86 if kind == -1:
87 return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB})
88 else:
89 return handle_is_simple_lookup_forward(
90 line, index, {token.LPAR, token.LSQB}
91 )
93 def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool:
94 # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple
95 # lookup (see above), with or without a preceding unary operator.
96 start = line.leaves[index]
97 if start.type in {token.NAME, token.NUMBER}:
98 return is_simple_lookup(index, kind)
100 if start.type in {token.PLUS, token.MINUS, token.TILDE}:
101 if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:
102 # kind is always one as bases with a preceding unary op will be checked
103 # for simplicity starting from the next token (so it'll hit the check
104 # above).
105 return is_simple_lookup(index + 1, kind=1)
107 return False
109 new_line = line.clone()
110 should_hug = False
111 for idx, leaf in enumerate(line.leaves):
112 hug_this_leaf = should_hug
113 should_hug = (
114 (0 < idx < len(line.leaves) - 1)
115 and leaf.type == token.DOUBLESTAR
116 and is_simple_operand(idx - 1, kind=-1)
117 and line.leaves[idx - 1].value != "lambda"
118 and is_simple_operand(idx + 1, kind=1)
119 )
121 if hug_this_leaf or should_hug:
122 new_leaf = leaf.clone()
123 new_leaf.prefix = ""
124 else:
125 # Only the operands around a hugged `**` need a copy. Reuse the leaf
126 # otherwise: a clone has no parent, and the trailing-comma guards in
127 # `Line.append` read the tree to tell a syntactically required comma
128 # (a one-tuple, or a one-element subscript like `a[x,]`) from a magic
129 # one. Without a parent they can't, so the comma was being dropped
130 # under --skip-magic-trailing-comma.
131 new_leaf = leaf
133 # We have to be careful to make a new line properly:
134 # - bracket related metadata must be maintained (handled by Line.append)
135 # - comments need to copied over, updating the leaf IDs they're attached to
136 new_line.append(new_leaf, preformatted=True)
137 for comment_leaf in line.comments_after(leaf):
138 new_line.append(comment_leaf, preformatted=True)
140 yield new_line
143# Remove when `simplify_power_operator_hugging` becomes stable.
144def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool:
145 """
146 Handling the determination of is_simple_lookup for the lines prior to the doublestar
147 token. This is required because of the need to isolate the chained expression
148 to determine the bracket or parenthesis belong to the single expression.
149 """
150 contains_disallowed = False
151 chain = []
153 while 0 <= index < len(line.leaves):
154 current = line.leaves[index]
155 chain.append(current)
156 if not contains_disallowed and current.type in disallowed:
157 contains_disallowed = True
158 if not is_expression_chained(chain):
159 return not contains_disallowed
161 index -= 1
163 return True
166# Remove when `simplify_power_operator_hugging` becomes stable.
167def handle_is_simple_lookup_forward(
168 line: Line, index: int, disallowed: set[int]
169) -> bool:
170 """
171 Handling decision is_simple_lookup for the lines behind the doublestar token.
172 This function is simplified to keep consistent with the prior logic and the forward
173 case are more straightforward and do not need to care about chained expressions.
174 """
175 while 0 <= index < len(line.leaves):
176 current = line.leaves[index]
177 if current.type in disallowed:
178 return False
179 if current.type not in {token.NAME, token.DOT} or (
180 current.type == token.NAME and current.value == "for"
181 ):
182 # If the current token isn't disallowed, we'll assume this is simple as
183 # only the disallowed tokens are semantically attached to this lookup
184 # expression we're checking. Also, stop early if we hit the 'for' bit
185 # of a comprehension.
186 return True
188 index += 1
190 return True
193# Remove when `simplify_power_operator_hugging` becomes stable.
194def is_expression_chained(chained_leaves: list[Leaf]) -> bool:
195 """
196 Function to determine if the variable is a chained call.
197 (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call)
198 """
199 if len(chained_leaves) < 2:
200 return True
202 current_leaf = chained_leaves[-1]
203 past_leaf = chained_leaves[-2]
205 if past_leaf.type == token.NAME:
206 return current_leaf.type in {token.DOT}
207 elif past_leaf.type in {token.RPAR, token.RSQB}:
208 return current_leaf.type in {token.RSQB, token.RPAR}
209 elif past_leaf.type in {token.LPAR, token.LSQB}:
210 return current_leaf.type in {token.NAME, token.LPAR, token.LSQB}
211 else:
212 return False
215class StringTransformer(ABC):
216 """
217 An implementation of the Transformer protocol that relies on its
218 subclasses overriding the template methods `do_match(...)` and
219 `do_transform(...)`.
221 This Transformer works exclusively on strings (for example, by merging
222 or splitting them).
224 The following sections can be found among the docstrings of each concrete
225 StringTransformer subclass.
227 Requirements:
228 Which requirements must be met of the given Line for this
229 StringTransformer to be applied?
231 Transformations:
232 If the given Line meets all of the above requirements, which string
233 transformations can you expect to be applied to it by this
234 StringTransformer?
236 Collaborations:
237 What contractual agreements does this StringTransformer have with other
238 StringTransformers? Such collaborations should be eliminated/minimized
239 as much as possible.
240 """
242 __name__: Final = "StringTransformer"
244 # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
245 # `abc.ABC`.
246 def __init__(self, line_length: int, normalize_strings: bool) -> None:
247 self.line_length = line_length
248 self.normalize_strings = normalize_strings
250 @abstractmethod
251 def do_match(self, line: Line) -> TMatchResult:
252 """
253 Returns:
254 * Ok(string_indices) such that for each index, `line.leaves[index]`
255 is our target string if a match was able to be made. For
256 transformers that don't result in more lines (e.g. StringMerger,
257 StringParenStripper), multiple matches and transforms are done at
258 once to reduce the complexity.
259 OR
260 * Err(CannotTransform), if no match could be made.
261 """
263 @abstractmethod
264 def do_transform(
265 self, line: Line, string_indices: list[int]
266 ) -> Iterator[TResult[Line]]:
267 """
268 Yields:
269 * Ok(new_line) where new_line is the new transformed line.
270 OR
271 * Err(CannotTransform) if the transformation failed for some reason. The
272 `do_match(...)` template method should usually be used to reject
273 the form of the given Line, but in some cases it is difficult to
274 know whether or not a Line meets the StringTransformer's
275 requirements until the transformation is already midway.
277 Side Effects:
278 This method should NOT mutate @line directly, but it MAY mutate the
279 Line's underlying Node structure. (WARNING: If the underlying Node
280 structure IS altered, then this method should NOT be allowed to
281 yield an CannotTransform after that point.)
282 """
284 def __call__(
285 self, line: Line, _features: Collection[Feature], _mode: Mode
286 ) -> Iterator[Line]:
287 """
288 StringTransformer instances have a call signature that mirrors that of
289 the Transformer type.
291 Raises:
292 CannotTransform(...) if the concrete StringTransformer class is unable
293 to transform @line.
294 """
295 # Optimization to avoid calling `self.do_match(...)` when the line does
296 # not contain any string.
297 if not any(leaf.type == token.STRING for leaf in line.leaves):
298 raise CannotTransform("There are no strings in this line.")
300 match_result = self.do_match(line)
302 if isinstance(match_result, Err):
303 cant_transform = match_result.err()
304 raise CannotTransform(
305 f"The string transformer {self.__class__.__name__} does not recognize"
306 " this line as one that it can transform."
307 ) from cant_transform
309 string_indices = match_result.ok()
311 for line_result in self.do_transform(line, string_indices):
312 if isinstance(line_result, Err):
313 cant_transform = line_result.err()
314 raise CannotTransform(
315 "StringTransformer failed while attempting to transform string."
316 ) from cant_transform
317 line = line_result.ok()
318 yield line
321@dataclass
322class CustomSplit:
323 """A custom (i.e. manual) string split.
325 A single CustomSplit instance represents a single substring.
327 Examples:
328 Consider the following string:
329 ```
330 "Hi there friend."
331 " This is a custom"
332 f" string {split}."
333 ```
335 This string will correspond to the following three CustomSplit instances:
336 ```
337 CustomSplit(False, 16)
338 CustomSplit(False, 17)
339 CustomSplit(True, 16)
340 ```
341 """
343 has_prefix: bool
344 break_idx: int
347CustomSplitMapKey = tuple[StringID, str]
350@trait
351class CustomSplitMapMixin:
352 """
353 This mixin class is used to map merged strings to a sequence of
354 CustomSplits, which will then be used to re-split the strings iff none of
355 the resultant substrings go over the configured max line length.
356 """
358 _CUSTOM_SPLIT_MAP: ClassVar[dict[CustomSplitMapKey, tuple[CustomSplit, ...]]] = (
359 defaultdict(tuple)
360 )
362 @staticmethod
363 def _get_key(string: str) -> CustomSplitMapKey:
364 """
365 Returns:
366 A unique identifier that is used internally to map @string to a
367 group of custom splits.
368 """
369 return (id(string), string)
371 def add_custom_splits(
372 self, string: str, custom_splits: Iterable[CustomSplit]
373 ) -> None:
374 """Custom Split Map Setter Method
376 Side Effects:
377 Adds a mapping from @string to the custom splits @custom_splits.
378 """
379 key = self._get_key(string)
380 self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
382 def pop_custom_splits(self, string: str) -> list[CustomSplit]:
383 """Custom Split Map Getter Method
385 Returns:
386 * A list of the custom splits that are mapped to @string, if any
387 exist.
388 OR
389 * [], otherwise.
391 Side Effects:
392 Deletes the mapping between @string and its associated custom
393 splits (which are returned to the caller).
394 """
395 key = self._get_key(string)
397 custom_splits = self._CUSTOM_SPLIT_MAP[key]
398 del self._CUSTOM_SPLIT_MAP[key]
400 return list(custom_splits)
402 def has_custom_splits(self, string: str) -> bool:
403 """
404 Returns:
405 True iff @string is associated with a set of custom splits.
406 """
407 key = self._get_key(string)
408 return key in self._CUSTOM_SPLIT_MAP
411class StringMerger(StringTransformer, CustomSplitMapMixin):
412 """StringTransformer that merges strings together.
414 Requirements:
415 (A) The line contains adjacent strings such that ALL of the validation checks
416 listed in StringMerger._validate_msg(...)'s docstring pass.
417 OR
418 (B) The line contains a string which uses line continuation backslashes.
420 Transformations:
421 Depending on which of the two requirements above where met, either:
423 (A) The string group associated with the target string is merged.
424 OR
425 (B) All line-continuation backslashes are removed from the target string.
427 Collaborations:
428 StringMerger provides custom split information to StringSplitter.
429 """
431 def do_match(self, line: Line) -> TMatchResult:
432 LL = line.leaves
434 is_valid_index = is_valid_index_factory(LL)
436 string_indices = []
437 idx = 0
438 while is_valid_index(idx):
439 leaf = LL[idx]
440 if (
441 leaf.type == token.STRING
442 and is_valid_index(idx + 1)
443 and LL[idx + 1].type == token.STRING
444 ):
445 # Let's check if the string group contains an inline comment
446 # If we have a comment inline, we don't merge the strings
447 contains_comment = False
448 i = idx
449 while is_valid_index(i):
450 if LL[i].type != token.STRING:
451 break
452 if line.comments_after(LL[i]):
453 contains_comment = True
454 break
455 i += 1
457 if not contains_comment and not is_part_of_annotation(leaf):
458 string_indices.append(idx)
460 # Advance to the next non-STRING leaf.
461 idx += 2
462 while is_valid_index(idx) and LL[idx].type == token.STRING:
463 idx += 1
465 elif leaf.type == token.STRING and "\\\n" in leaf.value:
466 string_indices.append(idx)
467 # Advance to the next non-STRING leaf.
468 idx += 1
469 while is_valid_index(idx) and LL[idx].type == token.STRING:
470 idx += 1
472 else:
473 idx += 1
475 if string_indices:
476 return Ok(string_indices)
477 else:
478 return TErr("This line has no strings that need merging.")
480 def do_transform(
481 self, line: Line, string_indices: list[int]
482 ) -> Iterator[TResult[Line]]:
483 new_line = line
485 rblc_result = self._remove_backslash_line_continuation_chars(
486 new_line, string_indices
487 )
488 if isinstance(rblc_result, Ok):
489 new_line = rblc_result.ok()
491 msg_result = self._merge_string_group(new_line, string_indices)
492 if isinstance(msg_result, Ok):
493 new_line = msg_result.ok()
495 if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
496 msg_cant_transform = msg_result.err()
497 rblc_cant_transform = rblc_result.err()
498 cant_transform = CannotTransform(
499 "StringMerger failed to merge any strings in this line."
500 )
502 # Chain the errors together using `__cause__`.
503 msg_cant_transform.__cause__ = rblc_cant_transform
504 cant_transform.__cause__ = msg_cant_transform
506 yield Err(cant_transform)
507 else:
508 yield Ok(new_line)
510 @staticmethod
511 def _remove_backslash_line_continuation_chars(
512 line: Line, string_indices: list[int]
513 ) -> TResult[Line]:
514 """
515 Merge strings that were split across multiple lines using
516 line-continuation backslashes.
518 Returns:
519 Ok(new_line), if @line contains backslash line-continuation
520 characters.
521 OR
522 Err(CannotTransform), otherwise.
523 """
524 LL = line.leaves
526 indices_to_transform = []
527 for string_idx in string_indices:
528 string_leaf = LL[string_idx]
529 if (
530 string_leaf.type == token.STRING
531 and "\\\n" in string_leaf.value
532 and not has_triple_quotes(string_leaf.value)
533 ):
534 indices_to_transform.append(string_idx)
536 if not indices_to_transform:
537 return TErr(
538 "Found no string leaves that contain backslash line continuation"
539 " characters."
540 )
542 new_line = line.clone()
543 new_line.comments = line.comments.copy()
544 append_leaves(new_line, line, LL)
546 for string_idx in indices_to_transform:
547 new_string_leaf = new_line.leaves[string_idx]
548 new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
550 return Ok(new_line)
552 def _merge_string_group(
553 self, line: Line, string_indices: list[int]
554 ) -> TResult[Line]:
555 """
556 Merges string groups (i.e. set of adjacent strings).
558 Each index from `string_indices` designates one string group's first
559 leaf in `line.leaves`.
561 Returns:
562 Ok(new_line), if ALL of the validation checks found in
563 _validate_msg(...) pass.
564 OR
565 Err(CannotTransform), otherwise.
566 """
567 LL = line.leaves
569 is_valid_index = is_valid_index_factory(LL)
571 # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.
572 merged_string_idx_dict: dict[int, tuple[int, Leaf]] = {}
573 for string_idx in string_indices:
574 vresult = self._validate_msg(line, string_idx)
575 if isinstance(vresult, Err):
576 continue
577 merged_string_idx_dict[string_idx] = self._merge_one_string_group(
578 LL, string_idx, is_valid_index
579 )
581 if not merged_string_idx_dict:
582 return TErr("No string group is merged")
584 # Build the final line ('new_line') that this method will later return.
585 new_line = line.clone()
586 previous_merged_string_idx = -1
587 previous_merged_num_of_strings = -1
588 # Leaves outside any merged string group are copied in runs rather than
589 # one at a time. append_leaves resumes the search for each leaf's
590 # position from where the previous sibling of the same parent was found,
591 # so copying a run of leaves that share a parent (the operand tuple of
592 # "%s ..." % (a, b, c, ...)) stays linear; a fresh call per leaf restarts
593 # that search from the front every time and is quadratic in the operands.
594 pending: list[Leaf] = []
595 for i, leaf in enumerate(LL):
596 if i in merged_string_idx_dict:
597 if pending:
598 append_leaves(new_line, line, pending)
599 pending = []
600 previous_merged_string_idx = i
601 previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]
602 new_line.append(string_leaf)
604 if (
605 previous_merged_string_idx
606 <= i
607 < previous_merged_string_idx + previous_merged_num_of_strings
608 ):
609 for comment_leaf in line.comments_after(leaf):
610 new_line.append(comment_leaf, preformatted=True)
611 continue
613 pending.append(leaf)
615 if pending:
616 append_leaves(new_line, line, pending)
618 return Ok(new_line)
620 def _merge_one_string_group(
621 self, LL: list[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]
622 ) -> tuple[int, Leaf]:
623 """
624 Merges one string group where the first string in the group is
625 `LL[string_idx]`.
627 Returns:
628 A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the
629 number of strings merged and `leaf` is the newly merged string
630 to be replaced in the new line.
631 """
632 # If the string group is wrapped inside an Atom node, we must make sure
633 # to later replace that Atom with our new (merged) string leaf.
634 atom_node = LL[string_idx].parent
636 # We will place BREAK_MARK in between every two substrings that we
637 # merge. We will then later go through our final result and use the
638 # various instances of BREAK_MARK we find to add the right values to
639 # the custom split map.
640 BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
642 QUOTE = LL[string_idx].value[-1]
644 def make_naked(string: str, string_prefix: str) -> str:
645 """Strip @string (i.e. make it a "naked" string)
647 Pre-conditions:
648 * assert_is_leaf_string(@string)
650 Returns:
651 A string that is identical to @string except that
652 @string_prefix has been stripped, the surrounding QUOTE
653 characters have been removed, and any remaining QUOTE
654 characters have been escaped.
655 """
656 assert_is_leaf_string(string)
657 if "f" in string_prefix:
658 f_expressions = [
659 string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces
660 for span in iter_fexpr_spans(string)
661 ]
662 debug_expressions_contain_visible_quotes = any(
663 re.search(r"[\'\"].*(?<![!:=])={1}(?!=)(?![^\s:])", expression)
664 for expression in f_expressions
665 )
666 if not debug_expressions_contain_visible_quotes:
667 # We don't want to toggle visible quotes in debug f-strings, as
668 # that would modify the AST
669 string = _toggle_fexpr_quotes(string, QUOTE)
670 # After quotes toggling, quotes in expressions won't be escaped
671 # because quotes can't be reused in f-strings. So we can simply
672 # let the escaping logic below run without knowing f-string
673 # expressions.
675 RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
676 naked_string = string[len(string_prefix) + 1 : -1]
677 naked_string = re.sub(
678 "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
679 )
680 return naked_string
682 # Holds the CustomSplit objects that will later be added to the custom
683 # split map.
684 custom_splits = []
686 # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
687 prefix_tracker = []
689 # Sets the 'prefix' variable. This is the prefix that the final merged
690 # string will have.
691 next_str_idx = string_idx
692 prefix = ""
693 while (
694 not prefix
695 and is_valid_index(next_str_idx)
696 and LL[next_str_idx].type == token.STRING
697 ):
698 prefix = get_string_prefix(LL[next_str_idx].value).lower()
699 next_str_idx += 1
701 # The next loop merges the string group. The final string will be
702 # contained in 'S'.
703 #
704 # The following convenience variables are used:
705 #
706 # S: string
707 # NS: naked string
708 # SS: next string
709 # NSS: naked next string
710 NS = ""
711 num_of_strings = 0
712 next_str_idx = string_idx
713 while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
714 num_of_strings += 1
716 SS = LL[next_str_idx].value
717 next_prefix = get_string_prefix(SS).lower()
719 # If this is an f-string group but this substring is not prefixed
720 # with 'f'...
721 if "f" in prefix and "f" not in next_prefix:
722 # Then we must escape any braces contained in this substring.
723 SS = re.sub(r"(\{|\})", r"\1\1", SS)
725 NSS = make_naked(SS, next_prefix)
727 has_prefix = bool(next_prefix)
728 prefix_tracker.append(has_prefix)
730 # Each NSS is already naked (prefix and quotes stripped, inner quotes
731 # escaped, f-string expression quotes toggled), and the parts are
732 # separated by BREAK_MARK which contains no quote or backslash, so the
733 # naked group is just their concatenation. Re-running make_naked over the
734 # whole accumulated string on every iteration rescans all previously
735 # merged substrings, which is quadratic in the size of the group.
736 NS = NS + NSS + BREAK_MARK
738 next_str_idx += 1
740 # Take a note on the index of the non-STRING leaf.
741 non_string_idx = next_str_idx
743 S = prefix + QUOTE + NS + QUOTE
744 S_leaf = Leaf(token.STRING, S)
745 if self.normalize_strings:
746 S_leaf.value = normalize_string_quotes(S_leaf.value)
748 # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
749 temp_string = S_leaf.value[len(prefix) + 1 : -1]
750 for has_prefix in prefix_tracker:
751 mark_idx = temp_string.find(BREAK_MARK)
752 assert (
753 mark_idx >= 0
754 ), "Logic error while filling the custom string breakpoint cache."
756 temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
757 breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
758 custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
760 string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
762 if atom_node is not None:
763 # If not all children of the atom node are merged (this can happen
764 # when there is a standalone comment in the middle) ...
765 if non_string_idx - string_idx < len(atom_node.children):
766 # We need to replace the old STRING leaves with the new string leaf.
767 first_child_idx = LL[string_idx].remove()
768 for idx in range(string_idx + 1, non_string_idx):
769 LL[idx].remove()
770 if first_child_idx is not None:
771 atom_node.insert_child(first_child_idx, string_leaf)
772 else:
773 # Else replace the atom node with the new string leaf.
774 replace_child(atom_node, string_leaf)
776 self.add_custom_splits(string_leaf.value, custom_splits)
777 return num_of_strings, string_leaf
779 @staticmethod
780 def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
781 """Validate (M)erge (S)tring (G)roup
783 Transform-time string validation logic for _merge_string_group(...).
785 Returns:
786 * Ok(None), if ALL validation checks (listed below) pass.
787 OR
788 * Err(CannotTransform), if any of the following are true:
789 - The target string group does not contain ANY stand-alone comments.
790 - The target string is not in a string group (i.e. it has no
791 adjacent strings).
792 - The string group has more than one inline comment.
793 - The string group has an inline comment that appears to be a pragma.
794 - The set of all string prefixes in the string group is of
795 length greater than one and is not equal to {"", "f"}.
796 - The string group consists of raw strings.
797 - The string group would merge f-strings with different quote types
798 and internal quotes.
799 - The string group is stringified type annotations. We don't want to
800 process stringified type annotations since pyright doesn't support
801 them spanning multiple string values. (NOTE: mypy, pytype, pyre do
802 support them, so we can change if pyright also gains support in the
803 future. See https://github.com/microsoft/pyright/issues/4359.)
804 """
805 # We first check for "inner" stand-alone comments (i.e. stand-alone
806 # comments that have a string leaf before them AND after them).
807 for inc in [1, -1]:
808 i = string_idx
809 found_sa_comment = False
810 is_valid_index = is_valid_index_factory(line.leaves)
811 while is_valid_index(i) and line.leaves[i].type in [
812 token.STRING,
813 STANDALONE_COMMENT,
814 ]:
815 if line.leaves[i].type == STANDALONE_COMMENT:
816 found_sa_comment = True
817 elif found_sa_comment:
818 return TErr(
819 "StringMerger does NOT merge string groups which contain "
820 "stand-alone comments."
821 )
823 i += inc
825 QUOTE = line.leaves[string_idx].value[-1]
827 num_of_inline_string_comments = 0
828 set_of_prefixes = set()
829 num_of_strings = 0
830 for leaf in line.leaves[string_idx:]:
831 if leaf.type != token.STRING:
832 # If the string group is trailed by a comma, we count the
833 # comments trailing the comma to be one of the string group's
834 # comments.
835 if leaf.type == token.COMMA and id(leaf) in line.comments:
836 num_of_inline_string_comments += 1
837 break
839 if has_triple_quotes(leaf.value):
840 return TErr("StringMerger does NOT merge multiline strings.")
842 num_of_strings += 1
843 prefix = get_string_prefix(leaf.value).lower()
844 if "r" in prefix:
845 return TErr("StringMerger does NOT merge raw strings.")
847 set_of_prefixes.add(prefix)
849 if (
850 "f" in prefix
851 and leaf.value[-1] != QUOTE
852 and (
853 "'" in leaf.value[len(prefix) + 1 : -1]
854 or '"' in leaf.value[len(prefix) + 1 : -1]
855 )
856 ):
857 return TErr(
858 "StringMerger does NOT merge f-strings with different quote types"
859 " and internal quotes."
860 )
862 if id(leaf) in line.comments:
863 num_of_inline_string_comments += 1
864 if contains_pragma_comment(line.comments[id(leaf)]):
865 return TErr("Cannot merge strings which have pragma comments.")
867 if num_of_strings < 2:
868 return TErr(
869 f"Not enough strings to merge (num_of_strings={num_of_strings})."
870 )
872 # Also check for pragma comments on tokens that follow the string
873 # group (e.g. a closing bracket). Merging strings when a pragma
874 # comment like `# type: ignore` follows would produce an unsplittable
875 # long line.
876 is_valid_index = is_valid_index_factory(line.leaves)
877 next_idx = string_idx + num_of_strings
878 while is_valid_index(next_idx):
879 next_leaf = line.leaves[next_idx]
880 if id(next_leaf) in line.comments:
881 if contains_pragma_comment(line.comments[id(next_leaf)]):
882 return TErr(
883 "Cannot merge strings when a pragma comment follows"
884 " the string group."
885 )
886 if next_leaf.type not in CLOSING_BRACKETS:
887 break
888 next_idx += 1
890 if num_of_inline_string_comments > 1:
891 return TErr(
892 f"Too many inline string comments ({num_of_inline_string_comments})."
893 )
895 if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
896 return TErr(f"Too many different prefixes ({set_of_prefixes}).")
898 return Ok(None)
901class StringParenStripper(StringTransformer):
902 """StringTransformer that strips surrounding parentheses from strings.
904 Requirements:
905 The line contains a string which is surrounded by parentheses and:
906 - The target string is NOT the only argument to a function call.
907 - The target string is NOT a "pointless" string.
908 - The target string is NOT a dictionary value.
909 - If the target string contains a PERCENT, the brackets are not
910 preceded or followed by an operator with higher precedence than
911 PERCENT.
913 Transformations:
914 The parentheses mentioned in the 'Requirements' section are stripped.
916 Collaborations:
917 StringParenStripper has its own inherent usefulness, but it is also
918 relied on to clean up the parentheses created by StringParenWrapper (in
919 the event that they are no longer needed).
920 """
922 def do_match(self, line: Line) -> TMatchResult:
923 LL = line.leaves
925 is_valid_index = is_valid_index_factory(LL)
927 string_indices = []
929 idx = -1
930 while True:
931 idx += 1
932 if idx >= len(LL):
933 break
934 leaf = LL[idx]
936 # Should be a string...
937 if leaf.type != token.STRING:
938 continue
940 # If this is a "pointless" string...
941 if (
942 leaf.parent
943 and leaf.parent.parent
944 and leaf.parent.parent.type == syms.simple_stmt
945 ):
946 continue
948 # Should be preceded by a non-empty LPAR...
949 if (
950 not is_valid_index(idx - 1)
951 or LL[idx - 1].type != token.LPAR
952 or is_empty_lpar(LL[idx - 1])
953 ):
954 continue
956 # That LPAR should NOT be preceded by a colon (which could be a
957 # dictionary value), function name, or a closing bracket (which
958 # could be a function returning a function or a list/dictionary
959 # containing a function)...
960 if is_valid_index(idx - 2) and (
961 LL[idx - 2].type == token.COLON
962 or LL[idx - 2].type == token.NAME
963 or LL[idx - 2].type in CLOSING_BRACKETS
964 ):
965 continue
967 string_idx = idx
969 # Skip the string trailer, if one exists.
970 string_parser = StringParser()
971 next_idx = string_parser.parse(LL, string_idx)
973 # if the leaves in the parsed string include a PERCENT, we need to
974 # make sure the initial LPAR is NOT preceded by an operator with
975 # higher or equal precedence to PERCENT
976 if is_valid_index(idx - 2):
977 # mypy can't quite follow unless we name this
978 before_lpar = LL[idx - 2]
979 if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
980 (
981 before_lpar.type in {
982 token.STAR,
983 token.AT,
984 token.SLASH,
985 token.DOUBLESLASH,
986 token.PERCENT,
987 token.TILDE,
988 token.DOUBLESTAR,
989 token.AWAIT,
990 token.LSQB,
991 token.LPAR,
992 }
993 )
994 or (
995 # only unary PLUS/MINUS
996 before_lpar.parent
997 and before_lpar.parent.type == syms.factor
998 and (before_lpar.type in {token.PLUS, token.MINUS})
999 )
1000 ):
1001 continue
1003 # Should be followed by a non-empty RPAR...
1004 if (
1005 is_valid_index(next_idx)
1006 and LL[next_idx].type == token.RPAR
1007 and not is_empty_rpar(LL[next_idx])
1008 ):
1009 # That RPAR should NOT be followed by anything with higher
1010 # precedence than PERCENT
1011 if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
1012 token.DOUBLESTAR,
1013 token.LSQB,
1014 token.LPAR,
1015 token.DOT,
1016 }:
1017 continue
1019 string_indices.append(string_idx)
1020 idx = string_idx
1021 while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING:
1022 idx += 1
1024 if string_indices:
1025 return Ok(string_indices)
1026 return TErr("This line has no strings wrapped in parens.")
1028 def do_transform(
1029 self, line: Line, string_indices: list[int]
1030 ) -> Iterator[TResult[Line]]:
1031 LL = line.leaves
1033 string_and_rpar_indices: list[int] = []
1034 for string_idx in string_indices:
1035 string_parser = StringParser()
1036 rpar_idx = string_parser.parse(LL, string_idx)
1038 should_transform = True
1039 for leaf in (LL[string_idx - 1], LL[rpar_idx]):
1040 if line.comments_after(leaf):
1041 # Should not strip parentheses which have comments attached
1042 # to them.
1043 should_transform = False
1044 break
1045 if should_transform:
1046 string_and_rpar_indices.extend((string_idx, rpar_idx))
1048 if string_and_rpar_indices:
1049 yield Ok(self._transform_to_new_line(line, string_and_rpar_indices))
1050 else:
1051 yield Err(
1052 CannotTransform("All string groups have comments attached to them.")
1053 )
1055 def _transform_to_new_line(
1056 self, line: Line, string_and_rpar_indices: list[int]
1057 ) -> Line:
1058 LL = line.leaves
1060 new_line = line.clone()
1061 new_line.comments = line.comments.copy()
1063 previous_idx = -1
1064 # We need to sort the indices, since string_idx and its matching
1065 # rpar_idx may not come in order, e.g. in
1066 # `("outer" % ("inner".join(items)))`, the "inner" string's
1067 # string_idx is smaller than "outer" string's rpar_idx.
1068 for idx in sorted(string_and_rpar_indices):
1069 leaf = LL[idx]
1070 lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx
1071 append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx])
1072 if leaf.type == token.STRING:
1073 string_leaf = Leaf(token.STRING, LL[idx].value)
1074 LL[lpar_or_rpar_idx].remove() # Remove lpar.
1075 replace_child(LL[idx], string_leaf)
1076 new_line.append(string_leaf)
1077 # replace comments
1078 old_comments = new_line.comments.pop(id(LL[idx]), [])
1079 new_line.comments.setdefault(id(string_leaf), []).extend(old_comments)
1080 else:
1081 LL[lpar_or_rpar_idx].remove() # This is a rpar.
1083 previous_idx = idx
1085 # Append the leaves after the last idx:
1086 append_leaves(new_line, line, LL[idx + 1 :])
1088 return new_line
1091class BaseStringSplitter(StringTransformer):
1092 """
1093 Abstract class for StringTransformers which transform a Line's strings by splitting
1094 them or placing them on their own lines where necessary to avoid going over
1095 the configured line length.
1097 Requirements:
1098 * The target string value is responsible for the line going over the
1099 line length limit. It follows that after all of black's other line
1100 split methods have been exhausted, this line (or one of the resulting
1101 lines after all line splits are performed) would still be over the
1102 line_length limit unless we split this string.
1103 AND
1105 * The target string is NOT a "pointless" string (i.e. a string that has
1106 no parent or siblings).
1107 AND
1109 * The target string is not followed by an inline comment that appears
1110 to be a pragma.
1111 AND
1113 * The target string is not a multiline (i.e. triple-quote) string.
1114 """
1116 STRING_OPERATORS: Final = [
1117 token.EQEQUAL,
1118 token.GREATER,
1119 token.GREATEREQUAL,
1120 token.LESS,
1121 token.LESSEQUAL,
1122 token.NOTEQUAL,
1123 token.PERCENT,
1124 token.PLUS,
1125 token.STAR,
1126 ]
1128 @abstractmethod
1129 def do_splitter_match(self, line: Line) -> TMatchResult:
1130 """
1131 BaseStringSplitter asks its clients to override this method instead of
1132 `StringTransformer.do_match(...)`.
1134 Follows the same protocol as `StringTransformer.do_match(...)`.
1136 Refer to `help(StringTransformer.do_match)` for more information.
1137 """
1139 def do_match(self, line: Line) -> TMatchResult:
1140 match_result = self.do_splitter_match(line)
1141 if isinstance(match_result, Err):
1142 return match_result
1144 string_indices = match_result.ok()
1145 assert len(string_indices) == 1, (
1146 f"{self.__class__.__name__} should only find one match at a time, found"
1147 f" {len(string_indices)}"
1148 )
1149 string_idx = string_indices[0]
1150 vresult = self._validate(line, string_idx)
1151 if isinstance(vresult, Err):
1152 return vresult
1154 return match_result
1156 def _validate(self, line: Line, string_idx: int) -> TResult[None]:
1157 """
1158 Checks that @line meets all of the requirements listed in this classes'
1159 docstring. Refer to `help(BaseStringSplitter)` for a detailed
1160 description of those requirements.
1162 Returns:
1163 * Ok(None), if ALL of the requirements are met.
1164 OR
1165 * Err(CannotTransform), if ANY of the requirements are NOT met.
1166 """
1167 LL = line.leaves
1169 string_leaf = LL[string_idx]
1171 max_string_length = self._get_max_string_length(line, string_idx)
1172 if len(string_leaf.value) <= max_string_length:
1173 return TErr(
1174 "The string itself is not what is causing this line to be too long."
1175 )
1177 if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
1178 token.STRING,
1179 token.NEWLINE,
1180 ]:
1181 return TErr(
1182 f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
1183 " no parent)."
1184 )
1186 if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
1187 line.comments[id(line.leaves[string_idx])]
1188 ):
1189 return TErr(
1190 "Line appears to end with an inline pragma comment. Splitting the line"
1191 " could modify the pragma's behavior."
1192 )
1194 if has_triple_quotes(string_leaf.value):
1195 return TErr("We cannot split multiline strings.")
1197 return Ok(None)
1199 def _get_max_string_length(self, line: Line, string_idx: int) -> int:
1200 """
1201 Calculates the max string length used when attempting to determine
1202 whether or not the target string is responsible for causing the line to
1203 go over the line length limit.
1205 WARNING: This method is tightly coupled to both StringSplitter and
1206 (especially) StringParenWrapper. There is probably a better way to
1207 accomplish what is being done here.
1209 Returns:
1210 max_string_length: such that `line.leaves[string_idx].value >
1211 max_string_length` implies that the target string IS responsible
1212 for causing this line to exceed the line length limit.
1213 """
1214 LL = line.leaves
1216 is_valid_index = is_valid_index_factory(LL)
1218 # We use the shorthand "WMA4" in comments to abbreviate "We must
1219 # account for". When giving examples, we use STRING to mean some/any
1220 # valid string.
1221 #
1222 # Finally, we use the following convenience variables:
1223 #
1224 # P: The leaf that is before the target string leaf.
1225 # N: The leaf that is after the target string leaf.
1226 # NN: The leaf that is after N.
1228 # WMA4 the whitespace at the beginning of the line.
1229 offset = line.depth * 4
1231 if is_valid_index(string_idx - 1):
1232 p_idx = string_idx - 1
1233 if (
1234 LL[string_idx - 1].type == token.LPAR
1235 and LL[string_idx - 1].value == ""
1236 and string_idx >= 2
1237 ):
1238 # If the previous leaf is an empty LPAR placeholder, we should skip it.
1239 p_idx -= 1
1241 P = LL[p_idx]
1242 if P.type in self.STRING_OPERATORS:
1243 # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
1244 offset += len(str(P)) + 1
1246 if P.type == token.COMMA:
1247 # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
1248 offset += 3
1250 if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
1251 # This conditional branch is meant to handle dictionary keys,
1252 # variable assignments, 'return STRING' statement lines, and
1253 # 'else STRING' ternary expression lines.
1255 # WMA4 a single space.
1256 offset += 1
1258 # WMA4 the lengths of any leaves that came before that space,
1259 # but after any closing bracket before that space.
1260 for leaf in reversed(LL[: p_idx + 1]):
1261 offset += len(str(leaf))
1262 if leaf.type in CLOSING_BRACKETS:
1263 break
1265 if is_valid_index(string_idx + 1):
1266 N = LL[string_idx + 1]
1267 if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
1268 # If the next leaf is an empty RPAR placeholder, we should skip it.
1269 N = LL[string_idx + 2]
1271 if N.type == token.COMMA:
1272 # WMA4 a single comma at the end of the string (e.g `STRING,`).
1273 offset += 1
1275 if is_valid_index(string_idx + 2):
1276 NN = LL[string_idx + 2]
1278 if N.type == token.DOT and NN.type == token.NAME:
1279 # This conditional branch is meant to handle method calls invoked
1280 # off of a string literal up to and including the LPAR character.
1282 # WMA4 the '.' character.
1283 offset += 1
1285 if (
1286 is_valid_index(string_idx + 3)
1287 and LL[string_idx + 3].type == token.LPAR
1288 ):
1289 # WMA4 the left parenthesis character.
1290 offset += 1
1292 # WMA4 the length of the method's name.
1293 offset += len(NN.value)
1295 has_comments = False
1296 for comment_leaf in line.comments_after(LL[string_idx]):
1297 if not has_comments:
1298 has_comments = True
1299 # WMA4 two spaces before the '#' character.
1300 offset += 2
1302 # WMA4 the length of the inline comment.
1303 offset += len(comment_leaf.value)
1305 max_string_length = count_chars_in_width(str(line), self.line_length - offset)
1306 return max_string_length
1308 @staticmethod
1309 def _prefer_paren_wrap_match(LL: list[Leaf]) -> int | None:
1310 """
1311 Returns:
1312 string_idx such that @LL[string_idx] is equal to our target (i.e.
1313 matched) string, if this line matches the "prefer paren wrap" statement
1314 requirements listed in the 'Requirements' section of the StringParenWrapper
1315 class's docstring.
1316 OR
1317 None, otherwise.
1318 """
1319 # The line must start with a string.
1320 if LL[0].type != token.STRING:
1321 return None
1323 matching_nodes = [
1324 syms.listmaker,
1325 syms.dictsetmaker,
1326 syms.testlist_gexp,
1327 ]
1328 # If the string is an immediate child of a list/set/tuple literal...
1329 if (
1330 parent_type(LL[0]) in matching_nodes
1331 or parent_type(LL[0].parent) in matching_nodes
1332 ):
1333 # And the string is surrounded by commas (or is the first/last child)...
1334 prev_sibling = LL[0].prev_sibling
1335 next_sibling = LL[0].next_sibling
1336 if (
1337 not prev_sibling
1338 and not next_sibling
1339 and parent_type(LL[0]) == syms.atom
1340 ):
1341 # If it's an atom string, we need to check the parent atom's siblings.
1342 parent = LL[0].parent
1343 assert parent is not None # For type checkers.
1344 prev_sibling = parent.prev_sibling
1345 next_sibling = parent.next_sibling
1346 if (not prev_sibling or prev_sibling.type == token.COMMA) and (
1347 not next_sibling or next_sibling.type == token.COMMA
1348 ):
1349 return 0
1351 return None
1354def iter_fexpr_spans(s: str) -> Iterator[tuple[int, int]]:
1355 """
1356 Yields spans corresponding to expressions in a given f-string.
1357 Spans are half-open ranges (left inclusive, right exclusive).
1358 Assumes the input string is a valid f-string, but will not crash if the input
1359 string is invalid.
1360 """
1361 stack: list[int] = [] # our curly paren stack
1362 i = 0
1363 while i < len(s):
1364 if s[i] == "{":
1365 # if we're in a string part of the f-string, ignore escaped curly braces
1366 if not stack and i + 1 < len(s) and s[i + 1] == "{":
1367 i += 2
1368 continue
1369 stack.append(i)
1370 i += 1
1371 continue
1373 if s[i] == "}":
1374 if not stack:
1375 i += 1
1376 continue
1377 j = stack.pop()
1378 # we've made it back out of the expression! yield the span
1379 if not stack:
1380 yield (j, i + 1)
1381 i += 1
1382 continue
1384 # if we're in an expression part of the f-string, fast-forward through strings
1385 # note that backslashes are not legal in the expression portion of f-strings
1386 if stack:
1387 delim = None
1388 if s[i : i + 3] in ("'''", '"""'):
1389 delim = s[i : i + 3]
1390 elif s[i] in ("'", '"'):
1391 delim = s[i]
1392 if delim:
1393 i += len(delim)
1394 while i < len(s) and s[i : i + len(delim)] != delim:
1395 i += 1
1396 i += len(delim)
1397 continue
1398 i += 1
1401def fstring_contains_expr(s: str) -> bool:
1402 return any(iter_fexpr_spans(s))
1405def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:
1406 """
1407 Toggles quotes used in f-string expressions that are `old_quote`.
1409 f-string expressions can't contain backslashes, so we need to toggle the
1410 quotes if the f-string itself will end up using the same quote. We can
1411 simply toggle without escaping because, quotes can't be reused in f-string
1412 expressions. They will fail to parse.
1414 NOTE: If PEP 701 is accepted, above statement will no longer be true.
1415 Though if quotes can be reused, we can simply reuse them without updates or
1416 escaping, once Black figures out how to parse the new grammar.
1417 """
1418 new_quote = "'" if old_quote == '"' else '"'
1419 parts = []
1420 previous_index = 0
1421 for start, end in iter_fexpr_spans(fstring):
1422 parts.append(fstring[previous_index:start])
1423 parts.append(fstring[start:end].replace(old_quote, new_quote))
1424 previous_index = end
1425 parts.append(fstring[previous_index:])
1426 return "".join(parts)
1429class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
1430 """
1431 StringTransformer that splits "atom" strings (i.e. strings which exist on
1432 lines by themselves).
1434 Requirements:
1435 * The line consists ONLY of a single string (possibly prefixed by a
1436 string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE
1437 a trailing comma.
1438 AND
1439 * All of the requirements listed in BaseStringSplitter's docstring.
1441 Transformations:
1442 The string mentioned in the 'Requirements' section is split into as
1443 many substrings as necessary to adhere to the configured line length.
1445 In the final set of substrings, no substring should be smaller than
1446 MIN_SUBSTR_SIZE characters.
1448 The string will ONLY be split on spaces (i.e. each new substring should
1449 start with a space). Note that the string will NOT be split on a space
1450 which is escaped with a backslash.
1452 If the string is an f-string, it will NOT be split in the middle of an
1453 f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
1454 else bar()} is an f-expression).
1456 If the string that is being split has an associated set of custom split
1457 records and those custom splits will NOT result in any line going over
1458 the configured line length, those custom splits are used. Otherwise the
1459 string is split as late as possible (from left-to-right) while still
1460 adhering to the transformation rules listed above.
1462 Collaborations:
1463 StringSplitter relies on StringMerger to construct the appropriate
1464 CustomSplit objects and add them to the custom split map.
1465 """
1467 MIN_SUBSTR_SIZE: Final = 6
1469 def do_splitter_match(self, line: Line) -> TMatchResult:
1470 LL = line.leaves
1472 if self._prefer_paren_wrap_match(LL) is not None:
1473 return TErr("Line needs to be wrapped in parens first.")
1475 # If the line is just STRING + COMMA (a one-item tuple) and not inside
1476 # brackets, we need to defer to StringParenWrapper to wrap it first.
1477 # Otherwise, splitting the string would create multiple expressions where
1478 # only the last has the comma, breaking AST equivalence. See issue #4912.
1479 if (
1480 not line.inside_brackets
1481 and len(LL) == 2
1482 and LL[0].type == token.STRING
1483 and LL[1].type == token.COMMA
1484 ):
1485 return TErr(
1486 "Line with trailing comma tuple needs to be wrapped in parens first."
1487 )
1489 is_valid_index = is_valid_index_factory(LL)
1491 idx = 0
1493 # The first two leaves MAY be the 'not in' keywords...
1494 if (
1495 is_valid_index(idx)
1496 and is_valid_index(idx + 1)
1497 and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]
1498 and str(LL[idx]) + str(LL[idx + 1]) == "not in"
1499 ):
1500 idx += 2
1501 # Else the first leaf MAY be a string operator symbol or the 'in' keyword...
1502 elif is_valid_index(idx) and (
1503 LL[idx].type in self.STRING_OPERATORS
1504 or LL[idx].type == token.NAME
1505 and str(LL[idx]) == "in"
1506 ):
1507 idx += 1
1509 # The next/first leaf MAY be an empty LPAR...
1510 if is_valid_index(idx) and is_empty_lpar(LL[idx]):
1511 idx += 1
1513 # The next/first leaf MUST be a string...
1514 if not is_valid_index(idx) or LL[idx].type != token.STRING:
1515 return TErr("Line does not start with a string.")
1517 string_idx = idx
1519 # Skip the string trailer, if one exists.
1520 string_parser = StringParser()
1521 idx = string_parser.parse(LL, string_idx)
1523 # That string MAY be followed by an empty RPAR...
1524 if is_valid_index(idx) and is_empty_rpar(LL[idx]):
1525 idx += 1
1527 # That string / empty RPAR leaf MAY be followed by a comma...
1528 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1529 idx += 1
1531 # But no more leaves are allowed...
1532 if is_valid_index(idx):
1533 return TErr("This line does not end with a string.")
1535 return Ok([string_idx])
1537 def do_transform(
1538 self, line: Line, string_indices: list[int]
1539 ) -> Iterator[TResult[Line]]:
1540 LL = line.leaves
1541 assert len(string_indices) == 1, (
1542 f"{self.__class__.__name__} should only find one match at a time, found"
1543 f" {len(string_indices)}"
1544 )
1545 string_idx = string_indices[0]
1547 QUOTE = LL[string_idx].value[-1]
1549 is_valid_index = is_valid_index_factory(LL)
1550 insert_str_child = insert_str_child_factory(LL[string_idx])
1552 prefix = get_string_prefix(LL[string_idx].value).lower()
1554 # We MAY choose to drop the 'f' prefix from substrings that don't
1555 # contain any f-expressions, but ONLY if the original f-string
1556 # contains at least one f-expression. Otherwise, we will alter the AST
1557 # of the program.
1558 drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr(
1559 LL[string_idx].value
1560 )
1562 first_string_line = True
1564 string_op_leaves = self._get_string_operator_leaves(LL)
1565 string_op_leaves_length = (
1566 sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1
1567 if string_op_leaves
1568 else 0
1569 )
1571 def maybe_append_string_operators(new_line: Line) -> None:
1572 """
1573 Side Effects:
1574 If @line starts with a string operator and this is the first
1575 line we are constructing, this function appends the string
1576 operator to @new_line and replaces the old string operator leaf
1577 in the node structure. Otherwise this function does nothing.
1578 """
1579 maybe_prefix_leaves = string_op_leaves if first_string_line else []
1580 for i, prefix_leaf in enumerate(maybe_prefix_leaves):
1581 replace_child(LL[i], prefix_leaf)
1582 new_line.append(prefix_leaf)
1584 ends_with_comma = (
1585 is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
1586 )
1588 def max_last_string_column() -> int:
1589 """
1590 Returns:
1591 The max allowed width of the string value used for the last
1592 line we will construct. Note that this value means the width
1593 rather than the number of characters (e.g., many East Asian
1594 characters expand to two columns).
1595 """
1596 result = self.line_length
1597 result -= line.depth * 4
1598 result -= 1 if ends_with_comma else 0
1599 result -= string_op_leaves_length
1600 return result
1602 # --- Calculate Max Break Width (for string value)
1603 # We start with the line length limit
1604 max_break_width = self.line_length
1605 # The last index of a string of length N is N-1.
1606 max_break_width -= 1
1607 # Leading whitespace is not present in the string value (e.g. Leaf.value).
1608 max_break_width -= line.depth * 4
1609 if max_break_width < 0:
1610 yield TErr(
1611 f"Unable to split {LL[string_idx].value} at such high of a line depth:"
1612 f" {line.depth}"
1613 )
1614 return
1616 # Check if StringMerger registered any custom splits.
1617 custom_splits = self.pop_custom_splits(LL[string_idx].value)
1618 # We use them ONLY if none of them would produce lines that exceed the
1619 # line limit.
1620 use_custom_breakpoints = bool(
1621 custom_splits
1622 and all(csplit.break_idx <= max_break_width for csplit in custom_splits)
1623 )
1625 # Temporary storage for the remaining chunk of the string line that
1626 # can't fit onto the line currently being constructed.
1627 rest_value = LL[string_idx].value
1629 # Each substring is a suffix of this value (plus the prefix and quote),
1630 # so a missing "\N" here means no substring can hold a named escape and
1631 # the per-substring scan for them can be skipped entirely.
1632 has_named_escape = "\\N" in rest_value
1634 def more_splits_should_be_made() -> bool:
1635 """
1636 Returns:
1637 True iff `rest_value` (the remaining string value from the last
1638 split), should be split again.
1639 """
1640 if use_custom_breakpoints:
1641 return len(custom_splits) > 1
1642 else:
1643 return str_width(rest_value) > max_last_string_column()
1645 string_line_results: list[Ok[Line]] = []
1646 while more_splits_should_be_made():
1647 if use_custom_breakpoints:
1648 # Custom User Split (manual)
1649 csplit = custom_splits.pop(0)
1650 break_idx = csplit.break_idx
1651 else:
1652 # Algorithmic Split (automatic)
1653 max_bidx = (
1654 count_chars_in_width(rest_value, max_break_width)
1655 - string_op_leaves_length
1656 )
1657 maybe_break_idx = self._get_break_idx(
1658 rest_value, max_bidx, has_named_escape
1659 )
1660 if maybe_break_idx is None:
1661 # If we are unable to algorithmically determine a good split
1662 # and this string has custom splits registered to it, we
1663 # fall back to using them--which means we have to start
1664 # over from the beginning.
1665 if custom_splits:
1666 rest_value = LL[string_idx].value
1667 string_line_results = []
1668 first_string_line = True
1669 use_custom_breakpoints = True
1670 continue
1672 # Otherwise, we stop splitting here.
1673 break
1675 break_idx = maybe_break_idx
1677 # --- Construct `next_value`
1678 next_value = rest_value[:break_idx] + QUOTE
1680 # HACK: The following 'if' statement is a hack to fix the custom
1681 # breakpoint index in the case of either: (a) substrings that were
1682 # f-strings but will have the 'f' prefix removed OR (b) substrings
1683 # that were not f-strings but will now become f-strings because of
1684 # redundant use of the 'f' prefix (i.e. none of the substrings
1685 # contain f-expressions but one or more of them had the 'f' prefix
1686 # anyway; in which case, we will prepend 'f' to _all_ substrings).
1687 #
1688 # There is probably a better way to accomplish what is being done
1689 # here...
1690 #
1691 # If this substring is an f-string, we _could_ remove the 'f'
1692 # prefix, and the current custom split did NOT originally use a
1693 # prefix...
1694 if (
1695 use_custom_breakpoints
1696 and not csplit.has_prefix
1697 and (
1698 # `next_value == prefix + QUOTE` happens when the custom
1699 # split is an empty string.
1700 next_value == prefix + QUOTE
1701 or next_value != self._normalize_f_string(next_value, prefix)
1702 )
1703 ):
1704 # Then `csplit.break_idx` will be off by one after removing
1705 # the 'f' prefix.
1706 break_idx += 1
1707 next_value = rest_value[:break_idx] + QUOTE
1709 if drop_pointless_f_prefix:
1710 next_value = self._normalize_f_string(next_value, prefix)
1712 # --- Construct `next_leaf`
1713 next_leaf = Leaf(token.STRING, next_value)
1714 insert_str_child(next_leaf)
1715 self._maybe_normalize_string_quotes(next_leaf)
1717 # --- Construct `next_line`
1718 next_line = line.clone()
1719 maybe_append_string_operators(next_line)
1720 next_line.append(next_leaf)
1721 string_line_results.append(Ok(next_line))
1723 rest_value = prefix + QUOTE + rest_value[break_idx:]
1724 first_string_line = False
1726 yield from string_line_results
1728 if drop_pointless_f_prefix:
1729 rest_value = self._normalize_f_string(rest_value, prefix)
1731 rest_leaf = Leaf(token.STRING, rest_value)
1732 insert_str_child(rest_leaf)
1734 # NOTE: I could not find a test case that verifies that the following
1735 # line is actually necessary, but it seems to be. Otherwise we risk
1736 # not normalizing the last substring, right?
1737 self._maybe_normalize_string_quotes(rest_leaf)
1739 last_line = line.clone()
1740 maybe_append_string_operators(last_line)
1742 # If there are any leaves to the right of the target string...
1743 if is_valid_index(string_idx + 1):
1744 # We use `temp_value` here to determine how long the last line
1745 # would be if we were to append all the leaves to the right of the
1746 # target string to the last string line.
1747 temp_value = rest_value
1748 for leaf in LL[string_idx + 1 :]:
1749 temp_value += str(leaf)
1750 if leaf.type == token.LPAR:
1751 break
1753 # Try to fit them all on the same line with the last substring...
1754 if (
1755 str_width(temp_value) <= max_last_string_column()
1756 or LL[string_idx + 1].type == token.COMMA
1757 ):
1758 last_line.append(rest_leaf)
1759 append_leaves(last_line, line, LL[string_idx + 1 :])
1760 yield Ok(last_line)
1761 # Otherwise, place the last substring on one line and everything
1762 # else on a line below that...
1763 else:
1764 last_line.append(rest_leaf)
1765 yield Ok(last_line)
1767 non_string_line = line.clone()
1768 append_leaves(non_string_line, line, LL[string_idx + 1 :])
1769 yield Ok(non_string_line)
1770 # Else the target string was the last leaf...
1771 else:
1772 last_line.append(rest_leaf)
1773 last_line.comments = line.comments.copy()
1774 yield Ok(last_line)
1776 def _iter_nameescape_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1777 r"""
1778 Yields:
1779 All ranges of @string which, if @string were to be split there,
1780 would result in the splitting of an \N{...} expression (which is NOT
1781 allowed).
1782 """
1783 # True - the previous backslash was unescaped
1784 # False - the previous backslash was escaped *or* there was no backslash
1785 previous_was_unescaped_backslash = False
1786 it = iter(enumerate(string))
1787 for idx, c in it:
1788 if c == "\\":
1789 previous_was_unescaped_backslash = not previous_was_unescaped_backslash
1790 continue
1791 if not previous_was_unescaped_backslash or c != "N":
1792 previous_was_unescaped_backslash = False
1793 continue
1794 previous_was_unescaped_backslash = False
1796 begin = idx - 1 # the position of backslash before \N{...}
1797 for idx, c in it:
1798 if c == "}":
1799 end = idx
1800 break
1801 else:
1802 # malformed nameescape expression?
1803 # should have been detected by AST parsing earlier...
1804 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
1805 yield begin, end
1807 def _iter_fexpr_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1808 """
1809 Yields:
1810 All ranges of @string which, if @string were to be split there,
1811 would result in the splitting of an f-expression (which is NOT
1812 allowed).
1813 """
1814 if "f" not in get_string_prefix(string).lower():
1815 return
1816 yield from iter_fexpr_spans(string)
1818 def _get_illegal_split_indices(
1819 self, string: str, has_named_escape: bool = True
1820 ) -> set[Index]:
1821 illegal_indices: set[Index] = set()
1822 iterators = [self._iter_fexpr_slices(string)]
1823 # Scanning for \N{...} ranges walks the whole string, so skip it when the
1824 # caller already knows the string cannot contain a named escape.
1825 if has_named_escape:
1826 iterators.append(self._iter_nameescape_slices(string))
1827 for it in iterators:
1828 for begin, end in it:
1829 illegal_indices.update(range(begin, end))
1830 return illegal_indices
1832 def _get_break_idx(
1833 self, string: str, max_break_idx: int, has_named_escape: bool = True
1834 ) -> int | None:
1835 """
1836 This method contains the algorithm that StringSplitter uses to
1837 determine which character to split each string at.
1839 Args:
1840 @string: The substring that we are attempting to split.
1841 @max_break_idx: The ideal break index. We will return this value if it
1842 meets all the necessary conditions. In the likely event that it
1843 doesn't we will try to find the closest index BELOW @max_break_idx
1844 that does. If that fails, we will expand our search by also
1845 considering all valid indices ABOVE @max_break_idx.
1846 @has_named_escape: Whether the original string can contain a named
1847 escape. False lets us skip the per-substring scan for them.
1849 Pre-Conditions:
1850 * assert_is_leaf_string(@string)
1851 * 0 <= @max_break_idx < len(@string)
1853 Returns:
1854 break_idx, if an index is able to be found that meets all of the
1855 conditions listed in the 'Transformations' section of this classes'
1856 docstring.
1857 OR
1858 None, otherwise.
1859 """
1860 is_valid_index = is_valid_index_factory(string)
1862 assert is_valid_index(max_break_idx)
1863 assert_is_leaf_string(string)
1865 _illegal_split_indices = self._get_illegal_split_indices(
1866 string, has_named_escape
1867 )
1869 def breaks_unsplittable_expression(i: Index) -> bool:
1870 """
1871 Returns:
1872 True iff returning @i would result in the splitting of an
1873 unsplittable expression (which is NOT allowed).
1874 """
1875 return i in _illegal_split_indices
1877 def passes_all_checks(i: Index) -> bool:
1878 """
1879 Returns:
1880 True iff ALL of the conditions listed in the 'Transformations'
1881 section of this classes' docstring would be met by returning @i.
1882 """
1883 is_space = string[i] == " "
1884 is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS
1886 is_not_escaped = True
1887 j = i - 1
1888 while is_valid_index(j) and string[j] == "\\":
1889 is_not_escaped = not is_not_escaped
1890 j -= 1
1892 is_big_enough = (
1893 len(string) - i >= self.MIN_SUBSTR_SIZE and i >= self.MIN_SUBSTR_SIZE
1894 )
1895 return (
1896 (is_space or is_split_safe)
1897 and is_not_escaped
1898 and is_big_enough
1899 and not breaks_unsplittable_expression(i)
1900 )
1902 # First, we check all indices BELOW @max_break_idx.
1903 break_idx = max_break_idx
1904 while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
1905 break_idx -= 1
1907 if not passes_all_checks(break_idx):
1908 # If that fails, we check all indices ABOVE @max_break_idx.
1909 #
1910 # If we are able to find a valid index here, the next line is going
1911 # to be longer than the specified line length, but it's probably
1912 # better than doing nothing at all.
1913 break_idx = max_break_idx + 1
1914 while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
1915 break_idx += 1
1917 if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
1918 return None
1920 return break_idx
1922 def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
1923 if self.normalize_strings:
1924 leaf.value = normalize_string_quotes(leaf.value)
1926 def _normalize_f_string(self, string: str, prefix: str) -> str:
1927 """
1928 Pre-Conditions:
1929 * assert_is_leaf_string(@string)
1931 Returns:
1932 * If @string is an f-string that contains no f-expressions, we
1933 return a string identical to @string except that the 'f' prefix
1934 has been stripped and all double braces (i.e. '{{' or '}}') have
1935 been normalized (i.e. turned into '{' or '}').
1936 OR
1937 * Otherwise, we return @string.
1938 """
1939 assert_is_leaf_string(string)
1941 if "f" in prefix and not fstring_contains_expr(string):
1942 new_prefix = prefix.replace("f", "")
1944 temp = string[len(prefix) :]
1945 temp = re.sub(r"\{\{", "{", temp)
1946 temp = re.sub(r"\}\}", "}", temp)
1947 new_string = temp
1949 return f"{new_prefix}{new_string}"
1950 else:
1951 return string
1953 def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> list[Leaf]:
1954 LL = list(leaves)
1956 string_op_leaves = []
1957 i = 0
1958 while LL[i].type in self.STRING_OPERATORS + [token.NAME]:
1959 prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())
1960 string_op_leaves.append(prefix_leaf)
1961 i += 1
1962 return string_op_leaves
1965class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):
1966 """
1967 StringTransformer that wraps strings in parens and then splits at the LPAR.
1969 Requirements:
1970 All of the requirements listed in BaseStringSplitter's docstring in
1971 addition to the requirements listed below:
1973 * The line is a return/yield statement, which returns/yields a string.
1974 OR
1975 * The line is part of a ternary expression (e.g. `x = y if cond else
1976 z`) such that the line starts with `else <string>`, where <string> is
1977 some string.
1978 OR
1979 * The line is an assert statement, which ends with a string.
1980 OR
1981 * The line is an assignment statement (e.g. `x = <string>` or `x +=
1982 <string>`) such that the variable is being assigned the value of some
1983 string.
1984 OR
1985 * The line is a dictionary key assignment where some valid key is being
1986 assigned the value of some string.
1987 OR
1988 * The line is a lambda expression and the value is a string.
1989 OR
1990 * The line starts with an "atom" string that prefers to be wrapped in
1991 parens. It's preferred to be wrapped when it's is an immediate child of
1992 a list/set/tuple literal, AND the string is surrounded by commas (or is
1993 the first/last child).
1995 Transformations:
1996 The chosen string is wrapped in parentheses and then split at the LPAR.
1998 We then have one line which ends with an LPAR and another line that
1999 starts with the chosen string. The latter line is then split again at
2000 the RPAR. This results in the RPAR (and possibly a trailing comma)
2001 being placed on its own line.
2003 NOTE: If any leaves exist to the right of the chosen string (except
2004 for a trailing comma, which would be placed after the RPAR), those
2005 leaves are placed inside the parentheses. In effect, the chosen
2006 string is not necessarily being "wrapped" by parentheses. We can,
2007 however, count on the LPAR being placed directly before the chosen
2008 string.
2010 In other words, StringParenWrapper creates "atom" strings. These
2011 can then be split again by StringSplitter, if necessary.
2013 Collaborations:
2014 In the event that a string line split by StringParenWrapper is
2015 changed such that it no longer needs to be given its own line,
2016 StringParenWrapper relies on StringParenStripper to clean up the
2017 parentheses it created.
2019 For "atom" strings that prefers to be wrapped in parens, it requires
2020 StringSplitter to hold the split until the string is wrapped in parens.
2021 """
2023 def do_splitter_match(self, line: Line) -> TMatchResult:
2024 LL = line.leaves
2026 if line.leaves[-1].type in OPENING_BRACKETS:
2027 return TErr(
2028 "Cannot wrap parens around a line that ends in an opening bracket."
2029 )
2031 string_idx = (
2032 self._return_match(LL)
2033 or self._else_match(LL)
2034 or self._assert_match(LL)
2035 or self._assign_match(LL)
2036 or self._dict_or_lambda_match(LL)
2037 )
2039 if string_idx is None:
2040 string_idx = self._trailing_comma_tuple_match(line)
2042 if string_idx is None:
2043 string_idx = self._prefer_paren_wrap_match(LL)
2045 if string_idx is not None:
2046 string_value = line.leaves[string_idx].value
2047 # If the string has neither spaces nor East Asian stops...
2048 if not any(
2049 char == " " or char in SPLIT_SAFE_CHARS for char in string_value
2050 ):
2051 # And will still violate the line length limit when split...
2052 max_string_width = self.line_length - ((line.depth + 1) * 4)
2053 if str_width(string_value) > max_string_width:
2054 # And has no associated custom splits...
2055 if not self.has_custom_splits(string_value):
2056 # Then we should NOT put this string on its own line.
2057 return TErr(
2058 "We do not wrap long strings in parentheses when the"
2059 " resultant line would still be over the specified line"
2060 " length and can't be split further by StringSplitter."
2061 )
2062 return Ok([string_idx])
2064 return TErr("This line does not contain any non-atomic strings.")
2066 @staticmethod
2067 def _return_match(LL: list[Leaf]) -> int | None:
2068 """
2069 Returns:
2070 string_idx such that @LL[string_idx] is equal to our target (i.e.
2071 matched) string, if this line matches the return/yield statement
2072 requirements listed in the 'Requirements' section of this classes'
2073 docstring.
2074 OR
2075 None, otherwise.
2076 """
2077 # If this line is a part of a return/yield statement and the first leaf
2078 # contains either the "return" or "yield" keywords...
2079 if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
2080 0
2081 ].value in ["return", "yield"]:
2082 is_valid_index = is_valid_index_factory(LL)
2084 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2085 # The next visible leaf MUST contain a string...
2086 if is_valid_index(idx) and LL[idx].type == token.STRING:
2087 return idx
2089 return None
2091 @staticmethod
2092 def _else_match(LL: list[Leaf]) -> int | None:
2093 """
2094 Returns:
2095 string_idx such that @LL[string_idx] is equal to our target (i.e.
2096 matched) string, if this line matches the ternary expression
2097 requirements listed in the 'Requirements' section of this classes'
2098 docstring.
2099 OR
2100 None, otherwise.
2101 """
2102 # If this line is a part of a ternary expression and the first leaf
2103 # contains the "else" keyword...
2104 if (
2105 parent_type(LL[0]) == syms.test
2106 and LL[0].type == token.NAME
2107 and LL[0].value == "else"
2108 ):
2109 is_valid_index = is_valid_index_factory(LL)
2111 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2112 # The next visible leaf MUST contain a string...
2113 if is_valid_index(idx) and LL[idx].type == token.STRING:
2114 return idx
2116 return None
2118 @staticmethod
2119 def _assert_match(LL: list[Leaf]) -> int | None:
2120 """
2121 Returns:
2122 string_idx such that @LL[string_idx] is equal to our target (i.e.
2123 matched) string, if this line matches the assert statement
2124 requirements listed in the 'Requirements' section of this classes'
2125 docstring.
2126 OR
2127 None, otherwise.
2128 """
2129 # If this line is a part of an assert statement and the first leaf
2130 # contains the "assert" keyword...
2131 if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
2132 is_valid_index = is_valid_index_factory(LL)
2134 for i, leaf in enumerate(LL):
2135 # We MUST find a comma...
2136 if leaf.type == token.COMMA:
2137 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2139 # That comma MUST be followed by a string...
2140 if is_valid_index(idx) and LL[idx].type == token.STRING:
2141 string_idx = idx
2143 # Skip the string trailer, if one exists.
2144 string_parser = StringParser()
2145 idx = string_parser.parse(LL, string_idx)
2147 # But no more leaves are allowed...
2148 if not is_valid_index(idx):
2149 return string_idx
2151 return None
2153 @staticmethod
2154 def _assign_match(LL: list[Leaf]) -> int | None:
2155 """
2156 Returns:
2157 string_idx such that @LL[string_idx] is equal to our target (i.e.
2158 matched) string, if this line matches the assignment statement
2159 requirements listed in the 'Requirements' section of this classes'
2160 docstring.
2161 OR
2162 None, otherwise.
2163 """
2164 # If this line is a part of an expression statement or is a function
2165 # argument AND the first leaf contains a variable name...
2166 if (
2167 parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
2168 and LL[0].type == token.NAME
2169 ):
2170 is_valid_index = is_valid_index_factory(LL)
2172 for i, leaf in enumerate(LL):
2173 # We MUST find either an '=' or '+=' symbol...
2174 if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
2175 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2177 # That symbol MUST be followed by a string...
2178 if is_valid_index(idx) and LL[idx].type == token.STRING:
2179 string_idx = idx
2181 # Skip the string trailer, if one exists.
2182 string_parser = StringParser()
2183 idx = string_parser.parse(LL, string_idx)
2185 # The next leaf MAY be a comma iff this line is a part
2186 # of a function argument...
2187 if (
2188 parent_type(LL[0]) == syms.argument
2189 and is_valid_index(idx)
2190 and LL[idx].type == token.COMMA
2191 ):
2192 idx += 1
2194 # But no more leaves are allowed...
2195 if not is_valid_index(idx):
2196 return string_idx
2198 return None
2200 @staticmethod
2201 def _dict_or_lambda_match(LL: list[Leaf]) -> int | None:
2202 """
2203 Returns:
2204 string_idx such that @LL[string_idx] is equal to our target (i.e.
2205 matched) string, if this line matches the dictionary key assignment
2206 statement or lambda expression requirements listed in the
2207 'Requirements' section of this classes' docstring.
2208 OR
2209 None, otherwise.
2210 """
2211 # If this line is a part of a dictionary key assignment or lambda expression...
2212 parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)]
2213 if syms.dictsetmaker in parent_types or syms.lambdef in parent_types:
2214 is_valid_index = is_valid_index_factory(LL)
2216 for i, leaf in enumerate(LL):
2217 # We MUST find a colon, it can either be dict's or lambda's colon...
2218 if leaf.type == token.COLON and i < len(LL) - 1:
2219 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2221 # That colon MUST be followed by a string...
2222 if is_valid_index(idx) and LL[idx].type == token.STRING:
2223 string_idx = idx
2225 # Skip the string trailer, if one exists.
2226 string_parser = StringParser()
2227 idx = string_parser.parse(LL, string_idx)
2229 # That string MAY be followed by a comma...
2230 if is_valid_index(idx) and LL[idx].type == token.COMMA:
2231 idx += 1
2233 # But no more leaves are allowed...
2234 if not is_valid_index(idx):
2235 return string_idx
2237 return None
2239 @staticmethod
2240 def _trailing_comma_tuple_match(line: Line) -> int | None:
2241 """
2242 Returns:
2243 string_idx such that @line.leaves[string_idx] is equal to our target
2244 (i.e. matched) string, if the line is a bare trailing comma tuple
2245 (STRING + COMMA) not inside brackets.
2246 OR
2247 None, otherwise.
2249 This handles the case from issue #4912 where a long string with a
2250 trailing comma (making it a one-item tuple) needs to be wrapped in
2251 parentheses before splitting to preserve AST equivalence.
2252 """
2253 LL = line.leaves
2254 # Match: STRING followed by COMMA, not inside brackets
2255 if (
2256 not line.inside_brackets
2257 and len(LL) == 2
2258 and LL[0].type == token.STRING
2259 and LL[1].type == token.COMMA
2260 ):
2261 return 0
2263 return None
2265 def do_transform(
2266 self, line: Line, string_indices: list[int]
2267 ) -> Iterator[TResult[Line]]:
2268 LL = line.leaves
2269 assert len(string_indices) == 1, (
2270 f"{self.__class__.__name__} should only find one match at a time, found"
2271 f" {len(string_indices)}"
2272 )
2273 string_idx = string_indices[0]
2275 is_valid_index = is_valid_index_factory(LL)
2276 insert_str_child = insert_str_child_factory(LL[string_idx])
2278 comma_idx = -1
2279 ends_with_comma = False
2280 if LL[comma_idx].type == token.COMMA:
2281 ends_with_comma = True
2283 leaves_to_steal_comments_from = [LL[string_idx]]
2284 if ends_with_comma:
2285 leaves_to_steal_comments_from.append(LL[comma_idx])
2287 # --- First Line
2288 first_line = line.clone()
2289 left_leaves = LL[:string_idx]
2291 # We have to remember to account for (possibly invisible) LPAR and RPAR
2292 # leaves that already wrapped the target string. If these leaves do
2293 # exist, we will replace them with our own LPAR and RPAR leaves.
2294 old_parens_exist = False
2295 if left_leaves and left_leaves[-1].type == token.LPAR:
2296 old_parens_exist = True
2297 leaves_to_steal_comments_from.append(left_leaves[-1])
2298 left_leaves.pop()
2300 append_leaves(first_line, line, left_leaves)
2302 lpar_leaf = Leaf(token.LPAR, "(")
2303 if old_parens_exist:
2304 replace_child(LL[string_idx - 1], lpar_leaf)
2305 else:
2306 insert_str_child(lpar_leaf)
2307 first_line.append(lpar_leaf)
2309 # We throw inline comments that were originally to the right of the
2310 # target string to the top line. They will now be shown to the right of
2311 # the LPAR.
2312 for leaf in leaves_to_steal_comments_from:
2313 for comment_leaf in line.comments_after(leaf):
2314 first_line.append(comment_leaf, preformatted=True)
2316 yield Ok(first_line)
2318 # --- Middle (String) Line
2319 # We only need to yield one (possibly too long) string line, since the
2320 # `StringSplitter` will break it down further if necessary.
2321 string_value = LL[string_idx].value
2322 string_line = Line(
2323 mode=line.mode,
2324 depth=line.depth + 1,
2325 inside_brackets=True,
2326 should_split_rhs=line.should_split_rhs,
2327 magic_trailing_comma=line.magic_trailing_comma,
2328 )
2329 string_leaf = Leaf(token.STRING, string_value)
2330 insert_str_child(string_leaf)
2331 string_line.append(string_leaf)
2333 old_rpar_leaf = None
2334 if is_valid_index(string_idx + 1):
2335 right_leaves = LL[string_idx + 1 :]
2336 if ends_with_comma:
2337 right_leaves.pop()
2339 if old_parens_exist:
2340 assert right_leaves and right_leaves[-1].type == token.RPAR, (
2341 "Apparently, old parentheses do NOT exist?!"
2342 f" (left_leaves={left_leaves}, right_leaves={right_leaves})"
2343 )
2344 old_rpar_leaf = right_leaves.pop()
2345 elif right_leaves and right_leaves[-1].type == token.RPAR:
2346 # Special case for lambda expressions as dict's value, e.g.:
2347 # my_dict = {
2348 # "key": lambda x: f"formatted: {x}",
2349 # }
2350 # After wrapping the dict's value with parentheses, the string is
2351 # followed by a RPAR but its opening bracket is lambda's, not
2352 # the string's:
2353 # "key": (lambda x: f"formatted: {x}"),
2354 opening_bracket = right_leaves[-1].opening_bracket
2355 if opening_bracket is not None and opening_bracket in left_leaves:
2356 index = left_leaves.index(opening_bracket)
2357 if (
2358 0 < index < len(left_leaves) - 1
2359 and left_leaves[index - 1].type == token.COLON
2360 and left_leaves[index + 1].value == "lambda"
2361 ):
2362 right_leaves.pop()
2364 append_leaves(string_line, line, right_leaves)
2366 yield Ok(string_line)
2368 # --- Last Line
2369 last_line = line.clone()
2370 last_line.bracket_tracker = first_line.bracket_tracker
2372 new_rpar_leaf = Leaf(token.RPAR, ")")
2373 if old_rpar_leaf is not None:
2374 replace_child(old_rpar_leaf, new_rpar_leaf)
2375 else:
2376 insert_str_child(new_rpar_leaf)
2377 last_line.append(new_rpar_leaf)
2379 # If the target string ended with a comma, we place this comma to the
2380 # right of the RPAR on the last line.
2381 if ends_with_comma:
2382 comma_leaf = Leaf(token.COMMA, ",")
2383 replace_child(LL[comma_idx], comma_leaf)
2384 last_line.append(comma_leaf)
2386 yield Ok(last_line)
2389class StringParser:
2390 """
2391 A state machine that aids in parsing a string's "trailer", which can be
2392 either non-existent, an old-style formatting sequence (e.g. `% varX` or `%
2393 (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
2394 varY)`).
2396 NOTE: A new StringParser object MUST be instantiated for each string
2397 trailer we need to parse.
2399 Examples:
2400 We shall assume that `line` equals the `Line` object that corresponds
2401 to the following line of python code:
2402 ```
2403 x = "Some {}.".format("String") + some_other_string
2404 ```
2406 Furthermore, we will assume that `string_idx` is some index such that:
2407 ```
2408 assert line.leaves[string_idx].value == "Some {}."
2409 ```
2411 The following code snippet then holds:
2412 ```
2413 string_parser = StringParser()
2414 idx = string_parser.parse(line.leaves, string_idx)
2415 assert line.leaves[idx].type == token.PLUS
2416 ```
2417 """
2419 DEFAULT_TOKEN: Final = 20210605
2421 # String Parser States
2422 START: Final = 1
2423 DOT: Final = 2
2424 NAME: Final = 3
2425 PERCENT: Final = 4
2426 SINGLE_FMT_ARG: Final = 5
2427 LPAR: Final = 6
2428 RPAR: Final = 7
2429 DONE: Final = 8
2431 # Lookup Table for Next State
2432 _goto: Final[dict[tuple[ParserState, NodeType], ParserState]] = {
2433 # A string trailer may start with '.' OR '%'.
2434 (START, token.DOT): DOT,
2435 (START, token.PERCENT): PERCENT,
2436 (START, DEFAULT_TOKEN): DONE,
2437 # A '.' MUST be followed by an attribute or method name.
2438 (DOT, token.NAME): NAME,
2439 # A method name MUST be followed by an '(', whereas an attribute name
2440 # is the last symbol in the string trailer.
2441 (NAME, token.LPAR): LPAR,
2442 (NAME, DEFAULT_TOKEN): DONE,
2443 # A '%' symbol can be followed by an '(' or a single argument (e.g. a
2444 # string or variable name).
2445 (PERCENT, token.LPAR): LPAR,
2446 (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
2447 # If a '%' symbol is followed by a single argument, that argument is
2448 # the last leaf in the string trailer.
2449 (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
2450 # If present, a ')' symbol is the last symbol in a string trailer.
2451 # (NOTE: LPARS and nested RPARS are not included in this lookup table,
2452 # since they are treated as a special case by the parsing logic in this
2453 # classes' implementation.)
2454 (RPAR, DEFAULT_TOKEN): DONE,
2455 }
2457 def __init__(self) -> None:
2458 self._state = self.START
2459 self._unmatched_lpars = 0
2461 def parse(self, leaves: list[Leaf], string_idx: int) -> int:
2462 """
2463 Pre-conditions:
2464 * @leaves[@string_idx].type == token.STRING
2466 Returns:
2467 The index directly after the last leaf which is a part of the string
2468 trailer, if a "trailer" exists.
2469 OR
2470 @string_idx + 1, if no string "trailer" exists.
2471 """
2472 assert leaves[string_idx].type == token.STRING
2474 idx = string_idx + 1
2475 while idx < len(leaves) and self._next_state(leaves[idx]):
2476 idx += 1
2477 return idx
2479 def _next_state(self, leaf: Leaf) -> bool:
2480 """
2481 Pre-conditions:
2482 * On the first call to this function, @leaf MUST be the leaf that
2483 was directly after the string leaf in question (e.g. if our target
2484 string is `line.leaves[i]` then the first call to this method must
2485 be `line.leaves[i + 1]`).
2486 * On the next call to this function, the leaf parameter passed in
2487 MUST be the leaf directly following @leaf.
2489 Returns:
2490 True iff @leaf is a part of the string's trailer.
2491 """
2492 # We ignore empty LPAR or RPAR leaves.
2493 if is_empty_par(leaf):
2494 return True
2496 next_token = leaf.type
2497 if next_token == token.LPAR:
2498 self._unmatched_lpars += 1
2500 current_state = self._state
2502 # The LPAR parser state is a special case. We will return True until we
2503 # find the matching RPAR token.
2504 if current_state == self.LPAR:
2505 if next_token == token.RPAR:
2506 self._unmatched_lpars -= 1
2507 if self._unmatched_lpars == 0:
2508 self._state = self.RPAR
2509 # Otherwise, we use a lookup table to determine the next state.
2510 else:
2511 # If the lookup table matches the current state to the next
2512 # token, we use the lookup table.
2513 if (current_state, next_token) in self._goto:
2514 self._state = self._goto[current_state, next_token]
2515 else:
2516 # Otherwise, we check if the current state was assigned a
2517 # default.
2518 if (current_state, self.DEFAULT_TOKEN) in self._goto:
2519 self._state = self._goto[current_state, self.DEFAULT_TOKEN]
2520 # If no default has been assigned, then this parser has a logic
2521 # error.
2522 else:
2523 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
2525 if self._state == self.DONE:
2526 return False
2528 return True
2531def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
2532 """
2533 Factory for a convenience function that is used to orphan @string_leaf
2534 and then insert multiple new leaves into the same part of the node
2535 structure that @string_leaf had originally occupied.
2537 Examples:
2538 Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
2539 string_leaf.parent`. Assume the node `N` has the following
2540 original structure:
2542 Node(
2543 expr_stmt, [
2544 Leaf(NAME, 'x'),
2545 Leaf(EQUAL, '='),
2546 Leaf(STRING, '"foo"'),
2547 ]
2548 )
2550 We then run the code snippet shown below.
2551 ```
2552 insert_str_child = insert_str_child_factory(string_leaf)
2554 lpar = Leaf(token.LPAR, '(')
2555 insert_str_child(lpar)
2557 bar = Leaf(token.STRING, '"bar"')
2558 insert_str_child(bar)
2560 rpar = Leaf(token.RPAR, ')')
2561 insert_str_child(rpar)
2562 ```
2564 After which point, it follows that `string_leaf.parent is None` and
2565 the node `N` now has the following structure:
2567 Node(
2568 expr_stmt, [
2569 Leaf(NAME, 'x'),
2570 Leaf(EQUAL, '='),
2571 Leaf(LPAR, '('),
2572 Leaf(STRING, '"bar"'),
2573 Leaf(RPAR, ')'),
2574 ]
2575 )
2576 """
2577 string_parent = string_leaf.parent
2578 string_child_idx = string_leaf.remove()
2580 def insert_str_child(child: LN) -> None:
2581 nonlocal string_child_idx
2583 assert string_parent is not None
2584 assert string_child_idx is not None
2586 string_parent.insert_child(string_child_idx, child)
2587 string_child_idx += 1
2589 return insert_str_child
2592def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
2593 """
2594 Examples:
2595 ```
2596 my_list = [1, 2, 3]
2598 is_valid_index = is_valid_index_factory(my_list)
2600 assert is_valid_index(0)
2601 assert is_valid_index(2)
2603 assert not is_valid_index(3)
2604 assert not is_valid_index(-1)
2605 ```
2606 """
2608 def is_valid_index(idx: int) -> bool:
2609 """
2610 Returns:
2611 True iff @idx is positive AND seq[@idx] does NOT raise an
2612 IndexError.
2613 """
2614 return 0 <= idx < len(seq)
2616 return is_valid_index