Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/trans.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2String transformers that can split and merge strings.
3"""
5import re
6from abc import ABC, abstractmethod
7from collections import defaultdict
8from collections.abc import Callable, Collection, Iterable, Iterator, Sequence
9from dataclasses import dataclass
10from typing import Any, ClassVar, Final, Literal, TypeVar, Union
12from mypy_extensions import trait
14from black.comments import contains_pragma_comment
15from black.lines import Line, append_leaves
16from black.mode import Feature, Mode
17from black.nodes import (
18 CLOSING_BRACKETS,
19 OPENING_BRACKETS,
20 STANDALONE_COMMENT,
21 is_empty_lpar,
22 is_empty_par,
23 is_empty_rpar,
24 is_part_of_annotation,
25 parent_type,
26 replace_child,
27 syms,
28)
29from black.rusty import Err, Ok, Result
30from black.strings import (
31 assert_is_leaf_string,
32 count_chars_in_width,
33 get_string_prefix,
34 has_triple_quotes,
35 normalize_string_quotes,
36 str_width,
37)
38from blib2to3.pgen2 import token
39from blib2to3.pytree import Leaf, Node
42class CannotTransform(Exception):
43 """Base class for errors raised by Transformers."""
46# types
47T = TypeVar("T")
48LN = Union[Leaf, Node]
49Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]
50Index = int
51NodeType = int
52ParserState = int
53StringID = int
54TResult = Result[T, CannotTransform] # (T)ransform Result
55TMatchResult = TResult[list[Index]]
57SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops
60def TErr(err_msg: str) -> Err[CannotTransform]:
61 """(T)ransform Err
63 Convenience function used when working with the TResult type.
64 """
65 cant_transform = CannotTransform(err_msg)
66 return Err(cant_transform)
69# Remove when `simplify_power_operator_hugging` becomes stable.
70def hug_power_op(
71 line: Line, features: Collection[Feature], mode: Mode
72) -> Iterator[Line]:
73 """A transformer which normalizes spacing around power operators."""
75 # Performance optimization to avoid unnecessary Leaf clones and other ops.
76 for leaf in line.leaves:
77 if leaf.type == token.DOUBLESTAR:
78 break
79 else:
80 raise CannotTransform("No doublestar token was found in the line.")
82 def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool:
83 # Brackets and parentheses indicate calls, subscripts, etc. ...
84 # basically stuff that doesn't count as "simple". Only a NAME lookup
85 # or dotted lookup (eg. NAME.NAME) is OK.
86 if kind == -1:
87 return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB})
88 else:
89 return handle_is_simple_lookup_forward(
90 line, index, {token.LPAR, token.LSQB}
91 )
93 def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool:
94 # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple
95 # lookup (see above), with or without a preceding unary operator.
96 start = line.leaves[index]
97 if start.type in {token.NAME, token.NUMBER}:
98 return is_simple_lookup(index, kind)
100 if start.type in {token.PLUS, token.MINUS, token.TILDE}:
101 if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:
102 # kind is always one as bases with a preceding unary op will be checked
103 # for simplicity starting from the next token (so it'll hit the check
104 # above).
105 return is_simple_lookup(index + 1, kind=1)
107 return False
109 new_line = line.clone()
110 should_hug = False
111 for idx, leaf in enumerate(line.leaves):
112 new_leaf = leaf.clone()
113 if should_hug:
114 new_leaf.prefix = ""
115 should_hug = False
117 should_hug = (
118 (0 < idx < len(line.leaves) - 1)
119 and leaf.type == token.DOUBLESTAR
120 and is_simple_operand(idx - 1, kind=-1)
121 and line.leaves[idx - 1].value != "lambda"
122 and is_simple_operand(idx + 1, kind=1)
123 )
124 if should_hug:
125 new_leaf.prefix = ""
127 # We have to be careful to make a new line properly:
128 # - bracket related metadata must be maintained (handled by Line.append)
129 # - comments need to copied over, updating the leaf IDs they're attached to
130 new_line.append(new_leaf, preformatted=True)
131 for comment_leaf in line.comments_after(leaf):
132 new_line.append(comment_leaf, preformatted=True)
134 yield new_line
137# Remove when `simplify_power_operator_hugging` becomes stable.
138def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool:
139 """
140 Handling the determination of is_simple_lookup for the lines prior to the doublestar
141 token. This is required because of the need to isolate the chained expression
142 to determine the bracket or parenthesis belong to the single expression.
143 """
144 contains_disallowed = False
145 chain = []
147 while 0 <= index < len(line.leaves):
148 current = line.leaves[index]
149 chain.append(current)
150 if not contains_disallowed and current.type in disallowed:
151 contains_disallowed = True
152 if not is_expression_chained(chain):
153 return not contains_disallowed
155 index -= 1
157 return True
160# Remove when `simplify_power_operator_hugging` becomes stable.
161def handle_is_simple_lookup_forward(
162 line: Line, index: int, disallowed: set[int]
163) -> bool:
164 """
165 Handling decision is_simple_lookup for the lines behind the doublestar token.
166 This function is simplified to keep consistent with the prior logic and the forward
167 case are more straightforward and do not need to care about chained expressions.
168 """
169 while 0 <= index < len(line.leaves):
170 current = line.leaves[index]
171 if current.type in disallowed:
172 return False
173 if current.type not in {token.NAME, token.DOT} or (
174 current.type == token.NAME and current.value == "for"
175 ):
176 # If the current token isn't disallowed, we'll assume this is simple as
177 # only the disallowed tokens are semantically attached to this lookup
178 # expression we're checking. Also, stop early if we hit the 'for' bit
179 # of a comprehension.
180 return True
182 index += 1
184 return True
187# Remove when `simplify_power_operator_hugging` becomes stable.
188def is_expression_chained(chained_leaves: list[Leaf]) -> bool:
189 """
190 Function to determine if the variable is a chained call.
191 (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call)
192 """
193 if len(chained_leaves) < 2:
194 return True
196 current_leaf = chained_leaves[-1]
197 past_leaf = chained_leaves[-2]
199 if past_leaf.type == token.NAME:
200 return current_leaf.type in {token.DOT}
201 elif past_leaf.type in {token.RPAR, token.RSQB}:
202 return current_leaf.type in {token.RSQB, token.RPAR}
203 elif past_leaf.type in {token.LPAR, token.LSQB}:
204 return current_leaf.type in {token.NAME, token.LPAR, token.LSQB}
205 else:
206 return False
209class StringTransformer(ABC):
210 """
211 An implementation of the Transformer protocol that relies on its
212 subclasses overriding the template methods `do_match(...)` and
213 `do_transform(...)`.
215 This Transformer works exclusively on strings (for example, by merging
216 or splitting them).
218 The following sections can be found among the docstrings of each concrete
219 StringTransformer subclass.
221 Requirements:
222 Which requirements must be met of the given Line for this
223 StringTransformer to be applied?
225 Transformations:
226 If the given Line meets all of the above requirements, which string
227 transformations can you expect to be applied to it by this
228 StringTransformer?
230 Collaborations:
231 What contractual agreements does this StringTransformer have with other
232 StringTransformers? Such collaborations should be eliminated/minimized
233 as much as possible.
234 """
236 __name__: Final = "StringTransformer"
238 # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
239 # `abc.ABC`.
240 def __init__(self, line_length: int, normalize_strings: bool) -> None:
241 self.line_length = line_length
242 self.normalize_strings = normalize_strings
244 @abstractmethod
245 def do_match(self, line: Line) -> TMatchResult:
246 """
247 Returns:
248 * Ok(string_indices) such that for each index, `line.leaves[index]`
249 is our target string if a match was able to be made. For
250 transformers that don't result in more lines (e.g. StringMerger,
251 StringParenStripper), multiple matches and transforms are done at
252 once to reduce the complexity.
253 OR
254 * Err(CannotTransform), if no match could be made.
255 """
257 @abstractmethod
258 def do_transform(
259 self, line: Line, string_indices: list[int]
260 ) -> Iterator[TResult[Line]]:
261 """
262 Yields:
263 * Ok(new_line) where new_line is the new transformed line.
264 OR
265 * Err(CannotTransform) if the transformation failed for some reason. The
266 `do_match(...)` template method should usually be used to reject
267 the form of the given Line, but in some cases it is difficult to
268 know whether or not a Line meets the StringTransformer's
269 requirements until the transformation is already midway.
271 Side Effects:
272 This method should NOT mutate @line directly, but it MAY mutate the
273 Line's underlying Node structure. (WARNING: If the underlying Node
274 structure IS altered, then this method should NOT be allowed to
275 yield an CannotTransform after that point.)
276 """
278 def __call__(
279 self, line: Line, _features: Collection[Feature], _mode: Mode
280 ) -> Iterator[Line]:
281 """
282 StringTransformer instances have a call signature that mirrors that of
283 the Transformer type.
285 Raises:
286 CannotTransform(...) if the concrete StringTransformer class is unable
287 to transform @line.
288 """
289 # Optimization to avoid calling `self.do_match(...)` when the line does
290 # not contain any string.
291 if not any(leaf.type == token.STRING for leaf in line.leaves):
292 raise CannotTransform("There are no strings in this line.")
294 match_result = self.do_match(line)
296 if isinstance(match_result, Err):
297 cant_transform = match_result.err()
298 raise CannotTransform(
299 f"The string transformer {self.__class__.__name__} does not recognize"
300 " this line as one that it can transform."
301 ) from cant_transform
303 string_indices = match_result.ok()
305 for line_result in self.do_transform(line, string_indices):
306 if isinstance(line_result, Err):
307 cant_transform = line_result.err()
308 raise CannotTransform(
309 "StringTransformer failed while attempting to transform string."
310 ) from cant_transform
311 line = line_result.ok()
312 yield line
315@dataclass
316class CustomSplit:
317 """A custom (i.e. manual) string split.
319 A single CustomSplit instance represents a single substring.
321 Examples:
322 Consider the following string:
323 ```
324 "Hi there friend."
325 " This is a custom"
326 f" string {split}."
327 ```
329 This string will correspond to the following three CustomSplit instances:
330 ```
331 CustomSplit(False, 16)
332 CustomSplit(False, 17)
333 CustomSplit(True, 16)
334 ```
335 """
337 has_prefix: bool
338 break_idx: int
341CustomSplitMapKey = tuple[StringID, str]
344@trait
345class CustomSplitMapMixin:
346 """
347 This mixin class is used to map merged strings to a sequence of
348 CustomSplits, which will then be used to re-split the strings iff none of
349 the resultant substrings go over the configured max line length.
350 """
352 _CUSTOM_SPLIT_MAP: ClassVar[dict[CustomSplitMapKey, tuple[CustomSplit, ...]]] = (
353 defaultdict(tuple)
354 )
356 @staticmethod
357 def _get_key(string: str) -> CustomSplitMapKey:
358 """
359 Returns:
360 A unique identifier that is used internally to map @string to a
361 group of custom splits.
362 """
363 return (id(string), string)
365 def add_custom_splits(
366 self, string: str, custom_splits: Iterable[CustomSplit]
367 ) -> None:
368 """Custom Split Map Setter Method
370 Side Effects:
371 Adds a mapping from @string to the custom splits @custom_splits.
372 """
373 key = self._get_key(string)
374 self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
376 def pop_custom_splits(self, string: str) -> list[CustomSplit]:
377 """Custom Split Map Getter Method
379 Returns:
380 * A list of the custom splits that are mapped to @string, if any
381 exist.
382 OR
383 * [], otherwise.
385 Side Effects:
386 Deletes the mapping between @string and its associated custom
387 splits (which are returned to the caller).
388 """
389 key = self._get_key(string)
391 custom_splits = self._CUSTOM_SPLIT_MAP[key]
392 del self._CUSTOM_SPLIT_MAP[key]
394 return list(custom_splits)
396 def has_custom_splits(self, string: str) -> bool:
397 """
398 Returns:
399 True iff @string is associated with a set of custom splits.
400 """
401 key = self._get_key(string)
402 return key in self._CUSTOM_SPLIT_MAP
405class StringMerger(StringTransformer, CustomSplitMapMixin):
406 """StringTransformer that merges strings together.
408 Requirements:
409 (A) The line contains adjacent strings such that ALL of the validation checks
410 listed in StringMerger._validate_msg(...)'s docstring pass.
411 OR
412 (B) The line contains a string which uses line continuation backslashes.
414 Transformations:
415 Depending on which of the two requirements above where met, either:
417 (A) The string group associated with the target string is merged.
418 OR
419 (B) All line-continuation backslashes are removed from the target string.
421 Collaborations:
422 StringMerger provides custom split information to StringSplitter.
423 """
425 def do_match(self, line: Line) -> TMatchResult:
426 LL = line.leaves
428 is_valid_index = is_valid_index_factory(LL)
430 string_indices = []
431 idx = 0
432 while is_valid_index(idx):
433 leaf = LL[idx]
434 if (
435 leaf.type == token.STRING
436 and is_valid_index(idx + 1)
437 and LL[idx + 1].type == token.STRING
438 ):
439 # Let's check if the string group contains an inline comment
440 # If we have a comment inline, we don't merge the strings
441 contains_comment = False
442 i = idx
443 while is_valid_index(i):
444 if LL[i].type != token.STRING:
445 break
446 if line.comments_after(LL[i]):
447 contains_comment = True
448 break
449 i += 1
451 if not contains_comment and not is_part_of_annotation(leaf):
452 string_indices.append(idx)
454 # Advance to the next non-STRING leaf.
455 idx += 2
456 while is_valid_index(idx) and LL[idx].type == token.STRING:
457 idx += 1
459 elif leaf.type == token.STRING and "\\\n" in leaf.value:
460 string_indices.append(idx)
461 # Advance to the next non-STRING leaf.
462 idx += 1
463 while is_valid_index(idx) and LL[idx].type == token.STRING:
464 idx += 1
466 else:
467 idx += 1
469 if string_indices:
470 return Ok(string_indices)
471 else:
472 return TErr("This line has no strings that need merging.")
474 def do_transform(
475 self, line: Line, string_indices: list[int]
476 ) -> Iterator[TResult[Line]]:
477 new_line = line
479 rblc_result = self._remove_backslash_line_continuation_chars(
480 new_line, string_indices
481 )
482 if isinstance(rblc_result, Ok):
483 new_line = rblc_result.ok()
485 msg_result = self._merge_string_group(new_line, string_indices)
486 if isinstance(msg_result, Ok):
487 new_line = msg_result.ok()
489 if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
490 msg_cant_transform = msg_result.err()
491 rblc_cant_transform = rblc_result.err()
492 cant_transform = CannotTransform(
493 "StringMerger failed to merge any strings in this line."
494 )
496 # Chain the errors together using `__cause__`.
497 msg_cant_transform.__cause__ = rblc_cant_transform
498 cant_transform.__cause__ = msg_cant_transform
500 yield Err(cant_transform)
501 else:
502 yield Ok(new_line)
504 @staticmethod
505 def _remove_backslash_line_continuation_chars(
506 line: Line, string_indices: list[int]
507 ) -> TResult[Line]:
508 """
509 Merge strings that were split across multiple lines using
510 line-continuation backslashes.
512 Returns:
513 Ok(new_line), if @line contains backslash line-continuation
514 characters.
515 OR
516 Err(CannotTransform), otherwise.
517 """
518 LL = line.leaves
520 indices_to_transform = []
521 for string_idx in string_indices:
522 string_leaf = LL[string_idx]
523 if (
524 string_leaf.type == token.STRING
525 and "\\\n" in string_leaf.value
526 and not has_triple_quotes(string_leaf.value)
527 ):
528 indices_to_transform.append(string_idx)
530 if not indices_to_transform:
531 return TErr(
532 "Found no string leaves that contain backslash line continuation"
533 " characters."
534 )
536 new_line = line.clone()
537 new_line.comments = line.comments.copy()
538 append_leaves(new_line, line, LL)
540 for string_idx in indices_to_transform:
541 new_string_leaf = new_line.leaves[string_idx]
542 new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
544 return Ok(new_line)
546 def _merge_string_group(
547 self, line: Line, string_indices: list[int]
548 ) -> TResult[Line]:
549 """
550 Merges string groups (i.e. set of adjacent strings).
552 Each index from `string_indices` designates one string group's first
553 leaf in `line.leaves`.
555 Returns:
556 Ok(new_line), if ALL of the validation checks found in
557 _validate_msg(...) pass.
558 OR
559 Err(CannotTransform), otherwise.
560 """
561 LL = line.leaves
563 is_valid_index = is_valid_index_factory(LL)
565 # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.
566 merged_string_idx_dict: dict[int, tuple[int, Leaf]] = {}
567 for string_idx in string_indices:
568 vresult = self._validate_msg(line, string_idx)
569 if isinstance(vresult, Err):
570 continue
571 merged_string_idx_dict[string_idx] = self._merge_one_string_group(
572 LL, string_idx, is_valid_index
573 )
575 if not merged_string_idx_dict:
576 return TErr("No string group is merged")
578 # Build the final line ('new_line') that this method will later return.
579 new_line = line.clone()
580 previous_merged_string_idx = -1
581 previous_merged_num_of_strings = -1
582 for i, leaf in enumerate(LL):
583 if i in merged_string_idx_dict:
584 previous_merged_string_idx = i
585 previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]
586 new_line.append(string_leaf)
588 if (
589 previous_merged_string_idx
590 <= i
591 < previous_merged_string_idx + previous_merged_num_of_strings
592 ):
593 for comment_leaf in line.comments_after(leaf):
594 new_line.append(comment_leaf, preformatted=True)
595 continue
597 append_leaves(new_line, line, [leaf])
599 return Ok(new_line)
601 def _merge_one_string_group(
602 self, LL: list[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]
603 ) -> tuple[int, Leaf]:
604 """
605 Merges one string group where the first string in the group is
606 `LL[string_idx]`.
608 Returns:
609 A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the
610 number of strings merged and `leaf` is the newly merged string
611 to be replaced in the new line.
612 """
613 # If the string group is wrapped inside an Atom node, we must make sure
614 # to later replace that Atom with our new (merged) string leaf.
615 atom_node = LL[string_idx].parent
617 # We will place BREAK_MARK in between every two substrings that we
618 # merge. We will then later go through our final result and use the
619 # various instances of BREAK_MARK we find to add the right values to
620 # the custom split map.
621 BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
623 QUOTE = LL[string_idx].value[-1]
625 def make_naked(string: str, string_prefix: str) -> str:
626 """Strip @string (i.e. make it a "naked" string)
628 Pre-conditions:
629 * assert_is_leaf_string(@string)
631 Returns:
632 A string that is identical to @string except that
633 @string_prefix has been stripped, the surrounding QUOTE
634 characters have been removed, and any remaining QUOTE
635 characters have been escaped.
636 """
637 assert_is_leaf_string(string)
638 if "f" in string_prefix:
639 f_expressions = [
640 string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces
641 for span in iter_fexpr_spans(string)
642 ]
643 debug_expressions_contain_visible_quotes = any(
644 re.search(r"[\'\"].*(?<![!:=])={1}(?!=)(?![^\s:])", expression)
645 for expression in f_expressions
646 )
647 if not debug_expressions_contain_visible_quotes:
648 # We don't want to toggle visible quotes in debug f-strings, as
649 # that would modify the AST
650 string = _toggle_fexpr_quotes(string, QUOTE)
651 # After quotes toggling, quotes in expressions won't be escaped
652 # because quotes can't be reused in f-strings. So we can simply
653 # let the escaping logic below run without knowing f-string
654 # expressions.
656 RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
657 naked_string = string[len(string_prefix) + 1 : -1]
658 naked_string = re.sub(
659 "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
660 )
661 return naked_string
663 # Holds the CustomSplit objects that will later be added to the custom
664 # split map.
665 custom_splits = []
667 # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
668 prefix_tracker = []
670 # Sets the 'prefix' variable. This is the prefix that the final merged
671 # string will have.
672 next_str_idx = string_idx
673 prefix = ""
674 while (
675 not prefix
676 and is_valid_index(next_str_idx)
677 and LL[next_str_idx].type == token.STRING
678 ):
679 prefix = get_string_prefix(LL[next_str_idx].value).lower()
680 next_str_idx += 1
682 # The next loop merges the string group. The final string will be
683 # contained in 'S'.
684 #
685 # The following convenience variables are used:
686 #
687 # S: string
688 # NS: naked string
689 # SS: next string
690 # NSS: naked next string
691 NS = ""
692 num_of_strings = 0
693 next_str_idx = string_idx
694 while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
695 num_of_strings += 1
697 SS = LL[next_str_idx].value
698 next_prefix = get_string_prefix(SS).lower()
700 # If this is an f-string group but this substring is not prefixed
701 # with 'f'...
702 if "f" in prefix and "f" not in next_prefix:
703 # Then we must escape any braces contained in this substring.
704 SS = re.sub(r"(\{|\})", r"\1\1", SS)
706 NSS = make_naked(SS, next_prefix)
708 has_prefix = bool(next_prefix)
709 prefix_tracker.append(has_prefix)
711 # Each NSS is already naked (prefix and quotes stripped, inner quotes
712 # escaped, f-string expression quotes toggled), and the parts are
713 # separated by BREAK_MARK which contains no quote or backslash, so the
714 # naked group is just their concatenation. Re-running make_naked over the
715 # whole accumulated string on every iteration rescans all previously
716 # merged substrings, which is quadratic in the size of the group.
717 NS = NS + NSS + BREAK_MARK
719 next_str_idx += 1
721 # Take a note on the index of the non-STRING leaf.
722 non_string_idx = next_str_idx
724 S = prefix + QUOTE + NS + QUOTE
725 S_leaf = Leaf(token.STRING, S)
726 if self.normalize_strings:
727 S_leaf.value = normalize_string_quotes(S_leaf.value)
729 # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
730 temp_string = S_leaf.value[len(prefix) + 1 : -1]
731 for has_prefix in prefix_tracker:
732 mark_idx = temp_string.find(BREAK_MARK)
733 assert (
734 mark_idx >= 0
735 ), "Logic error while filling the custom string breakpoint cache."
737 temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
738 breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
739 custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
741 string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
743 if atom_node is not None:
744 # If not all children of the atom node are merged (this can happen
745 # when there is a standalone comment in the middle) ...
746 if non_string_idx - string_idx < len(atom_node.children):
747 # We need to replace the old STRING leaves with the new string leaf.
748 first_child_idx = LL[string_idx].remove()
749 for idx in range(string_idx + 1, non_string_idx):
750 LL[idx].remove()
751 if first_child_idx is not None:
752 atom_node.insert_child(first_child_idx, string_leaf)
753 else:
754 # Else replace the atom node with the new string leaf.
755 replace_child(atom_node, string_leaf)
757 self.add_custom_splits(string_leaf.value, custom_splits)
758 return num_of_strings, string_leaf
760 @staticmethod
761 def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
762 """Validate (M)erge (S)tring (G)roup
764 Transform-time string validation logic for _merge_string_group(...).
766 Returns:
767 * Ok(None), if ALL validation checks (listed below) pass.
768 OR
769 * Err(CannotTransform), if any of the following are true:
770 - The target string group does not contain ANY stand-alone comments.
771 - The target string is not in a string group (i.e. it has no
772 adjacent strings).
773 - The string group has more than one inline comment.
774 - The string group has an inline comment that appears to be a pragma.
775 - The set of all string prefixes in the string group is of
776 length greater than one and is not equal to {"", "f"}.
777 - The string group consists of raw strings.
778 - The string group would merge f-strings with different quote types
779 and internal quotes.
780 - The string group is stringified type annotations. We don't want to
781 process stringified type annotations since pyright doesn't support
782 them spanning multiple string values. (NOTE: mypy, pytype, pyre do
783 support them, so we can change if pyright also gains support in the
784 future. See https://github.com/microsoft/pyright/issues/4359.)
785 """
786 # We first check for "inner" stand-alone comments (i.e. stand-alone
787 # comments that have a string leaf before them AND after them).
788 for inc in [1, -1]:
789 i = string_idx
790 found_sa_comment = False
791 is_valid_index = is_valid_index_factory(line.leaves)
792 while is_valid_index(i) and line.leaves[i].type in [
793 token.STRING,
794 STANDALONE_COMMENT,
795 ]:
796 if line.leaves[i].type == STANDALONE_COMMENT:
797 found_sa_comment = True
798 elif found_sa_comment:
799 return TErr(
800 "StringMerger does NOT merge string groups which contain "
801 "stand-alone comments."
802 )
804 i += inc
806 QUOTE = line.leaves[string_idx].value[-1]
808 num_of_inline_string_comments = 0
809 set_of_prefixes = set()
810 num_of_strings = 0
811 for leaf in line.leaves[string_idx:]:
812 if leaf.type != token.STRING:
813 # If the string group is trailed by a comma, we count the
814 # comments trailing the comma to be one of the string group's
815 # comments.
816 if leaf.type == token.COMMA and id(leaf) in line.comments:
817 num_of_inline_string_comments += 1
818 break
820 if has_triple_quotes(leaf.value):
821 return TErr("StringMerger does NOT merge multiline strings.")
823 num_of_strings += 1
824 prefix = get_string_prefix(leaf.value).lower()
825 if "r" in prefix:
826 return TErr("StringMerger does NOT merge raw strings.")
828 set_of_prefixes.add(prefix)
830 if (
831 "f" in prefix
832 and leaf.value[-1] != QUOTE
833 and (
834 "'" in leaf.value[len(prefix) + 1 : -1]
835 or '"' in leaf.value[len(prefix) + 1 : -1]
836 )
837 ):
838 return TErr(
839 "StringMerger does NOT merge f-strings with different quote types"
840 " and internal quotes."
841 )
843 if id(leaf) in line.comments:
844 num_of_inline_string_comments += 1
845 if contains_pragma_comment(line.comments[id(leaf)]):
846 return TErr("Cannot merge strings which have pragma comments.")
848 if num_of_strings < 2:
849 return TErr(
850 f"Not enough strings to merge (num_of_strings={num_of_strings})."
851 )
853 # Also check for pragma comments on tokens that follow the string
854 # group (e.g. a closing bracket). Merging strings when a pragma
855 # comment like `# type: ignore` follows would produce an unsplittable
856 # long line.
857 is_valid_index = is_valid_index_factory(line.leaves)
858 next_idx = string_idx + num_of_strings
859 while is_valid_index(next_idx):
860 next_leaf = line.leaves[next_idx]
861 if id(next_leaf) in line.comments:
862 if contains_pragma_comment(line.comments[id(next_leaf)]):
863 return TErr(
864 "Cannot merge strings when a pragma comment follows"
865 " the string group."
866 )
867 if next_leaf.type not in CLOSING_BRACKETS:
868 break
869 next_idx += 1
871 if num_of_inline_string_comments > 1:
872 return TErr(
873 f"Too many inline string comments ({num_of_inline_string_comments})."
874 )
876 if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
877 return TErr(f"Too many different prefixes ({set_of_prefixes}).")
879 return Ok(None)
882class StringParenStripper(StringTransformer):
883 """StringTransformer that strips surrounding parentheses from strings.
885 Requirements:
886 The line contains a string which is surrounded by parentheses and:
887 - The target string is NOT the only argument to a function call.
888 - The target string is NOT a "pointless" string.
889 - The target string is NOT a dictionary value.
890 - If the target string contains a PERCENT, the brackets are not
891 preceded or followed by an operator with higher precedence than
892 PERCENT.
894 Transformations:
895 The parentheses mentioned in the 'Requirements' section are stripped.
897 Collaborations:
898 StringParenStripper has its own inherent usefulness, but it is also
899 relied on to clean up the parentheses created by StringParenWrapper (in
900 the event that they are no longer needed).
901 """
903 def do_match(self, line: Line) -> TMatchResult:
904 LL = line.leaves
906 is_valid_index = is_valid_index_factory(LL)
908 string_indices = []
910 idx = -1
911 while True:
912 idx += 1
913 if idx >= len(LL):
914 break
915 leaf = LL[idx]
917 # Should be a string...
918 if leaf.type != token.STRING:
919 continue
921 # If this is a "pointless" string...
922 if (
923 leaf.parent
924 and leaf.parent.parent
925 and leaf.parent.parent.type == syms.simple_stmt
926 ):
927 continue
929 # Should be preceded by a non-empty LPAR...
930 if (
931 not is_valid_index(idx - 1)
932 or LL[idx - 1].type != token.LPAR
933 or is_empty_lpar(LL[idx - 1])
934 ):
935 continue
937 # That LPAR should NOT be preceded by a colon (which could be a
938 # dictionary value), function name, or a closing bracket (which
939 # could be a function returning a function or a list/dictionary
940 # containing a function)...
941 if is_valid_index(idx - 2) and (
942 LL[idx - 2].type == token.COLON
943 or LL[idx - 2].type == token.NAME
944 or LL[idx - 2].type in CLOSING_BRACKETS
945 ):
946 continue
948 string_idx = idx
950 # Skip the string trailer, if one exists.
951 string_parser = StringParser()
952 next_idx = string_parser.parse(LL, string_idx)
954 # if the leaves in the parsed string include a PERCENT, we need to
955 # make sure the initial LPAR is NOT preceded by an operator with
956 # higher or equal precedence to PERCENT
957 if is_valid_index(idx - 2):
958 # mypy can't quite follow unless we name this
959 before_lpar = LL[idx - 2]
960 if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
961 (
962 before_lpar.type in {
963 token.STAR,
964 token.AT,
965 token.SLASH,
966 token.DOUBLESLASH,
967 token.PERCENT,
968 token.TILDE,
969 token.DOUBLESTAR,
970 token.AWAIT,
971 token.LSQB,
972 token.LPAR,
973 }
974 )
975 or (
976 # only unary PLUS/MINUS
977 before_lpar.parent
978 and before_lpar.parent.type == syms.factor
979 and (before_lpar.type in {token.PLUS, token.MINUS})
980 )
981 ):
982 continue
984 # Should be followed by a non-empty RPAR...
985 if (
986 is_valid_index(next_idx)
987 and LL[next_idx].type == token.RPAR
988 and not is_empty_rpar(LL[next_idx])
989 ):
990 # That RPAR should NOT be followed by anything with higher
991 # precedence than PERCENT
992 if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
993 token.DOUBLESTAR,
994 token.LSQB,
995 token.LPAR,
996 token.DOT,
997 }:
998 continue
1000 string_indices.append(string_idx)
1001 idx = string_idx
1002 while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING:
1003 idx += 1
1005 if string_indices:
1006 return Ok(string_indices)
1007 return TErr("This line has no strings wrapped in parens.")
1009 def do_transform(
1010 self, line: Line, string_indices: list[int]
1011 ) -> Iterator[TResult[Line]]:
1012 LL = line.leaves
1014 string_and_rpar_indices: list[int] = []
1015 for string_idx in string_indices:
1016 string_parser = StringParser()
1017 rpar_idx = string_parser.parse(LL, string_idx)
1019 should_transform = True
1020 for leaf in (LL[string_idx - 1], LL[rpar_idx]):
1021 if line.comments_after(leaf):
1022 # Should not strip parentheses which have comments attached
1023 # to them.
1024 should_transform = False
1025 break
1026 if should_transform:
1027 string_and_rpar_indices.extend((string_idx, rpar_idx))
1029 if string_and_rpar_indices:
1030 yield Ok(self._transform_to_new_line(line, string_and_rpar_indices))
1031 else:
1032 yield Err(
1033 CannotTransform("All string groups have comments attached to them.")
1034 )
1036 def _transform_to_new_line(
1037 self, line: Line, string_and_rpar_indices: list[int]
1038 ) -> Line:
1039 LL = line.leaves
1041 new_line = line.clone()
1042 new_line.comments = line.comments.copy()
1044 previous_idx = -1
1045 # We need to sort the indices, since string_idx and its matching
1046 # rpar_idx may not come in order, e.g. in
1047 # `("outer" % ("inner".join(items)))`, the "inner" string's
1048 # string_idx is smaller than "outer" string's rpar_idx.
1049 for idx in sorted(string_and_rpar_indices):
1050 leaf = LL[idx]
1051 lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx
1052 append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx])
1053 if leaf.type == token.STRING:
1054 string_leaf = Leaf(token.STRING, LL[idx].value)
1055 LL[lpar_or_rpar_idx].remove() # Remove lpar.
1056 replace_child(LL[idx], string_leaf)
1057 new_line.append(string_leaf)
1058 # replace comments
1059 old_comments = new_line.comments.pop(id(LL[idx]), [])
1060 new_line.comments.setdefault(id(string_leaf), []).extend(old_comments)
1061 else:
1062 LL[lpar_or_rpar_idx].remove() # This is a rpar.
1064 previous_idx = idx
1066 # Append the leaves after the last idx:
1067 append_leaves(new_line, line, LL[idx + 1 :])
1069 return new_line
1072class BaseStringSplitter(StringTransformer):
1073 """
1074 Abstract class for StringTransformers which transform a Line's strings by splitting
1075 them or placing them on their own lines where necessary to avoid going over
1076 the configured line length.
1078 Requirements:
1079 * The target string value is responsible for the line going over the
1080 line length limit. It follows that after all of black's other line
1081 split methods have been exhausted, this line (or one of the resulting
1082 lines after all line splits are performed) would still be over the
1083 line_length limit unless we split this string.
1084 AND
1086 * The target string is NOT a "pointless" string (i.e. a string that has
1087 no parent or siblings).
1088 AND
1090 * The target string is not followed by an inline comment that appears
1091 to be a pragma.
1092 AND
1094 * The target string is not a multiline (i.e. triple-quote) string.
1095 """
1097 STRING_OPERATORS: Final = [
1098 token.EQEQUAL,
1099 token.GREATER,
1100 token.GREATEREQUAL,
1101 token.LESS,
1102 token.LESSEQUAL,
1103 token.NOTEQUAL,
1104 token.PERCENT,
1105 token.PLUS,
1106 token.STAR,
1107 ]
1109 @abstractmethod
1110 def do_splitter_match(self, line: Line) -> TMatchResult:
1111 """
1112 BaseStringSplitter asks its clients to override this method instead of
1113 `StringTransformer.do_match(...)`.
1115 Follows the same protocol as `StringTransformer.do_match(...)`.
1117 Refer to `help(StringTransformer.do_match)` for more information.
1118 """
1120 def do_match(self, line: Line) -> TMatchResult:
1121 match_result = self.do_splitter_match(line)
1122 if isinstance(match_result, Err):
1123 return match_result
1125 string_indices = match_result.ok()
1126 assert len(string_indices) == 1, (
1127 f"{self.__class__.__name__} should only find one match at a time, found"
1128 f" {len(string_indices)}"
1129 )
1130 string_idx = string_indices[0]
1131 vresult = self._validate(line, string_idx)
1132 if isinstance(vresult, Err):
1133 return vresult
1135 return match_result
1137 def _validate(self, line: Line, string_idx: int) -> TResult[None]:
1138 """
1139 Checks that @line meets all of the requirements listed in this classes'
1140 docstring. Refer to `help(BaseStringSplitter)` for a detailed
1141 description of those requirements.
1143 Returns:
1144 * Ok(None), if ALL of the requirements are met.
1145 OR
1146 * Err(CannotTransform), if ANY of the requirements are NOT met.
1147 """
1148 LL = line.leaves
1150 string_leaf = LL[string_idx]
1152 max_string_length = self._get_max_string_length(line, string_idx)
1153 if len(string_leaf.value) <= max_string_length:
1154 return TErr(
1155 "The string itself is not what is causing this line to be too long."
1156 )
1158 if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
1159 token.STRING,
1160 token.NEWLINE,
1161 ]:
1162 return TErr(
1163 f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
1164 " no parent)."
1165 )
1167 if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
1168 line.comments[id(line.leaves[string_idx])]
1169 ):
1170 return TErr(
1171 "Line appears to end with an inline pragma comment. Splitting the line"
1172 " could modify the pragma's behavior."
1173 )
1175 if has_triple_quotes(string_leaf.value):
1176 return TErr("We cannot split multiline strings.")
1178 return Ok(None)
1180 def _get_max_string_length(self, line: Line, string_idx: int) -> int:
1181 """
1182 Calculates the max string length used when attempting to determine
1183 whether or not the target string is responsible for causing the line to
1184 go over the line length limit.
1186 WARNING: This method is tightly coupled to both StringSplitter and
1187 (especially) StringParenWrapper. There is probably a better way to
1188 accomplish what is being done here.
1190 Returns:
1191 max_string_length: such that `line.leaves[string_idx].value >
1192 max_string_length` implies that the target string IS responsible
1193 for causing this line to exceed the line length limit.
1194 """
1195 LL = line.leaves
1197 is_valid_index = is_valid_index_factory(LL)
1199 # We use the shorthand "WMA4" in comments to abbreviate "We must
1200 # account for". When giving examples, we use STRING to mean some/any
1201 # valid string.
1202 #
1203 # Finally, we use the following convenience variables:
1204 #
1205 # P: The leaf that is before the target string leaf.
1206 # N: The leaf that is after the target string leaf.
1207 # NN: The leaf that is after N.
1209 # WMA4 the whitespace at the beginning of the line.
1210 offset = line.depth * 4
1212 if is_valid_index(string_idx - 1):
1213 p_idx = string_idx - 1
1214 if (
1215 LL[string_idx - 1].type == token.LPAR
1216 and LL[string_idx - 1].value == ""
1217 and string_idx >= 2
1218 ):
1219 # If the previous leaf is an empty LPAR placeholder, we should skip it.
1220 p_idx -= 1
1222 P = LL[p_idx]
1223 if P.type in self.STRING_OPERATORS:
1224 # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
1225 offset += len(str(P)) + 1
1227 if P.type == token.COMMA:
1228 # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
1229 offset += 3
1231 if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
1232 # This conditional branch is meant to handle dictionary keys,
1233 # variable assignments, 'return STRING' statement lines, and
1234 # 'else STRING' ternary expression lines.
1236 # WMA4 a single space.
1237 offset += 1
1239 # WMA4 the lengths of any leaves that came before that space,
1240 # but after any closing bracket before that space.
1241 for leaf in reversed(LL[: p_idx + 1]):
1242 offset += len(str(leaf))
1243 if leaf.type in CLOSING_BRACKETS:
1244 break
1246 if is_valid_index(string_idx + 1):
1247 N = LL[string_idx + 1]
1248 if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
1249 # If the next leaf is an empty RPAR placeholder, we should skip it.
1250 N = LL[string_idx + 2]
1252 if N.type == token.COMMA:
1253 # WMA4 a single comma at the end of the string (e.g `STRING,`).
1254 offset += 1
1256 if is_valid_index(string_idx + 2):
1257 NN = LL[string_idx + 2]
1259 if N.type == token.DOT and NN.type == token.NAME:
1260 # This conditional branch is meant to handle method calls invoked
1261 # off of a string literal up to and including the LPAR character.
1263 # WMA4 the '.' character.
1264 offset += 1
1266 if (
1267 is_valid_index(string_idx + 3)
1268 and LL[string_idx + 3].type == token.LPAR
1269 ):
1270 # WMA4 the left parenthesis character.
1271 offset += 1
1273 # WMA4 the length of the method's name.
1274 offset += len(NN.value)
1276 has_comments = False
1277 for comment_leaf in line.comments_after(LL[string_idx]):
1278 if not has_comments:
1279 has_comments = True
1280 # WMA4 two spaces before the '#' character.
1281 offset += 2
1283 # WMA4 the length of the inline comment.
1284 offset += len(comment_leaf.value)
1286 max_string_length = count_chars_in_width(str(line), self.line_length - offset)
1287 return max_string_length
1289 @staticmethod
1290 def _prefer_paren_wrap_match(LL: list[Leaf]) -> int | None:
1291 """
1292 Returns:
1293 string_idx such that @LL[string_idx] is equal to our target (i.e.
1294 matched) string, if this line matches the "prefer paren wrap" statement
1295 requirements listed in the 'Requirements' section of the StringParenWrapper
1296 class's docstring.
1297 OR
1298 None, otherwise.
1299 """
1300 # The line must start with a string.
1301 if LL[0].type != token.STRING:
1302 return None
1304 matching_nodes = [
1305 syms.listmaker,
1306 syms.dictsetmaker,
1307 syms.testlist_gexp,
1308 ]
1309 # If the string is an immediate child of a list/set/tuple literal...
1310 if (
1311 parent_type(LL[0]) in matching_nodes
1312 or parent_type(LL[0].parent) in matching_nodes
1313 ):
1314 # And the string is surrounded by commas (or is the first/last child)...
1315 prev_sibling = LL[0].prev_sibling
1316 next_sibling = LL[0].next_sibling
1317 if (
1318 not prev_sibling
1319 and not next_sibling
1320 and parent_type(LL[0]) == syms.atom
1321 ):
1322 # If it's an atom string, we need to check the parent atom's siblings.
1323 parent = LL[0].parent
1324 assert parent is not None # For type checkers.
1325 prev_sibling = parent.prev_sibling
1326 next_sibling = parent.next_sibling
1327 if (not prev_sibling or prev_sibling.type == token.COMMA) and (
1328 not next_sibling or next_sibling.type == token.COMMA
1329 ):
1330 return 0
1332 return None
1335def iter_fexpr_spans(s: str) -> Iterator[tuple[int, int]]:
1336 """
1337 Yields spans corresponding to expressions in a given f-string.
1338 Spans are half-open ranges (left inclusive, right exclusive).
1339 Assumes the input string is a valid f-string, but will not crash if the input
1340 string is invalid.
1341 """
1342 stack: list[int] = [] # our curly paren stack
1343 i = 0
1344 while i < len(s):
1345 if s[i] == "{":
1346 # if we're in a string part of the f-string, ignore escaped curly braces
1347 if not stack and i + 1 < len(s) and s[i + 1] == "{":
1348 i += 2
1349 continue
1350 stack.append(i)
1351 i += 1
1352 continue
1354 if s[i] == "}":
1355 if not stack:
1356 i += 1
1357 continue
1358 j = stack.pop()
1359 # we've made it back out of the expression! yield the span
1360 if not stack:
1361 yield (j, i + 1)
1362 i += 1
1363 continue
1365 # if we're in an expression part of the f-string, fast-forward through strings
1366 # note that backslashes are not legal in the expression portion of f-strings
1367 if stack:
1368 delim = None
1369 if s[i : i + 3] in ("'''", '"""'):
1370 delim = s[i : i + 3]
1371 elif s[i] in ("'", '"'):
1372 delim = s[i]
1373 if delim:
1374 i += len(delim)
1375 while i < len(s) and s[i : i + len(delim)] != delim:
1376 i += 1
1377 i += len(delim)
1378 continue
1379 i += 1
1382def fstring_contains_expr(s: str) -> bool:
1383 return any(iter_fexpr_spans(s))
1386def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:
1387 """
1388 Toggles quotes used in f-string expressions that are `old_quote`.
1390 f-string expressions can't contain backslashes, so we need to toggle the
1391 quotes if the f-string itself will end up using the same quote. We can
1392 simply toggle without escaping because, quotes can't be reused in f-string
1393 expressions. They will fail to parse.
1395 NOTE: If PEP 701 is accepted, above statement will no longer be true.
1396 Though if quotes can be reused, we can simply reuse them without updates or
1397 escaping, once Black figures out how to parse the new grammar.
1398 """
1399 new_quote = "'" if old_quote == '"' else '"'
1400 parts = []
1401 previous_index = 0
1402 for start, end in iter_fexpr_spans(fstring):
1403 parts.append(fstring[previous_index:start])
1404 parts.append(fstring[start:end].replace(old_quote, new_quote))
1405 previous_index = end
1406 parts.append(fstring[previous_index:])
1407 return "".join(parts)
1410class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
1411 """
1412 StringTransformer that splits "atom" strings (i.e. strings which exist on
1413 lines by themselves).
1415 Requirements:
1416 * The line consists ONLY of a single string (possibly prefixed by a
1417 string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE
1418 a trailing comma.
1419 AND
1420 * All of the requirements listed in BaseStringSplitter's docstring.
1422 Transformations:
1423 The string mentioned in the 'Requirements' section is split into as
1424 many substrings as necessary to adhere to the configured line length.
1426 In the final set of substrings, no substring should be smaller than
1427 MIN_SUBSTR_SIZE characters.
1429 The string will ONLY be split on spaces (i.e. each new substring should
1430 start with a space). Note that the string will NOT be split on a space
1431 which is escaped with a backslash.
1433 If the string is an f-string, it will NOT be split in the middle of an
1434 f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
1435 else bar()} is an f-expression).
1437 If the string that is being split has an associated set of custom split
1438 records and those custom splits will NOT result in any line going over
1439 the configured line length, those custom splits are used. Otherwise the
1440 string is split as late as possible (from left-to-right) while still
1441 adhering to the transformation rules listed above.
1443 Collaborations:
1444 StringSplitter relies on StringMerger to construct the appropriate
1445 CustomSplit objects and add them to the custom split map.
1446 """
1448 MIN_SUBSTR_SIZE: Final = 6
1450 def do_splitter_match(self, line: Line) -> TMatchResult:
1451 LL = line.leaves
1453 if self._prefer_paren_wrap_match(LL) is not None:
1454 return TErr("Line needs to be wrapped in parens first.")
1456 # If the line is just STRING + COMMA (a one-item tuple) and not inside
1457 # brackets, we need to defer to StringParenWrapper to wrap it first.
1458 # Otherwise, splitting the string would create multiple expressions where
1459 # only the last has the comma, breaking AST equivalence. See issue #4912.
1460 if (
1461 not line.inside_brackets
1462 and len(LL) == 2
1463 and LL[0].type == token.STRING
1464 and LL[1].type == token.COMMA
1465 ):
1466 return TErr(
1467 "Line with trailing comma tuple needs to be wrapped in parens first."
1468 )
1470 is_valid_index = is_valid_index_factory(LL)
1472 idx = 0
1474 # The first two leaves MAY be the 'not in' keywords...
1475 if (
1476 is_valid_index(idx)
1477 and is_valid_index(idx + 1)
1478 and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]
1479 and str(LL[idx]) + str(LL[idx + 1]) == "not in"
1480 ):
1481 idx += 2
1482 # Else the first leaf MAY be a string operator symbol or the 'in' keyword...
1483 elif is_valid_index(idx) and (
1484 LL[idx].type in self.STRING_OPERATORS
1485 or LL[idx].type == token.NAME
1486 and str(LL[idx]) == "in"
1487 ):
1488 idx += 1
1490 # The next/first leaf MAY be an empty LPAR...
1491 if is_valid_index(idx) and is_empty_lpar(LL[idx]):
1492 idx += 1
1494 # The next/first leaf MUST be a string...
1495 if not is_valid_index(idx) or LL[idx].type != token.STRING:
1496 return TErr("Line does not start with a string.")
1498 string_idx = idx
1500 # Skip the string trailer, if one exists.
1501 string_parser = StringParser()
1502 idx = string_parser.parse(LL, string_idx)
1504 # That string MAY be followed by an empty RPAR...
1505 if is_valid_index(idx) and is_empty_rpar(LL[idx]):
1506 idx += 1
1508 # That string / empty RPAR leaf MAY be followed by a comma...
1509 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1510 idx += 1
1512 # But no more leaves are allowed...
1513 if is_valid_index(idx):
1514 return TErr("This line does not end with a string.")
1516 return Ok([string_idx])
1518 def do_transform(
1519 self, line: Line, string_indices: list[int]
1520 ) -> Iterator[TResult[Line]]:
1521 LL = line.leaves
1522 assert len(string_indices) == 1, (
1523 f"{self.__class__.__name__} should only find one match at a time, found"
1524 f" {len(string_indices)}"
1525 )
1526 string_idx = string_indices[0]
1528 QUOTE = LL[string_idx].value[-1]
1530 is_valid_index = is_valid_index_factory(LL)
1531 insert_str_child = insert_str_child_factory(LL[string_idx])
1533 prefix = get_string_prefix(LL[string_idx].value).lower()
1535 # We MAY choose to drop the 'f' prefix from substrings that don't
1536 # contain any f-expressions, but ONLY if the original f-string
1537 # contains at least one f-expression. Otherwise, we will alter the AST
1538 # of the program.
1539 drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr(
1540 LL[string_idx].value
1541 )
1543 first_string_line = True
1545 string_op_leaves = self._get_string_operator_leaves(LL)
1546 string_op_leaves_length = (
1547 sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1
1548 if string_op_leaves
1549 else 0
1550 )
1552 def maybe_append_string_operators(new_line: Line) -> None:
1553 """
1554 Side Effects:
1555 If @line starts with a string operator and this is the first
1556 line we are constructing, this function appends the string
1557 operator to @new_line and replaces the old string operator leaf
1558 in the node structure. Otherwise this function does nothing.
1559 """
1560 maybe_prefix_leaves = string_op_leaves if first_string_line else []
1561 for i, prefix_leaf in enumerate(maybe_prefix_leaves):
1562 replace_child(LL[i], prefix_leaf)
1563 new_line.append(prefix_leaf)
1565 ends_with_comma = (
1566 is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
1567 )
1569 def max_last_string_column() -> int:
1570 """
1571 Returns:
1572 The max allowed width of the string value used for the last
1573 line we will construct. Note that this value means the width
1574 rather than the number of characters (e.g., many East Asian
1575 characters expand to two columns).
1576 """
1577 result = self.line_length
1578 result -= line.depth * 4
1579 result -= 1 if ends_with_comma else 0
1580 result -= string_op_leaves_length
1581 return result
1583 # --- Calculate Max Break Width (for string value)
1584 # We start with the line length limit
1585 max_break_width = self.line_length
1586 # The last index of a string of length N is N-1.
1587 max_break_width -= 1
1588 # Leading whitespace is not present in the string value (e.g. Leaf.value).
1589 max_break_width -= line.depth * 4
1590 if max_break_width < 0:
1591 yield TErr(
1592 f"Unable to split {LL[string_idx].value} at such high of a line depth:"
1593 f" {line.depth}"
1594 )
1595 return
1597 # Check if StringMerger registered any custom splits.
1598 custom_splits = self.pop_custom_splits(LL[string_idx].value)
1599 # We use them ONLY if none of them would produce lines that exceed the
1600 # line limit.
1601 use_custom_breakpoints = bool(
1602 custom_splits
1603 and all(csplit.break_idx <= max_break_width for csplit in custom_splits)
1604 )
1606 # Temporary storage for the remaining chunk of the string line that
1607 # can't fit onto the line currently being constructed.
1608 rest_value = LL[string_idx].value
1610 # Each substring is a suffix of this value (plus the prefix and quote),
1611 # so a missing "\N" here means no substring can hold a named escape and
1612 # the per-substring scan for them can be skipped entirely.
1613 has_named_escape = "\\N" in rest_value
1615 def more_splits_should_be_made() -> bool:
1616 """
1617 Returns:
1618 True iff `rest_value` (the remaining string value from the last
1619 split), should be split again.
1620 """
1621 if use_custom_breakpoints:
1622 return len(custom_splits) > 1
1623 else:
1624 return str_width(rest_value) > max_last_string_column()
1626 string_line_results: list[Ok[Line]] = []
1627 while more_splits_should_be_made():
1628 if use_custom_breakpoints:
1629 # Custom User Split (manual)
1630 csplit = custom_splits.pop(0)
1631 break_idx = csplit.break_idx
1632 else:
1633 # Algorithmic Split (automatic)
1634 max_bidx = (
1635 count_chars_in_width(rest_value, max_break_width)
1636 - string_op_leaves_length
1637 )
1638 maybe_break_idx = self._get_break_idx(
1639 rest_value, max_bidx, has_named_escape
1640 )
1641 if maybe_break_idx is None:
1642 # If we are unable to algorithmically determine a good split
1643 # and this string has custom splits registered to it, we
1644 # fall back to using them--which means we have to start
1645 # over from the beginning.
1646 if custom_splits:
1647 rest_value = LL[string_idx].value
1648 string_line_results = []
1649 first_string_line = True
1650 use_custom_breakpoints = True
1651 continue
1653 # Otherwise, we stop splitting here.
1654 break
1656 break_idx = maybe_break_idx
1658 # --- Construct `next_value`
1659 next_value = rest_value[:break_idx] + QUOTE
1661 # HACK: The following 'if' statement is a hack to fix the custom
1662 # breakpoint index in the case of either: (a) substrings that were
1663 # f-strings but will have the 'f' prefix removed OR (b) substrings
1664 # that were not f-strings but will now become f-strings because of
1665 # redundant use of the 'f' prefix (i.e. none of the substrings
1666 # contain f-expressions but one or more of them had the 'f' prefix
1667 # anyway; in which case, we will prepend 'f' to _all_ substrings).
1668 #
1669 # There is probably a better way to accomplish what is being done
1670 # here...
1671 #
1672 # If this substring is an f-string, we _could_ remove the 'f'
1673 # prefix, and the current custom split did NOT originally use a
1674 # prefix...
1675 if (
1676 use_custom_breakpoints
1677 and not csplit.has_prefix
1678 and (
1679 # `next_value == prefix + QUOTE` happens when the custom
1680 # split is an empty string.
1681 next_value == prefix + QUOTE
1682 or next_value != self._normalize_f_string(next_value, prefix)
1683 )
1684 ):
1685 # Then `csplit.break_idx` will be off by one after removing
1686 # the 'f' prefix.
1687 break_idx += 1
1688 next_value = rest_value[:break_idx] + QUOTE
1690 if drop_pointless_f_prefix:
1691 next_value = self._normalize_f_string(next_value, prefix)
1693 # --- Construct `next_leaf`
1694 next_leaf = Leaf(token.STRING, next_value)
1695 insert_str_child(next_leaf)
1696 self._maybe_normalize_string_quotes(next_leaf)
1698 # --- Construct `next_line`
1699 next_line = line.clone()
1700 maybe_append_string_operators(next_line)
1701 next_line.append(next_leaf)
1702 string_line_results.append(Ok(next_line))
1704 rest_value = prefix + QUOTE + rest_value[break_idx:]
1705 first_string_line = False
1707 yield from string_line_results
1709 if drop_pointless_f_prefix:
1710 rest_value = self._normalize_f_string(rest_value, prefix)
1712 rest_leaf = Leaf(token.STRING, rest_value)
1713 insert_str_child(rest_leaf)
1715 # NOTE: I could not find a test case that verifies that the following
1716 # line is actually necessary, but it seems to be. Otherwise we risk
1717 # not normalizing the last substring, right?
1718 self._maybe_normalize_string_quotes(rest_leaf)
1720 last_line = line.clone()
1721 maybe_append_string_operators(last_line)
1723 # If there are any leaves to the right of the target string...
1724 if is_valid_index(string_idx + 1):
1725 # We use `temp_value` here to determine how long the last line
1726 # would be if we were to append all the leaves to the right of the
1727 # target string to the last string line.
1728 temp_value = rest_value
1729 for leaf in LL[string_idx + 1 :]:
1730 temp_value += str(leaf)
1731 if leaf.type == token.LPAR:
1732 break
1734 # Try to fit them all on the same line with the last substring...
1735 if (
1736 str_width(temp_value) <= max_last_string_column()
1737 or LL[string_idx + 1].type == token.COMMA
1738 ):
1739 last_line.append(rest_leaf)
1740 append_leaves(last_line, line, LL[string_idx + 1 :])
1741 yield Ok(last_line)
1742 # Otherwise, place the last substring on one line and everything
1743 # else on a line below that...
1744 else:
1745 last_line.append(rest_leaf)
1746 yield Ok(last_line)
1748 non_string_line = line.clone()
1749 append_leaves(non_string_line, line, LL[string_idx + 1 :])
1750 yield Ok(non_string_line)
1751 # Else the target string was the last leaf...
1752 else:
1753 last_line.append(rest_leaf)
1754 last_line.comments = line.comments.copy()
1755 yield Ok(last_line)
1757 def _iter_nameescape_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1758 r"""
1759 Yields:
1760 All ranges of @string which, if @string were to be split there,
1761 would result in the splitting of an \N{...} expression (which is NOT
1762 allowed).
1763 """
1764 # True - the previous backslash was unescaped
1765 # False - the previous backslash was escaped *or* there was no backslash
1766 previous_was_unescaped_backslash = False
1767 it = iter(enumerate(string))
1768 for idx, c in it:
1769 if c == "\\":
1770 previous_was_unescaped_backslash = not previous_was_unescaped_backslash
1771 continue
1772 if not previous_was_unescaped_backslash or c != "N":
1773 previous_was_unescaped_backslash = False
1774 continue
1775 previous_was_unescaped_backslash = False
1777 begin = idx - 1 # the position of backslash before \N{...}
1778 for idx, c in it:
1779 if c == "}":
1780 end = idx
1781 break
1782 else:
1783 # malformed nameescape expression?
1784 # should have been detected by AST parsing earlier...
1785 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
1786 yield begin, end
1788 def _iter_fexpr_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1789 """
1790 Yields:
1791 All ranges of @string which, if @string were to be split there,
1792 would result in the splitting of an f-expression (which is NOT
1793 allowed).
1794 """
1795 if "f" not in get_string_prefix(string).lower():
1796 return
1797 yield from iter_fexpr_spans(string)
1799 def _get_illegal_split_indices(
1800 self, string: str, has_named_escape: bool = True
1801 ) -> set[Index]:
1802 illegal_indices: set[Index] = set()
1803 iterators = [self._iter_fexpr_slices(string)]
1804 # Scanning for \N{...} ranges walks the whole string, so skip it when the
1805 # caller already knows the string cannot contain a named escape.
1806 if has_named_escape:
1807 iterators.append(self._iter_nameescape_slices(string))
1808 for it in iterators:
1809 for begin, end in it:
1810 illegal_indices.update(range(begin, end))
1811 return illegal_indices
1813 def _get_break_idx(
1814 self, string: str, max_break_idx: int, has_named_escape: bool = True
1815 ) -> int | None:
1816 """
1817 This method contains the algorithm that StringSplitter uses to
1818 determine which character to split each string at.
1820 Args:
1821 @string: The substring that we are attempting to split.
1822 @max_break_idx: The ideal break index. We will return this value if it
1823 meets all the necessary conditions. In the likely event that it
1824 doesn't we will try to find the closest index BELOW @max_break_idx
1825 that does. If that fails, we will expand our search by also
1826 considering all valid indices ABOVE @max_break_idx.
1827 @has_named_escape: Whether the original string can contain a named
1828 escape. False lets us skip the per-substring scan for them.
1830 Pre-Conditions:
1831 * assert_is_leaf_string(@string)
1832 * 0 <= @max_break_idx < len(@string)
1834 Returns:
1835 break_idx, if an index is able to be found that meets all of the
1836 conditions listed in the 'Transformations' section of this classes'
1837 docstring.
1838 OR
1839 None, otherwise.
1840 """
1841 is_valid_index = is_valid_index_factory(string)
1843 assert is_valid_index(max_break_idx)
1844 assert_is_leaf_string(string)
1846 _illegal_split_indices = self._get_illegal_split_indices(
1847 string, has_named_escape
1848 )
1850 def breaks_unsplittable_expression(i: Index) -> bool:
1851 """
1852 Returns:
1853 True iff returning @i would result in the splitting of an
1854 unsplittable expression (which is NOT allowed).
1855 """
1856 return i in _illegal_split_indices
1858 def passes_all_checks(i: Index) -> bool:
1859 """
1860 Returns:
1861 True iff ALL of the conditions listed in the 'Transformations'
1862 section of this classes' docstring would be met by returning @i.
1863 """
1864 is_space = string[i] == " "
1865 is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS
1867 is_not_escaped = True
1868 j = i - 1
1869 while is_valid_index(j) and string[j] == "\\":
1870 is_not_escaped = not is_not_escaped
1871 j -= 1
1873 is_big_enough = (
1874 len(string) - i >= self.MIN_SUBSTR_SIZE and i >= self.MIN_SUBSTR_SIZE
1875 )
1876 return (
1877 (is_space or is_split_safe)
1878 and is_not_escaped
1879 and is_big_enough
1880 and not breaks_unsplittable_expression(i)
1881 )
1883 # First, we check all indices BELOW @max_break_idx.
1884 break_idx = max_break_idx
1885 while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
1886 break_idx -= 1
1888 if not passes_all_checks(break_idx):
1889 # If that fails, we check all indices ABOVE @max_break_idx.
1890 #
1891 # If we are able to find a valid index here, the next line is going
1892 # to be longer than the specified line length, but it's probably
1893 # better than doing nothing at all.
1894 break_idx = max_break_idx + 1
1895 while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
1896 break_idx += 1
1898 if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
1899 return None
1901 return break_idx
1903 def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
1904 if self.normalize_strings:
1905 leaf.value = normalize_string_quotes(leaf.value)
1907 def _normalize_f_string(self, string: str, prefix: str) -> str:
1908 """
1909 Pre-Conditions:
1910 * assert_is_leaf_string(@string)
1912 Returns:
1913 * If @string is an f-string that contains no f-expressions, we
1914 return a string identical to @string except that the 'f' prefix
1915 has been stripped and all double braces (i.e. '{{' or '}}') have
1916 been normalized (i.e. turned into '{' or '}').
1917 OR
1918 * Otherwise, we return @string.
1919 """
1920 assert_is_leaf_string(string)
1922 if "f" in prefix and not fstring_contains_expr(string):
1923 new_prefix = prefix.replace("f", "")
1925 temp = string[len(prefix) :]
1926 temp = re.sub(r"\{\{", "{", temp)
1927 temp = re.sub(r"\}\}", "}", temp)
1928 new_string = temp
1930 return f"{new_prefix}{new_string}"
1931 else:
1932 return string
1934 def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> list[Leaf]:
1935 LL = list(leaves)
1937 string_op_leaves = []
1938 i = 0
1939 while LL[i].type in self.STRING_OPERATORS + [token.NAME]:
1940 prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())
1941 string_op_leaves.append(prefix_leaf)
1942 i += 1
1943 return string_op_leaves
1946class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):
1947 """
1948 StringTransformer that wraps strings in parens and then splits at the LPAR.
1950 Requirements:
1951 All of the requirements listed in BaseStringSplitter's docstring in
1952 addition to the requirements listed below:
1954 * The line is a return/yield statement, which returns/yields a string.
1955 OR
1956 * The line is part of a ternary expression (e.g. `x = y if cond else
1957 z`) such that the line starts with `else <string>`, where <string> is
1958 some string.
1959 OR
1960 * The line is an assert statement, which ends with a string.
1961 OR
1962 * The line is an assignment statement (e.g. `x = <string>` or `x +=
1963 <string>`) such that the variable is being assigned the value of some
1964 string.
1965 OR
1966 * The line is a dictionary key assignment where some valid key is being
1967 assigned the value of some string.
1968 OR
1969 * The line is an lambda expression and the value is a string.
1970 OR
1971 * The line starts with an "atom" string that prefers to be wrapped in
1972 parens. It's preferred to be wrapped when it's is an immediate child of
1973 a list/set/tuple literal, AND the string is surrounded by commas (or is
1974 the first/last child).
1976 Transformations:
1977 The chosen string is wrapped in parentheses and then split at the LPAR.
1979 We then have one line which ends with an LPAR and another line that
1980 starts with the chosen string. The latter line is then split again at
1981 the RPAR. This results in the RPAR (and possibly a trailing comma)
1982 being placed on its own line.
1984 NOTE: If any leaves exist to the right of the chosen string (except
1985 for a trailing comma, which would be placed after the RPAR), those
1986 leaves are placed inside the parentheses. In effect, the chosen
1987 string is not necessarily being "wrapped" by parentheses. We can,
1988 however, count on the LPAR being placed directly before the chosen
1989 string.
1991 In other words, StringParenWrapper creates "atom" strings. These
1992 can then be split again by StringSplitter, if necessary.
1994 Collaborations:
1995 In the event that a string line split by StringParenWrapper is
1996 changed such that it no longer needs to be given its own line,
1997 StringParenWrapper relies on StringParenStripper to clean up the
1998 parentheses it created.
2000 For "atom" strings that prefers to be wrapped in parens, it requires
2001 StringSplitter to hold the split until the string is wrapped in parens.
2002 """
2004 def do_splitter_match(self, line: Line) -> TMatchResult:
2005 LL = line.leaves
2007 if line.leaves[-1].type in OPENING_BRACKETS:
2008 return TErr(
2009 "Cannot wrap parens around a line that ends in an opening bracket."
2010 )
2012 string_idx = (
2013 self._return_match(LL)
2014 or self._else_match(LL)
2015 or self._assert_match(LL)
2016 or self._assign_match(LL)
2017 or self._dict_or_lambda_match(LL)
2018 )
2020 if string_idx is None:
2021 string_idx = self._trailing_comma_tuple_match(line)
2023 if string_idx is None:
2024 string_idx = self._prefer_paren_wrap_match(LL)
2026 if string_idx is not None:
2027 string_value = line.leaves[string_idx].value
2028 # If the string has neither spaces nor East Asian stops...
2029 if not any(
2030 char == " " or char in SPLIT_SAFE_CHARS for char in string_value
2031 ):
2032 # And will still violate the line length limit when split...
2033 max_string_width = self.line_length - ((line.depth + 1) * 4)
2034 if str_width(string_value) > max_string_width:
2035 # And has no associated custom splits...
2036 if not self.has_custom_splits(string_value):
2037 # Then we should NOT put this string on its own line.
2038 return TErr(
2039 "We do not wrap long strings in parentheses when the"
2040 " resultant line would still be over the specified line"
2041 " length and can't be split further by StringSplitter."
2042 )
2043 return Ok([string_idx])
2045 return TErr("This line does not contain any non-atomic strings.")
2047 @staticmethod
2048 def _return_match(LL: list[Leaf]) -> int | None:
2049 """
2050 Returns:
2051 string_idx such that @LL[string_idx] is equal to our target (i.e.
2052 matched) string, if this line matches the return/yield statement
2053 requirements listed in the 'Requirements' section of this classes'
2054 docstring.
2055 OR
2056 None, otherwise.
2057 """
2058 # If this line is a part of a return/yield statement and the first leaf
2059 # contains either the "return" or "yield" keywords...
2060 if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
2061 0
2062 ].value in ["return", "yield"]:
2063 is_valid_index = is_valid_index_factory(LL)
2065 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2066 # The next visible leaf MUST contain a string...
2067 if is_valid_index(idx) and LL[idx].type == token.STRING:
2068 return idx
2070 return None
2072 @staticmethod
2073 def _else_match(LL: list[Leaf]) -> int | None:
2074 """
2075 Returns:
2076 string_idx such that @LL[string_idx] is equal to our target (i.e.
2077 matched) string, if this line matches the ternary expression
2078 requirements listed in the 'Requirements' section of this classes'
2079 docstring.
2080 OR
2081 None, otherwise.
2082 """
2083 # If this line is a part of a ternary expression and the first leaf
2084 # contains the "else" keyword...
2085 if (
2086 parent_type(LL[0]) == syms.test
2087 and LL[0].type == token.NAME
2088 and LL[0].value == "else"
2089 ):
2090 is_valid_index = is_valid_index_factory(LL)
2092 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2093 # The next visible leaf MUST contain a string...
2094 if is_valid_index(idx) and LL[idx].type == token.STRING:
2095 return idx
2097 return None
2099 @staticmethod
2100 def _assert_match(LL: list[Leaf]) -> int | None:
2101 """
2102 Returns:
2103 string_idx such that @LL[string_idx] is equal to our target (i.e.
2104 matched) string, if this line matches the assert statement
2105 requirements listed in the 'Requirements' section of this classes'
2106 docstring.
2107 OR
2108 None, otherwise.
2109 """
2110 # If this line is a part of an assert statement and the first leaf
2111 # contains the "assert" keyword...
2112 if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
2113 is_valid_index = is_valid_index_factory(LL)
2115 for i, leaf in enumerate(LL):
2116 # We MUST find a comma...
2117 if leaf.type == token.COMMA:
2118 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2120 # That comma MUST be followed by a string...
2121 if is_valid_index(idx) and LL[idx].type == token.STRING:
2122 string_idx = idx
2124 # Skip the string trailer, if one exists.
2125 string_parser = StringParser()
2126 idx = string_parser.parse(LL, string_idx)
2128 # But no more leaves are allowed...
2129 if not is_valid_index(idx):
2130 return string_idx
2132 return None
2134 @staticmethod
2135 def _assign_match(LL: list[Leaf]) -> int | None:
2136 """
2137 Returns:
2138 string_idx such that @LL[string_idx] is equal to our target (i.e.
2139 matched) string, if this line matches the assignment statement
2140 requirements listed in the 'Requirements' section of this classes'
2141 docstring.
2142 OR
2143 None, otherwise.
2144 """
2145 # If this line is a part of an expression statement or is a function
2146 # argument AND the first leaf contains a variable name...
2147 if (
2148 parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
2149 and LL[0].type == token.NAME
2150 ):
2151 is_valid_index = is_valid_index_factory(LL)
2153 for i, leaf in enumerate(LL):
2154 # We MUST find either an '=' or '+=' symbol...
2155 if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
2156 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2158 # That symbol MUST be followed by a string...
2159 if is_valid_index(idx) and LL[idx].type == token.STRING:
2160 string_idx = idx
2162 # Skip the string trailer, if one exists.
2163 string_parser = StringParser()
2164 idx = string_parser.parse(LL, string_idx)
2166 # The next leaf MAY be a comma iff this line is a part
2167 # of a function argument...
2168 if (
2169 parent_type(LL[0]) == syms.argument
2170 and is_valid_index(idx)
2171 and LL[idx].type == token.COMMA
2172 ):
2173 idx += 1
2175 # But no more leaves are allowed...
2176 if not is_valid_index(idx):
2177 return string_idx
2179 return None
2181 @staticmethod
2182 def _dict_or_lambda_match(LL: list[Leaf]) -> int | None:
2183 """
2184 Returns:
2185 string_idx such that @LL[string_idx] is equal to our target (i.e.
2186 matched) string, if this line matches the dictionary key assignment
2187 statement or lambda expression requirements listed in the
2188 'Requirements' section of this classes' docstring.
2189 OR
2190 None, otherwise.
2191 """
2192 # If this line is a part of a dictionary key assignment or lambda expression...
2193 parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)]
2194 if syms.dictsetmaker in parent_types or syms.lambdef in parent_types:
2195 is_valid_index = is_valid_index_factory(LL)
2197 for i, leaf in enumerate(LL):
2198 # We MUST find a colon, it can either be dict's or lambda's colon...
2199 if leaf.type == token.COLON and i < len(LL) - 1:
2200 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2202 # That colon MUST be followed by a string...
2203 if is_valid_index(idx) and LL[idx].type == token.STRING:
2204 string_idx = idx
2206 # Skip the string trailer, if one exists.
2207 string_parser = StringParser()
2208 idx = string_parser.parse(LL, string_idx)
2210 # That string MAY be followed by a comma...
2211 if is_valid_index(idx) and LL[idx].type == token.COMMA:
2212 idx += 1
2214 # But no more leaves are allowed...
2215 if not is_valid_index(idx):
2216 return string_idx
2218 return None
2220 @staticmethod
2221 def _trailing_comma_tuple_match(line: Line) -> int | None:
2222 """
2223 Returns:
2224 string_idx such that @line.leaves[string_idx] is equal to our target
2225 (i.e. matched) string, if the line is a bare trailing comma tuple
2226 (STRING + COMMA) not inside brackets.
2227 OR
2228 None, otherwise.
2230 This handles the case from issue #4912 where a long string with a
2231 trailing comma (making it a one-item tuple) needs to be wrapped in
2232 parentheses before splitting to preserve AST equivalence.
2233 """
2234 LL = line.leaves
2235 # Match: STRING followed by COMMA, not inside brackets
2236 if (
2237 not line.inside_brackets
2238 and len(LL) == 2
2239 and LL[0].type == token.STRING
2240 and LL[1].type == token.COMMA
2241 ):
2242 return 0
2244 return None
2246 def do_transform(
2247 self, line: Line, string_indices: list[int]
2248 ) -> Iterator[TResult[Line]]:
2249 LL = line.leaves
2250 assert len(string_indices) == 1, (
2251 f"{self.__class__.__name__} should only find one match at a time, found"
2252 f" {len(string_indices)}"
2253 )
2254 string_idx = string_indices[0]
2256 is_valid_index = is_valid_index_factory(LL)
2257 insert_str_child = insert_str_child_factory(LL[string_idx])
2259 comma_idx = -1
2260 ends_with_comma = False
2261 if LL[comma_idx].type == token.COMMA:
2262 ends_with_comma = True
2264 leaves_to_steal_comments_from = [LL[string_idx]]
2265 if ends_with_comma:
2266 leaves_to_steal_comments_from.append(LL[comma_idx])
2268 # --- First Line
2269 first_line = line.clone()
2270 left_leaves = LL[:string_idx]
2272 # We have to remember to account for (possibly invisible) LPAR and RPAR
2273 # leaves that already wrapped the target string. If these leaves do
2274 # exist, we will replace them with our own LPAR and RPAR leaves.
2275 old_parens_exist = False
2276 if left_leaves and left_leaves[-1].type == token.LPAR:
2277 old_parens_exist = True
2278 leaves_to_steal_comments_from.append(left_leaves[-1])
2279 left_leaves.pop()
2281 append_leaves(first_line, line, left_leaves)
2283 lpar_leaf = Leaf(token.LPAR, "(")
2284 if old_parens_exist:
2285 replace_child(LL[string_idx - 1], lpar_leaf)
2286 else:
2287 insert_str_child(lpar_leaf)
2288 first_line.append(lpar_leaf)
2290 # We throw inline comments that were originally to the right of the
2291 # target string to the top line. They will now be shown to the right of
2292 # the LPAR.
2293 for leaf in leaves_to_steal_comments_from:
2294 for comment_leaf in line.comments_after(leaf):
2295 first_line.append(comment_leaf, preformatted=True)
2297 yield Ok(first_line)
2299 # --- Middle (String) Line
2300 # We only need to yield one (possibly too long) string line, since the
2301 # `StringSplitter` will break it down further if necessary.
2302 string_value = LL[string_idx].value
2303 string_line = Line(
2304 mode=line.mode,
2305 depth=line.depth + 1,
2306 inside_brackets=True,
2307 should_split_rhs=line.should_split_rhs,
2308 magic_trailing_comma=line.magic_trailing_comma,
2309 )
2310 string_leaf = Leaf(token.STRING, string_value)
2311 insert_str_child(string_leaf)
2312 string_line.append(string_leaf)
2314 old_rpar_leaf = None
2315 if is_valid_index(string_idx + 1):
2316 right_leaves = LL[string_idx + 1 :]
2317 if ends_with_comma:
2318 right_leaves.pop()
2320 if old_parens_exist:
2321 assert right_leaves and right_leaves[-1].type == token.RPAR, (
2322 "Apparently, old parentheses do NOT exist?!"
2323 f" (left_leaves={left_leaves}, right_leaves={right_leaves})"
2324 )
2325 old_rpar_leaf = right_leaves.pop()
2326 elif right_leaves and right_leaves[-1].type == token.RPAR:
2327 # Special case for lambda expressions as dict's value, e.g.:
2328 # my_dict = {
2329 # "key": lambda x: f"formatted: {x}",
2330 # }
2331 # After wrapping the dict's value with parentheses, the string is
2332 # followed by a RPAR but its opening bracket is lambda's, not
2333 # the string's:
2334 # "key": (lambda x: f"formatted: {x}"),
2335 opening_bracket = right_leaves[-1].opening_bracket
2336 if opening_bracket is not None and opening_bracket in left_leaves:
2337 index = left_leaves.index(opening_bracket)
2338 if (
2339 0 < index < len(left_leaves) - 1
2340 and left_leaves[index - 1].type == token.COLON
2341 and left_leaves[index + 1].value == "lambda"
2342 ):
2343 right_leaves.pop()
2345 append_leaves(string_line, line, right_leaves)
2347 yield Ok(string_line)
2349 # --- Last Line
2350 last_line = line.clone()
2351 last_line.bracket_tracker = first_line.bracket_tracker
2353 new_rpar_leaf = Leaf(token.RPAR, ")")
2354 if old_rpar_leaf is not None:
2355 replace_child(old_rpar_leaf, new_rpar_leaf)
2356 else:
2357 insert_str_child(new_rpar_leaf)
2358 last_line.append(new_rpar_leaf)
2360 # If the target string ended with a comma, we place this comma to the
2361 # right of the RPAR on the last line.
2362 if ends_with_comma:
2363 comma_leaf = Leaf(token.COMMA, ",")
2364 replace_child(LL[comma_idx], comma_leaf)
2365 last_line.append(comma_leaf)
2367 yield Ok(last_line)
2370class StringParser:
2371 """
2372 A state machine that aids in parsing a string's "trailer", which can be
2373 either non-existent, an old-style formatting sequence (e.g. `% varX` or `%
2374 (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
2375 varY)`).
2377 NOTE: A new StringParser object MUST be instantiated for each string
2378 trailer we need to parse.
2380 Examples:
2381 We shall assume that `line` equals the `Line` object that corresponds
2382 to the following line of python code:
2383 ```
2384 x = "Some {}.".format("String") + some_other_string
2385 ```
2387 Furthermore, we will assume that `string_idx` is some index such that:
2388 ```
2389 assert line.leaves[string_idx].value == "Some {}."
2390 ```
2392 The following code snippet then holds:
2393 ```
2394 string_parser = StringParser()
2395 idx = string_parser.parse(line.leaves, string_idx)
2396 assert line.leaves[idx].type == token.PLUS
2397 ```
2398 """
2400 DEFAULT_TOKEN: Final = 20210605
2402 # String Parser States
2403 START: Final = 1
2404 DOT: Final = 2
2405 NAME: Final = 3
2406 PERCENT: Final = 4
2407 SINGLE_FMT_ARG: Final = 5
2408 LPAR: Final = 6
2409 RPAR: Final = 7
2410 DONE: Final = 8
2412 # Lookup Table for Next State
2413 _goto: Final[dict[tuple[ParserState, NodeType], ParserState]] = {
2414 # A string trailer may start with '.' OR '%'.
2415 (START, token.DOT): DOT,
2416 (START, token.PERCENT): PERCENT,
2417 (START, DEFAULT_TOKEN): DONE,
2418 # A '.' MUST be followed by an attribute or method name.
2419 (DOT, token.NAME): NAME,
2420 # A method name MUST be followed by an '(', whereas an attribute name
2421 # is the last symbol in the string trailer.
2422 (NAME, token.LPAR): LPAR,
2423 (NAME, DEFAULT_TOKEN): DONE,
2424 # A '%' symbol can be followed by an '(' or a single argument (e.g. a
2425 # string or variable name).
2426 (PERCENT, token.LPAR): LPAR,
2427 (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
2428 # If a '%' symbol is followed by a single argument, that argument is
2429 # the last leaf in the string trailer.
2430 (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
2431 # If present, a ')' symbol is the last symbol in a string trailer.
2432 # (NOTE: LPARS and nested RPARS are not included in this lookup table,
2433 # since they are treated as a special case by the parsing logic in this
2434 # classes' implementation.)
2435 (RPAR, DEFAULT_TOKEN): DONE,
2436 }
2438 def __init__(self) -> None:
2439 self._state = self.START
2440 self._unmatched_lpars = 0
2442 def parse(self, leaves: list[Leaf], string_idx: int) -> int:
2443 """
2444 Pre-conditions:
2445 * @leaves[@string_idx].type == token.STRING
2447 Returns:
2448 The index directly after the last leaf which is a part of the string
2449 trailer, if a "trailer" exists.
2450 OR
2451 @string_idx + 1, if no string "trailer" exists.
2452 """
2453 assert leaves[string_idx].type == token.STRING
2455 idx = string_idx + 1
2456 while idx < len(leaves) and self._next_state(leaves[idx]):
2457 idx += 1
2458 return idx
2460 def _next_state(self, leaf: Leaf) -> bool:
2461 """
2462 Pre-conditions:
2463 * On the first call to this function, @leaf MUST be the leaf that
2464 was directly after the string leaf in question (e.g. if our target
2465 string is `line.leaves[i]` then the first call to this method must
2466 be `line.leaves[i + 1]`).
2467 * On the next call to this function, the leaf parameter passed in
2468 MUST be the leaf directly following @leaf.
2470 Returns:
2471 True iff @leaf is a part of the string's trailer.
2472 """
2473 # We ignore empty LPAR or RPAR leaves.
2474 if is_empty_par(leaf):
2475 return True
2477 next_token = leaf.type
2478 if next_token == token.LPAR:
2479 self._unmatched_lpars += 1
2481 current_state = self._state
2483 # The LPAR parser state is a special case. We will return True until we
2484 # find the matching RPAR token.
2485 if current_state == self.LPAR:
2486 if next_token == token.RPAR:
2487 self._unmatched_lpars -= 1
2488 if self._unmatched_lpars == 0:
2489 self._state = self.RPAR
2490 # Otherwise, we use a lookup table to determine the next state.
2491 else:
2492 # If the lookup table matches the current state to the next
2493 # token, we use the lookup table.
2494 if (current_state, next_token) in self._goto:
2495 self._state = self._goto[current_state, next_token]
2496 else:
2497 # Otherwise, we check if a the current state was assigned a
2498 # default.
2499 if (current_state, self.DEFAULT_TOKEN) in self._goto:
2500 self._state = self._goto[current_state, self.DEFAULT_TOKEN]
2501 # If no default has been assigned, then this parser has a logic
2502 # error.
2503 else:
2504 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
2506 if self._state == self.DONE:
2507 return False
2509 return True
2512def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
2513 """
2514 Factory for a convenience function that is used to orphan @string_leaf
2515 and then insert multiple new leaves into the same part of the node
2516 structure that @string_leaf had originally occupied.
2518 Examples:
2519 Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
2520 string_leaf.parent`. Assume the node `N` has the following
2521 original structure:
2523 Node(
2524 expr_stmt, [
2525 Leaf(NAME, 'x'),
2526 Leaf(EQUAL, '='),
2527 Leaf(STRING, '"foo"'),
2528 ]
2529 )
2531 We then run the code snippet shown below.
2532 ```
2533 insert_str_child = insert_str_child_factory(string_leaf)
2535 lpar = Leaf(token.LPAR, '(')
2536 insert_str_child(lpar)
2538 bar = Leaf(token.STRING, '"bar"')
2539 insert_str_child(bar)
2541 rpar = Leaf(token.RPAR, ')')
2542 insert_str_child(rpar)
2543 ```
2545 After which point, it follows that `string_leaf.parent is None` and
2546 the node `N` now has the following structure:
2548 Node(
2549 expr_stmt, [
2550 Leaf(NAME, 'x'),
2551 Leaf(EQUAL, '='),
2552 Leaf(LPAR, '('),
2553 Leaf(STRING, '"bar"'),
2554 Leaf(RPAR, ')'),
2555 ]
2556 )
2557 """
2558 string_parent = string_leaf.parent
2559 string_child_idx = string_leaf.remove()
2561 def insert_str_child(child: LN) -> None:
2562 nonlocal string_child_idx
2564 assert string_parent is not None
2565 assert string_child_idx is not None
2567 string_parent.insert_child(string_child_idx, child)
2568 string_child_idx += 1
2570 return insert_str_child
2573def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
2574 """
2575 Examples:
2576 ```
2577 my_list = [1, 2, 3]
2579 is_valid_index = is_valid_index_factory(my_list)
2581 assert is_valid_index(0)
2582 assert is_valid_index(2)
2584 assert not is_valid_index(3)
2585 assert not is_valid_index(-1)
2586 ```
2587 """
2589 def is_valid_index(idx: int) -> bool:
2590 """
2591 Returns:
2592 True iff @idx is positive AND seq[@idx] does NOT raise an
2593 IndexError.
2594 """
2595 return 0 <= idx < len(seq)
2597 return is_valid_index