Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/trans.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2String transformers that can split and merge strings.
3"""
5import re
6from abc import ABC, abstractmethod
7from collections import defaultdict
8from collections.abc import Callable, Collection, Iterable, Iterator, Sequence
9from dataclasses import dataclass
10from typing import Any, ClassVar, Final, Literal, TypeVar, Union
12from mypy_extensions import trait
14from black.comments import contains_pragma_comment
15from black.lines import Line, append_leaves
16from black.mode import Feature, Mode
17from black.nodes import (
18 CLOSING_BRACKETS,
19 OPENING_BRACKETS,
20 STANDALONE_COMMENT,
21 is_empty_lpar,
22 is_empty_par,
23 is_empty_rpar,
24 is_part_of_annotation,
25 parent_type,
26 replace_child,
27 syms,
28)
29from black.rusty import Err, Ok, Result
30from black.strings import (
31 assert_is_leaf_string,
32 count_chars_in_width,
33 get_string_prefix,
34 has_triple_quotes,
35 normalize_string_quotes,
36 str_width,
37)
38from blib2to3.pgen2 import token
39from blib2to3.pytree import Leaf, Node
42class CannotTransform(Exception):
43 """Base class for errors raised by Transformers."""
46# types
47T = TypeVar("T")
48LN = Union[Leaf, Node]
49Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]
50Index = int
51NodeType = int
52ParserState = int
53StringID = int
54TResult = Result[T, CannotTransform] # (T)ransform Result
55TMatchResult = TResult[list[Index]]
57SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops
60def TErr(err_msg: str) -> Err[CannotTransform]:
61 """(T)ransform Err
63 Convenience function used when working with the TResult type.
64 """
65 cant_transform = CannotTransform(err_msg)
66 return Err(cant_transform)
69# Remove when `simplify_power_operator_hugging` becomes stable.
70def hug_power_op(
71 line: Line, features: Collection[Feature], mode: Mode
72) -> Iterator[Line]:
73 """A transformer which normalizes spacing around power operators."""
75 # Performance optimization to avoid unnecessary Leaf clones and other ops.
76 for leaf in line.leaves:
77 if leaf.type == token.DOUBLESTAR:
78 break
79 else:
80 raise CannotTransform("No doublestar token was found in the line.")
82 def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool:
83 # Brackets and parentheses indicate calls, subscripts, etc. ...
84 # basically stuff that doesn't count as "simple". Only a NAME lookup
85 # or dotted lookup (eg. NAME.NAME) is OK.
86 if kind == -1:
87 return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB})
88 else:
89 return handle_is_simple_lookup_forward(
90 line, index, {token.LPAR, token.LSQB}
91 )
93 def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool:
94 # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple
95 # lookup (see above), with or without a preceding unary operator.
96 start = line.leaves[index]
97 if start.type in {token.NAME, token.NUMBER}:
98 return is_simple_lookup(index, kind)
100 if start.type in {token.PLUS, token.MINUS, token.TILDE}:
101 if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:
102 # kind is always one as bases with a preceding unary op will be checked
103 # for simplicity starting from the next token (so it'll hit the check
104 # above).
105 return is_simple_lookup(index + 1, kind=1)
107 return False
109 new_line = line.clone()
110 should_hug = False
111 for idx, leaf in enumerate(line.leaves):
112 new_leaf = leaf.clone()
113 if should_hug:
114 new_leaf.prefix = ""
115 should_hug = False
117 should_hug = (
118 (0 < idx < len(line.leaves) - 1)
119 and leaf.type == token.DOUBLESTAR
120 and is_simple_operand(idx - 1, kind=-1)
121 and line.leaves[idx - 1].value != "lambda"
122 and is_simple_operand(idx + 1, kind=1)
123 )
124 if should_hug:
125 new_leaf.prefix = ""
127 # We have to be careful to make a new line properly:
128 # - bracket related metadata must be maintained (handled by Line.append)
129 # - comments need to copied over, updating the leaf IDs they're attached to
130 new_line.append(new_leaf, preformatted=True)
131 for comment_leaf in line.comments_after(leaf):
132 new_line.append(comment_leaf, preformatted=True)
134 yield new_line
137# Remove when `simplify_power_operator_hugging` becomes stable.
138def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool:
139 """
140 Handling the determination of is_simple_lookup for the lines prior to the doublestar
141 token. This is required because of the need to isolate the chained expression
142 to determine the bracket or parenthesis belong to the single expression.
143 """
144 contains_disallowed = False
145 chain = []
147 while 0 <= index < len(line.leaves):
148 current = line.leaves[index]
149 chain.append(current)
150 if not contains_disallowed and current.type in disallowed:
151 contains_disallowed = True
152 if not is_expression_chained(chain):
153 return not contains_disallowed
155 index -= 1
157 return True
160# Remove when `simplify_power_operator_hugging` becomes stable.
161def handle_is_simple_lookup_forward(
162 line: Line, index: int, disallowed: set[int]
163) -> bool:
164 """
165 Handling decision is_simple_lookup for the lines behind the doublestar token.
166 This function is simplified to keep consistent with the prior logic and the forward
167 case are more straightforward and do not need to care about chained expressions.
168 """
169 while 0 <= index < len(line.leaves):
170 current = line.leaves[index]
171 if current.type in disallowed:
172 return False
173 if current.type not in {token.NAME, token.DOT} or (
174 current.type == token.NAME and current.value == "for"
175 ):
176 # If the current token isn't disallowed, we'll assume this is simple as
177 # only the disallowed tokens are semantically attached to this lookup
178 # expression we're checking. Also, stop early if we hit the 'for' bit
179 # of a comprehension.
180 return True
182 index += 1
184 return True
187# Remove when `simplify_power_operator_hugging` becomes stable.
188def is_expression_chained(chained_leaves: list[Leaf]) -> bool:
189 """
190 Function to determine if the variable is a chained call.
191 (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call)
192 """
193 if len(chained_leaves) < 2:
194 return True
196 current_leaf = chained_leaves[-1]
197 past_leaf = chained_leaves[-2]
199 if past_leaf.type == token.NAME:
200 return current_leaf.type in {token.DOT}
201 elif past_leaf.type in {token.RPAR, token.RSQB}:
202 return current_leaf.type in {token.RSQB, token.RPAR}
203 elif past_leaf.type in {token.LPAR, token.LSQB}:
204 return current_leaf.type in {token.NAME, token.LPAR, token.LSQB}
205 else:
206 return False
209class StringTransformer(ABC):
210 """
211 An implementation of the Transformer protocol that relies on its
212 subclasses overriding the template methods `do_match(...)` and
213 `do_transform(...)`.
215 This Transformer works exclusively on strings (for example, by merging
216 or splitting them).
218 The following sections can be found among the docstrings of each concrete
219 StringTransformer subclass.
221 Requirements:
222 Which requirements must be met of the given Line for this
223 StringTransformer to be applied?
225 Transformations:
226 If the given Line meets all of the above requirements, which string
227 transformations can you expect to be applied to it by this
228 StringTransformer?
230 Collaborations:
231 What contractual agreements does this StringTransformer have with other
232 StringTransformers? Such collaborations should be eliminated/minimized
233 as much as possible.
234 """
236 __name__: Final = "StringTransformer"
238 # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
239 # `abc.ABC`.
240 def __init__(self, line_length: int, normalize_strings: bool) -> None:
241 self.line_length = line_length
242 self.normalize_strings = normalize_strings
244 @abstractmethod
245 def do_match(self, line: Line) -> TMatchResult:
246 """
247 Returns:
248 * Ok(string_indices) such that for each index, `line.leaves[index]`
249 is our target string if a match was able to be made. For
250 transformers that don't result in more lines (e.g. StringMerger,
251 StringParenStripper), multiple matches and transforms are done at
252 once to reduce the complexity.
253 OR
254 * Err(CannotTransform), if no match could be made.
255 """
257 @abstractmethod
258 def do_transform(
259 self, line: Line, string_indices: list[int]
260 ) -> Iterator[TResult[Line]]:
261 """
262 Yields:
263 * Ok(new_line) where new_line is the new transformed line.
264 OR
265 * Err(CannotTransform) if the transformation failed for some reason. The
266 `do_match(...)` template method should usually be used to reject
267 the form of the given Line, but in some cases it is difficult to
268 know whether or not a Line meets the StringTransformer's
269 requirements until the transformation is already midway.
271 Side Effects:
272 This method should NOT mutate @line directly, but it MAY mutate the
273 Line's underlying Node structure. (WARNING: If the underlying Node
274 structure IS altered, then this method should NOT be allowed to
275 yield an CannotTransform after that point.)
276 """
278 def __call__(
279 self, line: Line, _features: Collection[Feature], _mode: Mode
280 ) -> Iterator[Line]:
281 """
282 StringTransformer instances have a call signature that mirrors that of
283 the Transformer type.
285 Raises:
286 CannotTransform(...) if the concrete StringTransformer class is unable
287 to transform @line.
288 """
289 # Optimization to avoid calling `self.do_match(...)` when the line does
290 # not contain any string.
291 if not any(leaf.type == token.STRING for leaf in line.leaves):
292 raise CannotTransform("There are no strings in this line.")
294 match_result = self.do_match(line)
296 if isinstance(match_result, Err):
297 cant_transform = match_result.err()
298 raise CannotTransform(
299 f"The string transformer {self.__class__.__name__} does not recognize"
300 " this line as one that it can transform."
301 ) from cant_transform
303 string_indices = match_result.ok()
305 for line_result in self.do_transform(line, string_indices):
306 if isinstance(line_result, Err):
307 cant_transform = line_result.err()
308 raise CannotTransform(
309 "StringTransformer failed while attempting to transform string."
310 ) from cant_transform
311 line = line_result.ok()
312 yield line
315@dataclass
316class CustomSplit:
317 """A custom (i.e. manual) string split.
319 A single CustomSplit instance represents a single substring.
321 Examples:
322 Consider the following string:
323 ```
324 "Hi there friend."
325 " This is a custom"
326 f" string {split}."
327 ```
329 This string will correspond to the following three CustomSplit instances:
330 ```
331 CustomSplit(False, 16)
332 CustomSplit(False, 17)
333 CustomSplit(True, 16)
334 ```
335 """
337 has_prefix: bool
338 break_idx: int
341CustomSplitMapKey = tuple[StringID, str]
344@trait
345class CustomSplitMapMixin:
346 """
347 This mixin class is used to map merged strings to a sequence of
348 CustomSplits, which will then be used to re-split the strings iff none of
349 the resultant substrings go over the configured max line length.
350 """
352 _CUSTOM_SPLIT_MAP: ClassVar[dict[CustomSplitMapKey, tuple[CustomSplit, ...]]] = (
353 defaultdict(tuple)
354 )
356 @staticmethod
357 def _get_key(string: str) -> CustomSplitMapKey:
358 """
359 Returns:
360 A unique identifier that is used internally to map @string to a
361 group of custom splits.
362 """
363 return (id(string), string)
365 def add_custom_splits(
366 self, string: str, custom_splits: Iterable[CustomSplit]
367 ) -> None:
368 """Custom Split Map Setter Method
370 Side Effects:
371 Adds a mapping from @string to the custom splits @custom_splits.
372 """
373 key = self._get_key(string)
374 self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
376 def pop_custom_splits(self, string: str) -> list[CustomSplit]:
377 """Custom Split Map Getter Method
379 Returns:
380 * A list of the custom splits that are mapped to @string, if any
381 exist.
382 OR
383 * [], otherwise.
385 Side Effects:
386 Deletes the mapping between @string and its associated custom
387 splits (which are returned to the caller).
388 """
389 key = self._get_key(string)
391 custom_splits = self._CUSTOM_SPLIT_MAP[key]
392 del self._CUSTOM_SPLIT_MAP[key]
394 return list(custom_splits)
396 def has_custom_splits(self, string: str) -> bool:
397 """
398 Returns:
399 True iff @string is associated with a set of custom splits.
400 """
401 key = self._get_key(string)
402 return key in self._CUSTOM_SPLIT_MAP
405class StringMerger(StringTransformer, CustomSplitMapMixin):
406 """StringTransformer that merges strings together.
408 Requirements:
409 (A) The line contains adjacent strings such that ALL of the validation checks
410 listed in StringMerger._validate_msg(...)'s docstring pass.
411 OR
412 (B) The line contains a string which uses line continuation backslashes.
414 Transformations:
415 Depending on which of the two requirements above where met, either:
417 (A) The string group associated with the target string is merged.
418 OR
419 (B) All line-continuation backslashes are removed from the target string.
421 Collaborations:
422 StringMerger provides custom split information to StringSplitter.
423 """
425 def do_match(self, line: Line) -> TMatchResult:
426 LL = line.leaves
428 is_valid_index = is_valid_index_factory(LL)
430 string_indices = []
431 idx = 0
432 while is_valid_index(idx):
433 leaf = LL[idx]
434 if (
435 leaf.type == token.STRING
436 and is_valid_index(idx + 1)
437 and LL[idx + 1].type == token.STRING
438 ):
439 # Let's check if the string group contains an inline comment
440 # If we have a comment inline, we don't merge the strings
441 contains_comment = False
442 i = idx
443 while is_valid_index(i):
444 if LL[i].type != token.STRING:
445 break
446 if line.comments_after(LL[i]):
447 contains_comment = True
448 break
449 i += 1
451 if not contains_comment and not is_part_of_annotation(leaf):
452 string_indices.append(idx)
454 # Advance to the next non-STRING leaf.
455 idx += 2
456 while is_valid_index(idx) and LL[idx].type == token.STRING:
457 idx += 1
459 elif leaf.type == token.STRING and "\\\n" in leaf.value:
460 string_indices.append(idx)
461 # Advance to the next non-STRING leaf.
462 idx += 1
463 while is_valid_index(idx) and LL[idx].type == token.STRING:
464 idx += 1
466 else:
467 idx += 1
469 if string_indices:
470 return Ok(string_indices)
471 else:
472 return TErr("This line has no strings that need merging.")
474 def do_transform(
475 self, line: Line, string_indices: list[int]
476 ) -> Iterator[TResult[Line]]:
477 new_line = line
479 rblc_result = self._remove_backslash_line_continuation_chars(
480 new_line, string_indices
481 )
482 if isinstance(rblc_result, Ok):
483 new_line = rblc_result.ok()
485 msg_result = self._merge_string_group(new_line, string_indices)
486 if isinstance(msg_result, Ok):
487 new_line = msg_result.ok()
489 if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
490 msg_cant_transform = msg_result.err()
491 rblc_cant_transform = rblc_result.err()
492 cant_transform = CannotTransform(
493 "StringMerger failed to merge any strings in this line."
494 )
496 # Chain the errors together using `__cause__`.
497 msg_cant_transform.__cause__ = rblc_cant_transform
498 cant_transform.__cause__ = msg_cant_transform
500 yield Err(cant_transform)
501 else:
502 yield Ok(new_line)
504 @staticmethod
505 def _remove_backslash_line_continuation_chars(
506 line: Line, string_indices: list[int]
507 ) -> TResult[Line]:
508 """
509 Merge strings that were split across multiple lines using
510 line-continuation backslashes.
512 Returns:
513 Ok(new_line), if @line contains backslash line-continuation
514 characters.
515 OR
516 Err(CannotTransform), otherwise.
517 """
518 LL = line.leaves
520 indices_to_transform = []
521 for string_idx in string_indices:
522 string_leaf = LL[string_idx]
523 if (
524 string_leaf.type == token.STRING
525 and "\\\n" in string_leaf.value
526 and not has_triple_quotes(string_leaf.value)
527 ):
528 indices_to_transform.append(string_idx)
530 if not indices_to_transform:
531 return TErr(
532 "Found no string leaves that contain backslash line continuation"
533 " characters."
534 )
536 new_line = line.clone()
537 new_line.comments = line.comments.copy()
538 append_leaves(new_line, line, LL)
540 for string_idx in indices_to_transform:
541 new_string_leaf = new_line.leaves[string_idx]
542 new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
544 return Ok(new_line)
546 def _merge_string_group(
547 self, line: Line, string_indices: list[int]
548 ) -> TResult[Line]:
549 """
550 Merges string groups (i.e. set of adjacent strings).
552 Each index from `string_indices` designates one string group's first
553 leaf in `line.leaves`.
555 Returns:
556 Ok(new_line), if ALL of the validation checks found in
557 _validate_msg(...) pass.
558 OR
559 Err(CannotTransform), otherwise.
560 """
561 LL = line.leaves
563 is_valid_index = is_valid_index_factory(LL)
565 # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.
566 merged_string_idx_dict: dict[int, tuple[int, Leaf]] = {}
567 for string_idx in string_indices:
568 vresult = self._validate_msg(line, string_idx)
569 if isinstance(vresult, Err):
570 continue
571 merged_string_idx_dict[string_idx] = self._merge_one_string_group(
572 LL, string_idx, is_valid_index
573 )
575 if not merged_string_idx_dict:
576 return TErr("No string group is merged")
578 # Build the final line ('new_line') that this method will later return.
579 new_line = line.clone()
580 previous_merged_string_idx = -1
581 previous_merged_num_of_strings = -1
582 for i, leaf in enumerate(LL):
583 if i in merged_string_idx_dict:
584 previous_merged_string_idx = i
585 previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]
586 new_line.append(string_leaf)
588 if (
589 previous_merged_string_idx
590 <= i
591 < previous_merged_string_idx + previous_merged_num_of_strings
592 ):
593 for comment_leaf in line.comments_after(leaf):
594 new_line.append(comment_leaf, preformatted=True)
595 continue
597 append_leaves(new_line, line, [leaf])
599 return Ok(new_line)
601 def _merge_one_string_group(
602 self, LL: list[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]
603 ) -> tuple[int, Leaf]:
604 """
605 Merges one string group where the first string in the group is
606 `LL[string_idx]`.
608 Returns:
609 A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the
610 number of strings merged and `leaf` is the newly merged string
611 to be replaced in the new line.
612 """
613 # If the string group is wrapped inside an Atom node, we must make sure
614 # to later replace that Atom with our new (merged) string leaf.
615 atom_node = LL[string_idx].parent
617 # We will place BREAK_MARK in between every two substrings that we
618 # merge. We will then later go through our final result and use the
619 # various instances of BREAK_MARK we find to add the right values to
620 # the custom split map.
621 BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
623 QUOTE = LL[string_idx].value[-1]
625 def make_naked(string: str, string_prefix: str) -> str:
626 """Strip @string (i.e. make it a "naked" string)
628 Pre-conditions:
629 * assert_is_leaf_string(@string)
631 Returns:
632 A string that is identical to @string except that
633 @string_prefix has been stripped, the surrounding QUOTE
634 characters have been removed, and any remaining QUOTE
635 characters have been escaped.
636 """
637 assert_is_leaf_string(string)
638 if "f" in string_prefix:
639 f_expressions = [
640 string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces
641 for span in iter_fexpr_spans(string)
642 ]
643 debug_expressions_contain_visible_quotes = any(
644 re.search(r".*[\'\"].*(?<![!:=])={1}(?!=)(?![^\s:])", expression)
645 for expression in f_expressions
646 )
647 if not debug_expressions_contain_visible_quotes:
648 # We don't want to toggle visible quotes in debug f-strings, as
649 # that would modify the AST
650 string = _toggle_fexpr_quotes(string, QUOTE)
651 # After quotes toggling, quotes in expressions won't be escaped
652 # because quotes can't be reused in f-strings. So we can simply
653 # let the escaping logic below run without knowing f-string
654 # expressions.
656 RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
657 naked_string = string[len(string_prefix) + 1 : -1]
658 naked_string = re.sub(
659 "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
660 )
661 return naked_string
663 # Holds the CustomSplit objects that will later be added to the custom
664 # split map.
665 custom_splits = []
667 # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
668 prefix_tracker = []
670 # Sets the 'prefix' variable. This is the prefix that the final merged
671 # string will have.
672 next_str_idx = string_idx
673 prefix = ""
674 while (
675 not prefix
676 and is_valid_index(next_str_idx)
677 and LL[next_str_idx].type == token.STRING
678 ):
679 prefix = get_string_prefix(LL[next_str_idx].value).lower()
680 next_str_idx += 1
682 # The next loop merges the string group. The final string will be
683 # contained in 'S'.
684 #
685 # The following convenience variables are used:
686 #
687 # S: string
688 # NS: naked string
689 # SS: next string
690 # NSS: naked next string
691 S = ""
692 NS = ""
693 num_of_strings = 0
694 next_str_idx = string_idx
695 while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
696 num_of_strings += 1
698 SS = LL[next_str_idx].value
699 next_prefix = get_string_prefix(SS).lower()
701 # If this is an f-string group but this substring is not prefixed
702 # with 'f'...
703 if "f" in prefix and "f" not in next_prefix:
704 # Then we must escape any braces contained in this substring.
705 SS = re.sub(r"(\{|\})", r"\1\1", SS)
707 NSS = make_naked(SS, next_prefix)
709 has_prefix = bool(next_prefix)
710 prefix_tracker.append(has_prefix)
712 S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE
713 NS = make_naked(S, prefix)
715 next_str_idx += 1
717 # Take a note on the index of the non-STRING leaf.
718 non_string_idx = next_str_idx
720 S_leaf = Leaf(token.STRING, S)
721 if self.normalize_strings:
722 S_leaf.value = normalize_string_quotes(S_leaf.value)
724 # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
725 temp_string = S_leaf.value[len(prefix) + 1 : -1]
726 for has_prefix in prefix_tracker:
727 mark_idx = temp_string.find(BREAK_MARK)
728 assert (
729 mark_idx >= 0
730 ), "Logic error while filling the custom string breakpoint cache."
732 temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
733 breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
734 custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
736 string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
738 if atom_node is not None:
739 # If not all children of the atom node are merged (this can happen
740 # when there is a standalone comment in the middle) ...
741 if non_string_idx - string_idx < len(atom_node.children):
742 # We need to replace the old STRING leaves with the new string leaf.
743 first_child_idx = LL[string_idx].remove()
744 for idx in range(string_idx + 1, non_string_idx):
745 LL[idx].remove()
746 if first_child_idx is not None:
747 atom_node.insert_child(first_child_idx, string_leaf)
748 else:
749 # Else replace the atom node with the new string leaf.
750 replace_child(atom_node, string_leaf)
752 self.add_custom_splits(string_leaf.value, custom_splits)
753 return num_of_strings, string_leaf
755 @staticmethod
756 def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
757 """Validate (M)erge (S)tring (G)roup
759 Transform-time string validation logic for _merge_string_group(...).
761 Returns:
762 * Ok(None), if ALL validation checks (listed below) pass.
763 OR
764 * Err(CannotTransform), if any of the following are true:
765 - The target string group does not contain ANY stand-alone comments.
766 - The target string is not in a string group (i.e. it has no
767 adjacent strings).
768 - The string group has more than one inline comment.
769 - The string group has an inline comment that appears to be a pragma.
770 - The set of all string prefixes in the string group is of
771 length greater than one and is not equal to {"", "f"}.
772 - The string group consists of raw strings.
773 - The string group would merge f-strings with different quote types
774 and internal quotes.
775 - The string group is stringified type annotations. We don't want to
776 process stringified type annotations since pyright doesn't support
777 them spanning multiple string values. (NOTE: mypy, pytype, pyre do
778 support them, so we can change if pyright also gains support in the
779 future. See https://github.com/microsoft/pyright/issues/4359.)
780 """
781 # We first check for "inner" stand-alone comments (i.e. stand-alone
782 # comments that have a string leaf before them AND after them).
783 for inc in [1, -1]:
784 i = string_idx
785 found_sa_comment = False
786 is_valid_index = is_valid_index_factory(line.leaves)
787 while is_valid_index(i) and line.leaves[i].type in [
788 token.STRING,
789 STANDALONE_COMMENT,
790 ]:
791 if line.leaves[i].type == STANDALONE_COMMENT:
792 found_sa_comment = True
793 elif found_sa_comment:
794 return TErr(
795 "StringMerger does NOT merge string groups which contain "
796 "stand-alone comments."
797 )
799 i += inc
801 QUOTE = line.leaves[string_idx].value[-1]
803 num_of_inline_string_comments = 0
804 set_of_prefixes = set()
805 num_of_strings = 0
806 for leaf in line.leaves[string_idx:]:
807 if leaf.type != token.STRING:
808 # If the string group is trailed by a comma, we count the
809 # comments trailing the comma to be one of the string group's
810 # comments.
811 if leaf.type == token.COMMA and id(leaf) in line.comments:
812 num_of_inline_string_comments += 1
813 break
815 if has_triple_quotes(leaf.value):
816 return TErr("StringMerger does NOT merge multiline strings.")
818 num_of_strings += 1
819 prefix = get_string_prefix(leaf.value).lower()
820 if "r" in prefix:
821 return TErr("StringMerger does NOT merge raw strings.")
823 set_of_prefixes.add(prefix)
825 if (
826 "f" in prefix
827 and leaf.value[-1] != QUOTE
828 and (
829 "'" in leaf.value[len(prefix) + 1 : -1]
830 or '"' in leaf.value[len(prefix) + 1 : -1]
831 )
832 ):
833 return TErr(
834 "StringMerger does NOT merge f-strings with different quote types"
835 " and internal quotes."
836 )
838 if id(leaf) in line.comments:
839 num_of_inline_string_comments += 1
840 if contains_pragma_comment(line.comments[id(leaf)]):
841 return TErr("Cannot merge strings which have pragma comments.")
843 if num_of_strings < 2:
844 return TErr(
845 f"Not enough strings to merge (num_of_strings={num_of_strings})."
846 )
848 # Also check for pragma comments on tokens that follow the string
849 # group (e.g. a closing bracket). Merging strings when a pragma
850 # comment like `# type: ignore` follows would produce an unsplittable
851 # long line.
852 is_valid_index = is_valid_index_factory(line.leaves)
853 next_idx = string_idx + num_of_strings
854 while is_valid_index(next_idx):
855 next_leaf = line.leaves[next_idx]
856 if id(next_leaf) in line.comments:
857 if contains_pragma_comment(line.comments[id(next_leaf)]):
858 return TErr(
859 "Cannot merge strings when a pragma comment follows"
860 " the string group."
861 )
862 if next_leaf.type not in CLOSING_BRACKETS:
863 break
864 next_idx += 1
866 if num_of_inline_string_comments > 1:
867 return TErr(
868 f"Too many inline string comments ({num_of_inline_string_comments})."
869 )
871 if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
872 return TErr(f"Too many different prefixes ({set_of_prefixes}).")
874 return Ok(None)
877class StringParenStripper(StringTransformer):
878 """StringTransformer that strips surrounding parentheses from strings.
880 Requirements:
881 The line contains a string which is surrounded by parentheses and:
882 - The target string is NOT the only argument to a function call.
883 - The target string is NOT a "pointless" string.
884 - The target string is NOT a dictionary value.
885 - If the target string contains a PERCENT, the brackets are not
886 preceded or followed by an operator with higher precedence than
887 PERCENT.
889 Transformations:
890 The parentheses mentioned in the 'Requirements' section are stripped.
892 Collaborations:
893 StringParenStripper has its own inherent usefulness, but it is also
894 relied on to clean up the parentheses created by StringParenWrapper (in
895 the event that they are no longer needed).
896 """
898 def do_match(self, line: Line) -> TMatchResult:
899 LL = line.leaves
901 is_valid_index = is_valid_index_factory(LL)
903 string_indices = []
905 idx = -1
906 while True:
907 idx += 1
908 if idx >= len(LL):
909 break
910 leaf = LL[idx]
912 # Should be a string...
913 if leaf.type != token.STRING:
914 continue
916 # If this is a "pointless" string...
917 if (
918 leaf.parent
919 and leaf.parent.parent
920 and leaf.parent.parent.type == syms.simple_stmt
921 ):
922 continue
924 # Should be preceded by a non-empty LPAR...
925 if (
926 not is_valid_index(idx - 1)
927 or LL[idx - 1].type != token.LPAR
928 or is_empty_lpar(LL[idx - 1])
929 ):
930 continue
932 # That LPAR should NOT be preceded by a colon (which could be a
933 # dictionary value), function name, or a closing bracket (which
934 # could be a function returning a function or a list/dictionary
935 # containing a function)...
936 if is_valid_index(idx - 2) and (
937 LL[idx - 2].type == token.COLON
938 or LL[idx - 2].type == token.NAME
939 or LL[idx - 2].type in CLOSING_BRACKETS
940 ):
941 continue
943 string_idx = idx
945 # Skip the string trailer, if one exists.
946 string_parser = StringParser()
947 next_idx = string_parser.parse(LL, string_idx)
949 # if the leaves in the parsed string include a PERCENT, we need to
950 # make sure the initial LPAR is NOT preceded by an operator with
951 # higher or equal precedence to PERCENT
952 if is_valid_index(idx - 2):
953 # mypy can't quite follow unless we name this
954 before_lpar = LL[idx - 2]
955 if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
956 (
957 before_lpar.type in {
958 token.STAR,
959 token.AT,
960 token.SLASH,
961 token.DOUBLESLASH,
962 token.PERCENT,
963 token.TILDE,
964 token.DOUBLESTAR,
965 token.AWAIT,
966 token.LSQB,
967 token.LPAR,
968 }
969 )
970 or (
971 # only unary PLUS/MINUS
972 before_lpar.parent
973 and before_lpar.parent.type == syms.factor
974 and (before_lpar.type in {token.PLUS, token.MINUS})
975 )
976 ):
977 continue
979 # Should be followed by a non-empty RPAR...
980 if (
981 is_valid_index(next_idx)
982 and LL[next_idx].type == token.RPAR
983 and not is_empty_rpar(LL[next_idx])
984 ):
985 # That RPAR should NOT be followed by anything with higher
986 # precedence than PERCENT
987 if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
988 token.DOUBLESTAR,
989 token.LSQB,
990 token.LPAR,
991 token.DOT,
992 }:
993 continue
995 string_indices.append(string_idx)
996 idx = string_idx
997 while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING:
998 idx += 1
1000 if string_indices:
1001 return Ok(string_indices)
1002 return TErr("This line has no strings wrapped in parens.")
1004 def do_transform(
1005 self, line: Line, string_indices: list[int]
1006 ) -> Iterator[TResult[Line]]:
1007 LL = line.leaves
1009 string_and_rpar_indices: list[int] = []
1010 for string_idx in string_indices:
1011 string_parser = StringParser()
1012 rpar_idx = string_parser.parse(LL, string_idx)
1014 should_transform = True
1015 for leaf in (LL[string_idx - 1], LL[rpar_idx]):
1016 if line.comments_after(leaf):
1017 # Should not strip parentheses which have comments attached
1018 # to them.
1019 should_transform = False
1020 break
1021 if should_transform:
1022 string_and_rpar_indices.extend((string_idx, rpar_idx))
1024 if string_and_rpar_indices:
1025 yield Ok(self._transform_to_new_line(line, string_and_rpar_indices))
1026 else:
1027 yield Err(
1028 CannotTransform("All string groups have comments attached to them.")
1029 )
1031 def _transform_to_new_line(
1032 self, line: Line, string_and_rpar_indices: list[int]
1033 ) -> Line:
1034 LL = line.leaves
1036 new_line = line.clone()
1037 new_line.comments = line.comments.copy()
1039 previous_idx = -1
1040 # We need to sort the indices, since string_idx and its matching
1041 # rpar_idx may not come in order, e.g. in
1042 # `("outer" % ("inner".join(items)))`, the "inner" string's
1043 # string_idx is smaller than "outer" string's rpar_idx.
1044 for idx in sorted(string_and_rpar_indices):
1045 leaf = LL[idx]
1046 lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx
1047 append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx])
1048 if leaf.type == token.STRING:
1049 string_leaf = Leaf(token.STRING, LL[idx].value)
1050 LL[lpar_or_rpar_idx].remove() # Remove lpar.
1051 replace_child(LL[idx], string_leaf)
1052 new_line.append(string_leaf)
1053 # replace comments
1054 old_comments = new_line.comments.pop(id(LL[idx]), [])
1055 new_line.comments.setdefault(id(string_leaf), []).extend(old_comments)
1056 else:
1057 LL[lpar_or_rpar_idx].remove() # This is a rpar.
1059 previous_idx = idx
1061 # Append the leaves after the last idx:
1062 append_leaves(new_line, line, LL[idx + 1 :])
1064 return new_line
1067class BaseStringSplitter(StringTransformer):
1068 """
1069 Abstract class for StringTransformers which transform a Line's strings by splitting
1070 them or placing them on their own lines where necessary to avoid going over
1071 the configured line length.
1073 Requirements:
1074 * The target string value is responsible for the line going over the
1075 line length limit. It follows that after all of black's other line
1076 split methods have been exhausted, this line (or one of the resulting
1077 lines after all line splits are performed) would still be over the
1078 line_length limit unless we split this string.
1079 AND
1081 * The target string is NOT a "pointless" string (i.e. a string that has
1082 no parent or siblings).
1083 AND
1085 * The target string is not followed by an inline comment that appears
1086 to be a pragma.
1087 AND
1089 * The target string is not a multiline (i.e. triple-quote) string.
1090 """
1092 STRING_OPERATORS: Final = [
1093 token.EQEQUAL,
1094 token.GREATER,
1095 token.GREATEREQUAL,
1096 token.LESS,
1097 token.LESSEQUAL,
1098 token.NOTEQUAL,
1099 token.PERCENT,
1100 token.PLUS,
1101 token.STAR,
1102 ]
1104 @abstractmethod
1105 def do_splitter_match(self, line: Line) -> TMatchResult:
1106 """
1107 BaseStringSplitter asks its clients to override this method instead of
1108 `StringTransformer.do_match(...)`.
1110 Follows the same protocol as `StringTransformer.do_match(...)`.
1112 Refer to `help(StringTransformer.do_match)` for more information.
1113 """
1115 def do_match(self, line: Line) -> TMatchResult:
1116 match_result = self.do_splitter_match(line)
1117 if isinstance(match_result, Err):
1118 return match_result
1120 string_indices = match_result.ok()
1121 assert len(string_indices) == 1, (
1122 f"{self.__class__.__name__} should only find one match at a time, found"
1123 f" {len(string_indices)}"
1124 )
1125 string_idx = string_indices[0]
1126 vresult = self._validate(line, string_idx)
1127 if isinstance(vresult, Err):
1128 return vresult
1130 return match_result
1132 def _validate(self, line: Line, string_idx: int) -> TResult[None]:
1133 """
1134 Checks that @line meets all of the requirements listed in this classes'
1135 docstring. Refer to `help(BaseStringSplitter)` for a detailed
1136 description of those requirements.
1138 Returns:
1139 * Ok(None), if ALL of the requirements are met.
1140 OR
1141 * Err(CannotTransform), if ANY of the requirements are NOT met.
1142 """
1143 LL = line.leaves
1145 string_leaf = LL[string_idx]
1147 max_string_length = self._get_max_string_length(line, string_idx)
1148 if len(string_leaf.value) <= max_string_length:
1149 return TErr(
1150 "The string itself is not what is causing this line to be too long."
1151 )
1153 if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
1154 token.STRING,
1155 token.NEWLINE,
1156 ]:
1157 return TErr(
1158 f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
1159 " no parent)."
1160 )
1162 if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
1163 line.comments[id(line.leaves[string_idx])]
1164 ):
1165 return TErr(
1166 "Line appears to end with an inline pragma comment. Splitting the line"
1167 " could modify the pragma's behavior."
1168 )
1170 if has_triple_quotes(string_leaf.value):
1171 return TErr("We cannot split multiline strings.")
1173 return Ok(None)
1175 def _get_max_string_length(self, line: Line, string_idx: int) -> int:
1176 """
1177 Calculates the max string length used when attempting to determine
1178 whether or not the target string is responsible for causing the line to
1179 go over the line length limit.
1181 WARNING: This method is tightly coupled to both StringSplitter and
1182 (especially) StringParenWrapper. There is probably a better way to
1183 accomplish what is being done here.
1185 Returns:
1186 max_string_length: such that `line.leaves[string_idx].value >
1187 max_string_length` implies that the target string IS responsible
1188 for causing this line to exceed the line length limit.
1189 """
1190 LL = line.leaves
1192 is_valid_index = is_valid_index_factory(LL)
1194 # We use the shorthand "WMA4" in comments to abbreviate "We must
1195 # account for". When giving examples, we use STRING to mean some/any
1196 # valid string.
1197 #
1198 # Finally, we use the following convenience variables:
1199 #
1200 # P: The leaf that is before the target string leaf.
1201 # N: The leaf that is after the target string leaf.
1202 # NN: The leaf that is after N.
1204 # WMA4 the whitespace at the beginning of the line.
1205 offset = line.depth * 4
1207 if is_valid_index(string_idx - 1):
1208 p_idx = string_idx - 1
1209 if (
1210 LL[string_idx - 1].type == token.LPAR
1211 and LL[string_idx - 1].value == ""
1212 and string_idx >= 2
1213 ):
1214 # If the previous leaf is an empty LPAR placeholder, we should skip it.
1215 p_idx -= 1
1217 P = LL[p_idx]
1218 if P.type in self.STRING_OPERATORS:
1219 # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
1220 offset += len(str(P)) + 1
1222 if P.type == token.COMMA:
1223 # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
1224 offset += 3
1226 if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
1227 # This conditional branch is meant to handle dictionary keys,
1228 # variable assignments, 'return STRING' statement lines, and
1229 # 'else STRING' ternary expression lines.
1231 # WMA4 a single space.
1232 offset += 1
1234 # WMA4 the lengths of any leaves that came before that space,
1235 # but after any closing bracket before that space.
1236 for leaf in reversed(LL[: p_idx + 1]):
1237 offset += len(str(leaf))
1238 if leaf.type in CLOSING_BRACKETS:
1239 break
1241 if is_valid_index(string_idx + 1):
1242 N = LL[string_idx + 1]
1243 if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
1244 # If the next leaf is an empty RPAR placeholder, we should skip it.
1245 N = LL[string_idx + 2]
1247 if N.type == token.COMMA:
1248 # WMA4 a single comma at the end of the string (e.g `STRING,`).
1249 offset += 1
1251 if is_valid_index(string_idx + 2):
1252 NN = LL[string_idx + 2]
1254 if N.type == token.DOT and NN.type == token.NAME:
1255 # This conditional branch is meant to handle method calls invoked
1256 # off of a string literal up to and including the LPAR character.
1258 # WMA4 the '.' character.
1259 offset += 1
1261 if (
1262 is_valid_index(string_idx + 3)
1263 and LL[string_idx + 3].type == token.LPAR
1264 ):
1265 # WMA4 the left parenthesis character.
1266 offset += 1
1268 # WMA4 the length of the method's name.
1269 offset += len(NN.value)
1271 has_comments = False
1272 for comment_leaf in line.comments_after(LL[string_idx]):
1273 if not has_comments:
1274 has_comments = True
1275 # WMA4 two spaces before the '#' character.
1276 offset += 2
1278 # WMA4 the length of the inline comment.
1279 offset += len(comment_leaf.value)
1281 max_string_length = count_chars_in_width(str(line), self.line_length - offset)
1282 return max_string_length
1284 @staticmethod
1285 def _prefer_paren_wrap_match(LL: list[Leaf]) -> int | None:
1286 """
1287 Returns:
1288 string_idx such that @LL[string_idx] is equal to our target (i.e.
1289 matched) string, if this line matches the "prefer paren wrap" statement
1290 requirements listed in the 'Requirements' section of the StringParenWrapper
1291 class's docstring.
1292 OR
1293 None, otherwise.
1294 """
1295 # The line must start with a string.
1296 if LL[0].type != token.STRING:
1297 return None
1299 matching_nodes = [
1300 syms.listmaker,
1301 syms.dictsetmaker,
1302 syms.testlist_gexp,
1303 ]
1304 # If the string is an immediate child of a list/set/tuple literal...
1305 if (
1306 parent_type(LL[0]) in matching_nodes
1307 or parent_type(LL[0].parent) in matching_nodes
1308 ):
1309 # And the string is surrounded by commas (or is the first/last child)...
1310 prev_sibling = LL[0].prev_sibling
1311 next_sibling = LL[0].next_sibling
1312 if (
1313 not prev_sibling
1314 and not next_sibling
1315 and parent_type(LL[0]) == syms.atom
1316 ):
1317 # If it's an atom string, we need to check the parent atom's siblings.
1318 parent = LL[0].parent
1319 assert parent is not None # For type checkers.
1320 prev_sibling = parent.prev_sibling
1321 next_sibling = parent.next_sibling
1322 if (not prev_sibling or prev_sibling.type == token.COMMA) and (
1323 not next_sibling or next_sibling.type == token.COMMA
1324 ):
1325 return 0
1327 return None
1330def iter_fexpr_spans(s: str) -> Iterator[tuple[int, int]]:
1331 """
1332 Yields spans corresponding to expressions in a given f-string.
1333 Spans are half-open ranges (left inclusive, right exclusive).
1334 Assumes the input string is a valid f-string, but will not crash if the input
1335 string is invalid.
1336 """
1337 stack: list[int] = [] # our curly paren stack
1338 i = 0
1339 while i < len(s):
1340 if s[i] == "{":
1341 # if we're in a string part of the f-string, ignore escaped curly braces
1342 if not stack and i + 1 < len(s) and s[i + 1] == "{":
1343 i += 2
1344 continue
1345 stack.append(i)
1346 i += 1
1347 continue
1349 if s[i] == "}":
1350 if not stack:
1351 i += 1
1352 continue
1353 j = stack.pop()
1354 # we've made it back out of the expression! yield the span
1355 if not stack:
1356 yield (j, i + 1)
1357 i += 1
1358 continue
1360 # if we're in an expression part of the f-string, fast-forward through strings
1361 # note that backslashes are not legal in the expression portion of f-strings
1362 if stack:
1363 delim = None
1364 if s[i : i + 3] in ("'''", '"""'):
1365 delim = s[i : i + 3]
1366 elif s[i] in ("'", '"'):
1367 delim = s[i]
1368 if delim:
1369 i += len(delim)
1370 while i < len(s) and s[i : i + len(delim)] != delim:
1371 i += 1
1372 i += len(delim)
1373 continue
1374 i += 1
1377def fstring_contains_expr(s: str) -> bool:
1378 return any(iter_fexpr_spans(s))
1381def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:
1382 """
1383 Toggles quotes used in f-string expressions that are `old_quote`.
1385 f-string expressions can't contain backslashes, so we need to toggle the
1386 quotes if the f-string itself will end up using the same quote. We can
1387 simply toggle without escaping because, quotes can't be reused in f-string
1388 expressions. They will fail to parse.
1390 NOTE: If PEP 701 is accepted, above statement will no longer be true.
1391 Though if quotes can be reused, we can simply reuse them without updates or
1392 escaping, once Black figures out how to parse the new grammar.
1393 """
1394 new_quote = "'" if old_quote == '"' else '"'
1395 parts = []
1396 previous_index = 0
1397 for start, end in iter_fexpr_spans(fstring):
1398 parts.append(fstring[previous_index:start])
1399 parts.append(fstring[start:end].replace(old_quote, new_quote))
1400 previous_index = end
1401 parts.append(fstring[previous_index:])
1402 return "".join(parts)
1405class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
1406 """
1407 StringTransformer that splits "atom" strings (i.e. strings which exist on
1408 lines by themselves).
1410 Requirements:
1411 * The line consists ONLY of a single string (possibly prefixed by a
1412 string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE
1413 a trailing comma.
1414 AND
1415 * All of the requirements listed in BaseStringSplitter's docstring.
1417 Transformations:
1418 The string mentioned in the 'Requirements' section is split into as
1419 many substrings as necessary to adhere to the configured line length.
1421 In the final set of substrings, no substring should be smaller than
1422 MIN_SUBSTR_SIZE characters.
1424 The string will ONLY be split on spaces (i.e. each new substring should
1425 start with a space). Note that the string will NOT be split on a space
1426 which is escaped with a backslash.
1428 If the string is an f-string, it will NOT be split in the middle of an
1429 f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
1430 else bar()} is an f-expression).
1432 If the string that is being split has an associated set of custom split
1433 records and those custom splits will NOT result in any line going over
1434 the configured line length, those custom splits are used. Otherwise the
1435 string is split as late as possible (from left-to-right) while still
1436 adhering to the transformation rules listed above.
1438 Collaborations:
1439 StringSplitter relies on StringMerger to construct the appropriate
1440 CustomSplit objects and add them to the custom split map.
1441 """
1443 MIN_SUBSTR_SIZE: Final = 6
1445 def do_splitter_match(self, line: Line) -> TMatchResult:
1446 LL = line.leaves
1448 if self._prefer_paren_wrap_match(LL) is not None:
1449 return TErr("Line needs to be wrapped in parens first.")
1451 # If the line is just STRING + COMMA (a one-item tuple) and not inside
1452 # brackets, we need to defer to StringParenWrapper to wrap it first.
1453 # Otherwise, splitting the string would create multiple expressions where
1454 # only the last has the comma, breaking AST equivalence. See issue #4912.
1455 if (
1456 not line.inside_brackets
1457 and len(LL) == 2
1458 and LL[0].type == token.STRING
1459 and LL[1].type == token.COMMA
1460 ):
1461 return TErr(
1462 "Line with trailing comma tuple needs to be wrapped in parens first."
1463 )
1465 is_valid_index = is_valid_index_factory(LL)
1467 idx = 0
1469 # The first two leaves MAY be the 'not in' keywords...
1470 if (
1471 is_valid_index(idx)
1472 and is_valid_index(idx + 1)
1473 and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]
1474 and str(LL[idx]) + str(LL[idx + 1]) == "not in"
1475 ):
1476 idx += 2
1477 # Else the first leaf MAY be a string operator symbol or the 'in' keyword...
1478 elif is_valid_index(idx) and (
1479 LL[idx].type in self.STRING_OPERATORS
1480 or LL[idx].type == token.NAME
1481 and str(LL[idx]) == "in"
1482 ):
1483 idx += 1
1485 # The next/first leaf MAY be an empty LPAR...
1486 if is_valid_index(idx) and is_empty_lpar(LL[idx]):
1487 idx += 1
1489 # The next/first leaf MUST be a string...
1490 if not is_valid_index(idx) or LL[idx].type != token.STRING:
1491 return TErr("Line does not start with a string.")
1493 string_idx = idx
1495 # Skip the string trailer, if one exists.
1496 string_parser = StringParser()
1497 idx = string_parser.parse(LL, string_idx)
1499 # That string MAY be followed by an empty RPAR...
1500 if is_valid_index(idx) and is_empty_rpar(LL[idx]):
1501 idx += 1
1503 # That string / empty RPAR leaf MAY be followed by a comma...
1504 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1505 idx += 1
1507 # But no more leaves are allowed...
1508 if is_valid_index(idx):
1509 return TErr("This line does not end with a string.")
1511 return Ok([string_idx])
1513 def do_transform(
1514 self, line: Line, string_indices: list[int]
1515 ) -> Iterator[TResult[Line]]:
1516 LL = line.leaves
1517 assert len(string_indices) == 1, (
1518 f"{self.__class__.__name__} should only find one match at a time, found"
1519 f" {len(string_indices)}"
1520 )
1521 string_idx = string_indices[0]
1523 QUOTE = LL[string_idx].value[-1]
1525 is_valid_index = is_valid_index_factory(LL)
1526 insert_str_child = insert_str_child_factory(LL[string_idx])
1528 prefix = get_string_prefix(LL[string_idx].value).lower()
1530 # We MAY choose to drop the 'f' prefix from substrings that don't
1531 # contain any f-expressions, but ONLY if the original f-string
1532 # contains at least one f-expression. Otherwise, we will alter the AST
1533 # of the program.
1534 drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr(
1535 LL[string_idx].value
1536 )
1538 first_string_line = True
1540 string_op_leaves = self._get_string_operator_leaves(LL)
1541 string_op_leaves_length = (
1542 sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1
1543 if string_op_leaves
1544 else 0
1545 )
1547 def maybe_append_string_operators(new_line: Line) -> None:
1548 """
1549 Side Effects:
1550 If @line starts with a string operator and this is the first
1551 line we are constructing, this function appends the string
1552 operator to @new_line and replaces the old string operator leaf
1553 in the node structure. Otherwise this function does nothing.
1554 """
1555 maybe_prefix_leaves = string_op_leaves if first_string_line else []
1556 for i, prefix_leaf in enumerate(maybe_prefix_leaves):
1557 replace_child(LL[i], prefix_leaf)
1558 new_line.append(prefix_leaf)
1560 ends_with_comma = (
1561 is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
1562 )
1564 def max_last_string_column() -> int:
1565 """
1566 Returns:
1567 The max allowed width of the string value used for the last
1568 line we will construct. Note that this value means the width
1569 rather than the number of characters (e.g., many East Asian
1570 characters expand to two columns).
1571 """
1572 result = self.line_length
1573 result -= line.depth * 4
1574 result -= 1 if ends_with_comma else 0
1575 result -= string_op_leaves_length
1576 return result
1578 # --- Calculate Max Break Width (for string value)
1579 # We start with the line length limit
1580 max_break_width = self.line_length
1581 # The last index of a string of length N is N-1.
1582 max_break_width -= 1
1583 # Leading whitespace is not present in the string value (e.g. Leaf.value).
1584 max_break_width -= line.depth * 4
1585 if max_break_width < 0:
1586 yield TErr(
1587 f"Unable to split {LL[string_idx].value} at such high of a line depth:"
1588 f" {line.depth}"
1589 )
1590 return
1592 # Check if StringMerger registered any custom splits.
1593 custom_splits = self.pop_custom_splits(LL[string_idx].value)
1594 # We use them ONLY if none of them would produce lines that exceed the
1595 # line limit.
1596 use_custom_breakpoints = bool(
1597 custom_splits
1598 and all(csplit.break_idx <= max_break_width for csplit in custom_splits)
1599 )
1601 # Temporary storage for the remaining chunk of the string line that
1602 # can't fit onto the line currently being constructed.
1603 rest_value = LL[string_idx].value
1605 def more_splits_should_be_made() -> bool:
1606 """
1607 Returns:
1608 True iff `rest_value` (the remaining string value from the last
1609 split), should be split again.
1610 """
1611 if use_custom_breakpoints:
1612 return len(custom_splits) > 1
1613 else:
1614 return str_width(rest_value) > max_last_string_column()
1616 string_line_results: list[Ok[Line]] = []
1617 while more_splits_should_be_made():
1618 if use_custom_breakpoints:
1619 # Custom User Split (manual)
1620 csplit = custom_splits.pop(0)
1621 break_idx = csplit.break_idx
1622 else:
1623 # Algorithmic Split (automatic)
1624 max_bidx = (
1625 count_chars_in_width(rest_value, max_break_width)
1626 - string_op_leaves_length
1627 )
1628 maybe_break_idx = self._get_break_idx(rest_value, max_bidx)
1629 if maybe_break_idx is None:
1630 # If we are unable to algorithmically determine a good split
1631 # and this string has custom splits registered to it, we
1632 # fall back to using them--which means we have to start
1633 # over from the beginning.
1634 if custom_splits:
1635 rest_value = LL[string_idx].value
1636 string_line_results = []
1637 first_string_line = True
1638 use_custom_breakpoints = True
1639 continue
1641 # Otherwise, we stop splitting here.
1642 break
1644 break_idx = maybe_break_idx
1646 # --- Construct `next_value`
1647 next_value = rest_value[:break_idx] + QUOTE
1649 # HACK: The following 'if' statement is a hack to fix the custom
1650 # breakpoint index in the case of either: (a) substrings that were
1651 # f-strings but will have the 'f' prefix removed OR (b) substrings
1652 # that were not f-strings but will now become f-strings because of
1653 # redundant use of the 'f' prefix (i.e. none of the substrings
1654 # contain f-expressions but one or more of them had the 'f' prefix
1655 # anyway; in which case, we will prepend 'f' to _all_ substrings).
1656 #
1657 # There is probably a better way to accomplish what is being done
1658 # here...
1659 #
1660 # If this substring is an f-string, we _could_ remove the 'f'
1661 # prefix, and the current custom split did NOT originally use a
1662 # prefix...
1663 if (
1664 use_custom_breakpoints
1665 and not csplit.has_prefix
1666 and (
1667 # `next_value == prefix + QUOTE` happens when the custom
1668 # split is an empty string.
1669 next_value == prefix + QUOTE
1670 or next_value != self._normalize_f_string(next_value, prefix)
1671 )
1672 ):
1673 # Then `csplit.break_idx` will be off by one after removing
1674 # the 'f' prefix.
1675 break_idx += 1
1676 next_value = rest_value[:break_idx] + QUOTE
1678 if drop_pointless_f_prefix:
1679 next_value = self._normalize_f_string(next_value, prefix)
1681 # --- Construct `next_leaf`
1682 next_leaf = Leaf(token.STRING, next_value)
1683 insert_str_child(next_leaf)
1684 self._maybe_normalize_string_quotes(next_leaf)
1686 # --- Construct `next_line`
1687 next_line = line.clone()
1688 maybe_append_string_operators(next_line)
1689 next_line.append(next_leaf)
1690 string_line_results.append(Ok(next_line))
1692 rest_value = prefix + QUOTE + rest_value[break_idx:]
1693 first_string_line = False
1695 yield from string_line_results
1697 if drop_pointless_f_prefix:
1698 rest_value = self._normalize_f_string(rest_value, prefix)
1700 rest_leaf = Leaf(token.STRING, rest_value)
1701 insert_str_child(rest_leaf)
1703 # NOTE: I could not find a test case that verifies that the following
1704 # line is actually necessary, but it seems to be. Otherwise we risk
1705 # not normalizing the last substring, right?
1706 self._maybe_normalize_string_quotes(rest_leaf)
1708 last_line = line.clone()
1709 maybe_append_string_operators(last_line)
1711 # If there are any leaves to the right of the target string...
1712 if is_valid_index(string_idx + 1):
1713 # We use `temp_value` here to determine how long the last line
1714 # would be if we were to append all the leaves to the right of the
1715 # target string to the last string line.
1716 temp_value = rest_value
1717 for leaf in LL[string_idx + 1 :]:
1718 temp_value += str(leaf)
1719 if leaf.type == token.LPAR:
1720 break
1722 # Try to fit them all on the same line with the last substring...
1723 if (
1724 str_width(temp_value) <= max_last_string_column()
1725 or LL[string_idx + 1].type == token.COMMA
1726 ):
1727 last_line.append(rest_leaf)
1728 append_leaves(last_line, line, LL[string_idx + 1 :])
1729 yield Ok(last_line)
1730 # Otherwise, place the last substring on one line and everything
1731 # else on a line below that...
1732 else:
1733 last_line.append(rest_leaf)
1734 yield Ok(last_line)
1736 non_string_line = line.clone()
1737 append_leaves(non_string_line, line, LL[string_idx + 1 :])
1738 yield Ok(non_string_line)
1739 # Else the target string was the last leaf...
1740 else:
1741 last_line.append(rest_leaf)
1742 last_line.comments = line.comments.copy()
1743 yield Ok(last_line)
1745 def _iter_nameescape_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1746 r"""
1747 Yields:
1748 All ranges of @string which, if @string were to be split there,
1749 would result in the splitting of an \N{...} expression (which is NOT
1750 allowed).
1751 """
1752 # True - the previous backslash was unescaped
1753 # False - the previous backslash was escaped *or* there was no backslash
1754 previous_was_unescaped_backslash = False
1755 it = iter(enumerate(string))
1756 for idx, c in it:
1757 if c == "\\":
1758 previous_was_unescaped_backslash = not previous_was_unescaped_backslash
1759 continue
1760 if not previous_was_unescaped_backslash or c != "N":
1761 previous_was_unescaped_backslash = False
1762 continue
1763 previous_was_unescaped_backslash = False
1765 begin = idx - 1 # the position of backslash before \N{...}
1766 for idx, c in it:
1767 if c == "}":
1768 end = idx
1769 break
1770 else:
1771 # malformed nameescape expression?
1772 # should have been detected by AST parsing earlier...
1773 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
1774 yield begin, end
1776 def _iter_fexpr_slices(self, string: str) -> Iterator[tuple[Index, Index]]:
1777 """
1778 Yields:
1779 All ranges of @string which, if @string were to be split there,
1780 would result in the splitting of an f-expression (which is NOT
1781 allowed).
1782 """
1783 if "f" not in get_string_prefix(string).lower():
1784 return
1785 yield from iter_fexpr_spans(string)
1787 def _get_illegal_split_indices(self, string: str) -> set[Index]:
1788 illegal_indices: set[Index] = set()
1789 iterators = [
1790 self._iter_fexpr_slices(string),
1791 self._iter_nameescape_slices(string),
1792 ]
1793 for it in iterators:
1794 for begin, end in it:
1795 illegal_indices.update(range(begin, end))
1796 return illegal_indices
1798 def _get_break_idx(self, string: str, max_break_idx: int) -> int | None:
1799 """
1800 This method contains the algorithm that StringSplitter uses to
1801 determine which character to split each string at.
1803 Args:
1804 @string: The substring that we are attempting to split.
1805 @max_break_idx: The ideal break index. We will return this value if it
1806 meets all the necessary conditions. In the likely event that it
1807 doesn't we will try to find the closest index BELOW @max_break_idx
1808 that does. If that fails, we will expand our search by also
1809 considering all valid indices ABOVE @max_break_idx.
1811 Pre-Conditions:
1812 * assert_is_leaf_string(@string)
1813 * 0 <= @max_break_idx < len(@string)
1815 Returns:
1816 break_idx, if an index is able to be found that meets all of the
1817 conditions listed in the 'Transformations' section of this classes'
1818 docstring.
1819 OR
1820 None, otherwise.
1821 """
1822 is_valid_index = is_valid_index_factory(string)
1824 assert is_valid_index(max_break_idx)
1825 assert_is_leaf_string(string)
1827 _illegal_split_indices = self._get_illegal_split_indices(string)
1829 def breaks_unsplittable_expression(i: Index) -> bool:
1830 """
1831 Returns:
1832 True iff returning @i would result in the splitting of an
1833 unsplittable expression (which is NOT allowed).
1834 """
1835 return i in _illegal_split_indices
1837 def passes_all_checks(i: Index) -> bool:
1838 """
1839 Returns:
1840 True iff ALL of the conditions listed in the 'Transformations'
1841 section of this classes' docstring would be met by returning @i.
1842 """
1843 is_space = string[i] == " "
1844 is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS
1846 is_not_escaped = True
1847 j = i - 1
1848 while is_valid_index(j) and string[j] == "\\":
1849 is_not_escaped = not is_not_escaped
1850 j -= 1
1852 is_big_enough = (
1853 len(string[i:]) >= self.MIN_SUBSTR_SIZE
1854 and len(string[:i]) >= self.MIN_SUBSTR_SIZE
1855 )
1856 return (
1857 (is_space or is_split_safe)
1858 and is_not_escaped
1859 and is_big_enough
1860 and not breaks_unsplittable_expression(i)
1861 )
1863 # First, we check all indices BELOW @max_break_idx.
1864 break_idx = max_break_idx
1865 while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
1866 break_idx -= 1
1868 if not passes_all_checks(break_idx):
1869 # If that fails, we check all indices ABOVE @max_break_idx.
1870 #
1871 # If we are able to find a valid index here, the next line is going
1872 # to be longer than the specified line length, but it's probably
1873 # better than doing nothing at all.
1874 break_idx = max_break_idx + 1
1875 while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
1876 break_idx += 1
1878 if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
1879 return None
1881 return break_idx
1883 def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
1884 if self.normalize_strings:
1885 leaf.value = normalize_string_quotes(leaf.value)
1887 def _normalize_f_string(self, string: str, prefix: str) -> str:
1888 """
1889 Pre-Conditions:
1890 * assert_is_leaf_string(@string)
1892 Returns:
1893 * If @string is an f-string that contains no f-expressions, we
1894 return a string identical to @string except that the 'f' prefix
1895 has been stripped and all double braces (i.e. '{{' or '}}') have
1896 been normalized (i.e. turned into '{' or '}').
1897 OR
1898 * Otherwise, we return @string.
1899 """
1900 assert_is_leaf_string(string)
1902 if "f" in prefix and not fstring_contains_expr(string):
1903 new_prefix = prefix.replace("f", "")
1905 temp = string[len(prefix) :]
1906 temp = re.sub(r"\{\{", "{", temp)
1907 temp = re.sub(r"\}\}", "}", temp)
1908 new_string = temp
1910 return f"{new_prefix}{new_string}"
1911 else:
1912 return string
1914 def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> list[Leaf]:
1915 LL = list(leaves)
1917 string_op_leaves = []
1918 i = 0
1919 while LL[i].type in self.STRING_OPERATORS + [token.NAME]:
1920 prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())
1921 string_op_leaves.append(prefix_leaf)
1922 i += 1
1923 return string_op_leaves
1926class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):
1927 """
1928 StringTransformer that wraps strings in parens and then splits at the LPAR.
1930 Requirements:
1931 All of the requirements listed in BaseStringSplitter's docstring in
1932 addition to the requirements listed below:
1934 * The line is a return/yield statement, which returns/yields a string.
1935 OR
1936 * The line is part of a ternary expression (e.g. `x = y if cond else
1937 z`) such that the line starts with `else <string>`, where <string> is
1938 some string.
1939 OR
1940 * The line is an assert statement, which ends with a string.
1941 OR
1942 * The line is an assignment statement (e.g. `x = <string>` or `x +=
1943 <string>`) such that the variable is being assigned the value of some
1944 string.
1945 OR
1946 * The line is a dictionary key assignment where some valid key is being
1947 assigned the value of some string.
1948 OR
1949 * The line is an lambda expression and the value is a string.
1950 OR
1951 * The line starts with an "atom" string that prefers to be wrapped in
1952 parens. It's preferred to be wrapped when it's is an immediate child of
1953 a list/set/tuple literal, AND the string is surrounded by commas (or is
1954 the first/last child).
1956 Transformations:
1957 The chosen string is wrapped in parentheses and then split at the LPAR.
1959 We then have one line which ends with an LPAR and another line that
1960 starts with the chosen string. The latter line is then split again at
1961 the RPAR. This results in the RPAR (and possibly a trailing comma)
1962 being placed on its own line.
1964 NOTE: If any leaves exist to the right of the chosen string (except
1965 for a trailing comma, which would be placed after the RPAR), those
1966 leaves are placed inside the parentheses. In effect, the chosen
1967 string is not necessarily being "wrapped" by parentheses. We can,
1968 however, count on the LPAR being placed directly before the chosen
1969 string.
1971 In other words, StringParenWrapper creates "atom" strings. These
1972 can then be split again by StringSplitter, if necessary.
1974 Collaborations:
1975 In the event that a string line split by StringParenWrapper is
1976 changed such that it no longer needs to be given its own line,
1977 StringParenWrapper relies on StringParenStripper to clean up the
1978 parentheses it created.
1980 For "atom" strings that prefers to be wrapped in parens, it requires
1981 StringSplitter to hold the split until the string is wrapped in parens.
1982 """
1984 def do_splitter_match(self, line: Line) -> TMatchResult:
1985 LL = line.leaves
1987 if line.leaves[-1].type in OPENING_BRACKETS:
1988 return TErr(
1989 "Cannot wrap parens around a line that ends in an opening bracket."
1990 )
1992 string_idx = (
1993 self._return_match(LL)
1994 or self._else_match(LL)
1995 or self._assert_match(LL)
1996 or self._assign_match(LL)
1997 or self._dict_or_lambda_match(LL)
1998 )
2000 if string_idx is None:
2001 string_idx = self._trailing_comma_tuple_match(line)
2003 if string_idx is None:
2004 string_idx = self._prefer_paren_wrap_match(LL)
2006 if string_idx is not None:
2007 string_value = line.leaves[string_idx].value
2008 # If the string has neither spaces nor East Asian stops...
2009 if not any(
2010 char == " " or char in SPLIT_SAFE_CHARS for char in string_value
2011 ):
2012 # And will still violate the line length limit when split...
2013 max_string_width = self.line_length - ((line.depth + 1) * 4)
2014 if str_width(string_value) > max_string_width:
2015 # And has no associated custom splits...
2016 if not self.has_custom_splits(string_value):
2017 # Then we should NOT put this string on its own line.
2018 return TErr(
2019 "We do not wrap long strings in parentheses when the"
2020 " resultant line would still be over the specified line"
2021 " length and can't be split further by StringSplitter."
2022 )
2023 return Ok([string_idx])
2025 return TErr("This line does not contain any non-atomic strings.")
2027 @staticmethod
2028 def _return_match(LL: list[Leaf]) -> int | None:
2029 """
2030 Returns:
2031 string_idx such that @LL[string_idx] is equal to our target (i.e.
2032 matched) string, if this line matches the return/yield statement
2033 requirements listed in the 'Requirements' section of this classes'
2034 docstring.
2035 OR
2036 None, otherwise.
2037 """
2038 # If this line is a part of a return/yield statement and the first leaf
2039 # contains either the "return" or "yield" keywords...
2040 if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
2041 0
2042 ].value in ["return", "yield"]:
2043 is_valid_index = is_valid_index_factory(LL)
2045 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2046 # The next visible leaf MUST contain a string...
2047 if is_valid_index(idx) and LL[idx].type == token.STRING:
2048 return idx
2050 return None
2052 @staticmethod
2053 def _else_match(LL: list[Leaf]) -> int | None:
2054 """
2055 Returns:
2056 string_idx such that @LL[string_idx] is equal to our target (i.e.
2057 matched) string, if this line matches the ternary expression
2058 requirements listed in the 'Requirements' section of this classes'
2059 docstring.
2060 OR
2061 None, otherwise.
2062 """
2063 # If this line is a part of a ternary expression and the first leaf
2064 # contains the "else" keyword...
2065 if (
2066 parent_type(LL[0]) == syms.test
2067 and LL[0].type == token.NAME
2068 and LL[0].value == "else"
2069 ):
2070 is_valid_index = is_valid_index_factory(LL)
2072 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
2073 # The next visible leaf MUST contain a string...
2074 if is_valid_index(idx) and LL[idx].type == token.STRING:
2075 return idx
2077 return None
2079 @staticmethod
2080 def _assert_match(LL: list[Leaf]) -> int | None:
2081 """
2082 Returns:
2083 string_idx such that @LL[string_idx] is equal to our target (i.e.
2084 matched) string, if this line matches the assert statement
2085 requirements listed in the 'Requirements' section of this classes'
2086 docstring.
2087 OR
2088 None, otherwise.
2089 """
2090 # If this line is a part of an assert statement and the first leaf
2091 # contains the "assert" keyword...
2092 if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
2093 is_valid_index = is_valid_index_factory(LL)
2095 for i, leaf in enumerate(LL):
2096 # We MUST find a comma...
2097 if leaf.type == token.COMMA:
2098 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2100 # That comma MUST be followed by a string...
2101 if is_valid_index(idx) and LL[idx].type == token.STRING:
2102 string_idx = idx
2104 # Skip the string trailer, if one exists.
2105 string_parser = StringParser()
2106 idx = string_parser.parse(LL, string_idx)
2108 # But no more leaves are allowed...
2109 if not is_valid_index(idx):
2110 return string_idx
2112 return None
2114 @staticmethod
2115 def _assign_match(LL: list[Leaf]) -> int | None:
2116 """
2117 Returns:
2118 string_idx such that @LL[string_idx] is equal to our target (i.e.
2119 matched) string, if this line matches the assignment statement
2120 requirements listed in the 'Requirements' section of this classes'
2121 docstring.
2122 OR
2123 None, otherwise.
2124 """
2125 # If this line is a part of an expression statement or is a function
2126 # argument AND the first leaf contains a variable name...
2127 if (
2128 parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
2129 and LL[0].type == token.NAME
2130 ):
2131 is_valid_index = is_valid_index_factory(LL)
2133 for i, leaf in enumerate(LL):
2134 # We MUST find either an '=' or '+=' symbol...
2135 if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
2136 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2138 # That symbol MUST be followed by a string...
2139 if is_valid_index(idx) and LL[idx].type == token.STRING:
2140 string_idx = idx
2142 # Skip the string trailer, if one exists.
2143 string_parser = StringParser()
2144 idx = string_parser.parse(LL, string_idx)
2146 # The next leaf MAY be a comma iff this line is a part
2147 # of a function argument...
2148 if (
2149 parent_type(LL[0]) == syms.argument
2150 and is_valid_index(idx)
2151 and LL[idx].type == token.COMMA
2152 ):
2153 idx += 1
2155 # But no more leaves are allowed...
2156 if not is_valid_index(idx):
2157 return string_idx
2159 return None
2161 @staticmethod
2162 def _dict_or_lambda_match(LL: list[Leaf]) -> int | None:
2163 """
2164 Returns:
2165 string_idx such that @LL[string_idx] is equal to our target (i.e.
2166 matched) string, if this line matches the dictionary key assignment
2167 statement or lambda expression requirements listed in the
2168 'Requirements' section of this classes' docstring.
2169 OR
2170 None, otherwise.
2171 """
2172 # If this line is a part of a dictionary key assignment or lambda expression...
2173 parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)]
2174 if syms.dictsetmaker in parent_types or syms.lambdef in parent_types:
2175 is_valid_index = is_valid_index_factory(LL)
2177 for i, leaf in enumerate(LL):
2178 # We MUST find a colon, it can either be dict's or lambda's colon...
2179 if leaf.type == token.COLON and i < len(LL) - 1:
2180 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2182 # That colon MUST be followed by a string...
2183 if is_valid_index(idx) and LL[idx].type == token.STRING:
2184 string_idx = idx
2186 # Skip the string trailer, if one exists.
2187 string_parser = StringParser()
2188 idx = string_parser.parse(LL, string_idx)
2190 # That string MAY be followed by a comma...
2191 if is_valid_index(idx) and LL[idx].type == token.COMMA:
2192 idx += 1
2194 # But no more leaves are allowed...
2195 if not is_valid_index(idx):
2196 return string_idx
2198 return None
2200 @staticmethod
2201 def _trailing_comma_tuple_match(line: Line) -> int | None:
2202 """
2203 Returns:
2204 string_idx such that @line.leaves[string_idx] is equal to our target
2205 (i.e. matched) string, if the line is a bare trailing comma tuple
2206 (STRING + COMMA) not inside brackets.
2207 OR
2208 None, otherwise.
2210 This handles the case from issue #4912 where a long string with a
2211 trailing comma (making it a one-item tuple) needs to be wrapped in
2212 parentheses before splitting to preserve AST equivalence.
2213 """
2214 LL = line.leaves
2215 # Match: STRING followed by COMMA, not inside brackets
2216 if (
2217 not line.inside_brackets
2218 and len(LL) == 2
2219 and LL[0].type == token.STRING
2220 and LL[1].type == token.COMMA
2221 ):
2222 return 0
2224 return None
2226 def do_transform(
2227 self, line: Line, string_indices: list[int]
2228 ) -> Iterator[TResult[Line]]:
2229 LL = line.leaves
2230 assert len(string_indices) == 1, (
2231 f"{self.__class__.__name__} should only find one match at a time, found"
2232 f" {len(string_indices)}"
2233 )
2234 string_idx = string_indices[0]
2236 is_valid_index = is_valid_index_factory(LL)
2237 insert_str_child = insert_str_child_factory(LL[string_idx])
2239 comma_idx = -1
2240 ends_with_comma = False
2241 if LL[comma_idx].type == token.COMMA:
2242 ends_with_comma = True
2244 leaves_to_steal_comments_from = [LL[string_idx]]
2245 if ends_with_comma:
2246 leaves_to_steal_comments_from.append(LL[comma_idx])
2248 # --- First Line
2249 first_line = line.clone()
2250 left_leaves = LL[:string_idx]
2252 # We have to remember to account for (possibly invisible) LPAR and RPAR
2253 # leaves that already wrapped the target string. If these leaves do
2254 # exist, we will replace them with our own LPAR and RPAR leaves.
2255 old_parens_exist = False
2256 if left_leaves and left_leaves[-1].type == token.LPAR:
2257 old_parens_exist = True
2258 leaves_to_steal_comments_from.append(left_leaves[-1])
2259 left_leaves.pop()
2261 append_leaves(first_line, line, left_leaves)
2263 lpar_leaf = Leaf(token.LPAR, "(")
2264 if old_parens_exist:
2265 replace_child(LL[string_idx - 1], lpar_leaf)
2266 else:
2267 insert_str_child(lpar_leaf)
2268 first_line.append(lpar_leaf)
2270 # We throw inline comments that were originally to the right of the
2271 # target string to the top line. They will now be shown to the right of
2272 # the LPAR.
2273 for leaf in leaves_to_steal_comments_from:
2274 for comment_leaf in line.comments_after(leaf):
2275 first_line.append(comment_leaf, preformatted=True)
2277 yield Ok(first_line)
2279 # --- Middle (String) Line
2280 # We only need to yield one (possibly too long) string line, since the
2281 # `StringSplitter` will break it down further if necessary.
2282 string_value = LL[string_idx].value
2283 string_line = Line(
2284 mode=line.mode,
2285 depth=line.depth + 1,
2286 inside_brackets=True,
2287 should_split_rhs=line.should_split_rhs,
2288 magic_trailing_comma=line.magic_trailing_comma,
2289 )
2290 string_leaf = Leaf(token.STRING, string_value)
2291 insert_str_child(string_leaf)
2292 string_line.append(string_leaf)
2294 old_rpar_leaf = None
2295 if is_valid_index(string_idx + 1):
2296 right_leaves = LL[string_idx + 1 :]
2297 if ends_with_comma:
2298 right_leaves.pop()
2300 if old_parens_exist:
2301 assert right_leaves and right_leaves[-1].type == token.RPAR, (
2302 "Apparently, old parentheses do NOT exist?!"
2303 f" (left_leaves={left_leaves}, right_leaves={right_leaves})"
2304 )
2305 old_rpar_leaf = right_leaves.pop()
2306 elif right_leaves and right_leaves[-1].type == token.RPAR:
2307 # Special case for lambda expressions as dict's value, e.g.:
2308 # my_dict = {
2309 # "key": lambda x: f"formatted: {x}",
2310 # }
2311 # After wrapping the dict's value with parentheses, the string is
2312 # followed by a RPAR but its opening bracket is lambda's, not
2313 # the string's:
2314 # "key": (lambda x: f"formatted: {x}"),
2315 opening_bracket = right_leaves[-1].opening_bracket
2316 if opening_bracket is not None and opening_bracket in left_leaves:
2317 index = left_leaves.index(opening_bracket)
2318 if (
2319 0 < index < len(left_leaves) - 1
2320 and left_leaves[index - 1].type == token.COLON
2321 and left_leaves[index + 1].value == "lambda"
2322 ):
2323 right_leaves.pop()
2325 append_leaves(string_line, line, right_leaves)
2327 yield Ok(string_line)
2329 # --- Last Line
2330 last_line = line.clone()
2331 last_line.bracket_tracker = first_line.bracket_tracker
2333 new_rpar_leaf = Leaf(token.RPAR, ")")
2334 if old_rpar_leaf is not None:
2335 replace_child(old_rpar_leaf, new_rpar_leaf)
2336 else:
2337 insert_str_child(new_rpar_leaf)
2338 last_line.append(new_rpar_leaf)
2340 # If the target string ended with a comma, we place this comma to the
2341 # right of the RPAR on the last line.
2342 if ends_with_comma:
2343 comma_leaf = Leaf(token.COMMA, ",")
2344 replace_child(LL[comma_idx], comma_leaf)
2345 last_line.append(comma_leaf)
2347 yield Ok(last_line)
2350class StringParser:
2351 """
2352 A state machine that aids in parsing a string's "trailer", which can be
2353 either non-existent, an old-style formatting sequence (e.g. `% varX` or `%
2354 (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
2355 varY)`).
2357 NOTE: A new StringParser object MUST be instantiated for each string
2358 trailer we need to parse.
2360 Examples:
2361 We shall assume that `line` equals the `Line` object that corresponds
2362 to the following line of python code:
2363 ```
2364 x = "Some {}.".format("String") + some_other_string
2365 ```
2367 Furthermore, we will assume that `string_idx` is some index such that:
2368 ```
2369 assert line.leaves[string_idx].value == "Some {}."
2370 ```
2372 The following code snippet then holds:
2373 ```
2374 string_parser = StringParser()
2375 idx = string_parser.parse(line.leaves, string_idx)
2376 assert line.leaves[idx].type == token.PLUS
2377 ```
2378 """
2380 DEFAULT_TOKEN: Final = 20210605
2382 # String Parser States
2383 START: Final = 1
2384 DOT: Final = 2
2385 NAME: Final = 3
2386 PERCENT: Final = 4
2387 SINGLE_FMT_ARG: Final = 5
2388 LPAR: Final = 6
2389 RPAR: Final = 7
2390 DONE: Final = 8
2392 # Lookup Table for Next State
2393 _goto: Final[dict[tuple[ParserState, NodeType], ParserState]] = {
2394 # A string trailer may start with '.' OR '%'.
2395 (START, token.DOT): DOT,
2396 (START, token.PERCENT): PERCENT,
2397 (START, DEFAULT_TOKEN): DONE,
2398 # A '.' MUST be followed by an attribute or method name.
2399 (DOT, token.NAME): NAME,
2400 # A method name MUST be followed by an '(', whereas an attribute name
2401 # is the last symbol in the string trailer.
2402 (NAME, token.LPAR): LPAR,
2403 (NAME, DEFAULT_TOKEN): DONE,
2404 # A '%' symbol can be followed by an '(' or a single argument (e.g. a
2405 # string or variable name).
2406 (PERCENT, token.LPAR): LPAR,
2407 (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
2408 # If a '%' symbol is followed by a single argument, that argument is
2409 # the last leaf in the string trailer.
2410 (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
2411 # If present, a ')' symbol is the last symbol in a string trailer.
2412 # (NOTE: LPARS and nested RPARS are not included in this lookup table,
2413 # since they are treated as a special case by the parsing logic in this
2414 # classes' implementation.)
2415 (RPAR, DEFAULT_TOKEN): DONE,
2416 }
2418 def __init__(self) -> None:
2419 self._state = self.START
2420 self._unmatched_lpars = 0
2422 def parse(self, leaves: list[Leaf], string_idx: int) -> int:
2423 """
2424 Pre-conditions:
2425 * @leaves[@string_idx].type == token.STRING
2427 Returns:
2428 The index directly after the last leaf which is a part of the string
2429 trailer, if a "trailer" exists.
2430 OR
2431 @string_idx + 1, if no string "trailer" exists.
2432 """
2433 assert leaves[string_idx].type == token.STRING
2435 idx = string_idx + 1
2436 while idx < len(leaves) and self._next_state(leaves[idx]):
2437 idx += 1
2438 return idx
2440 def _next_state(self, leaf: Leaf) -> bool:
2441 """
2442 Pre-conditions:
2443 * On the first call to this function, @leaf MUST be the leaf that
2444 was directly after the string leaf in question (e.g. if our target
2445 string is `line.leaves[i]` then the first call to this method must
2446 be `line.leaves[i + 1]`).
2447 * On the next call to this function, the leaf parameter passed in
2448 MUST be the leaf directly following @leaf.
2450 Returns:
2451 True iff @leaf is a part of the string's trailer.
2452 """
2453 # We ignore empty LPAR or RPAR leaves.
2454 if is_empty_par(leaf):
2455 return True
2457 next_token = leaf.type
2458 if next_token == token.LPAR:
2459 self._unmatched_lpars += 1
2461 current_state = self._state
2463 # The LPAR parser state is a special case. We will return True until we
2464 # find the matching RPAR token.
2465 if current_state == self.LPAR:
2466 if next_token == token.RPAR:
2467 self._unmatched_lpars -= 1
2468 if self._unmatched_lpars == 0:
2469 self._state = self.RPAR
2470 # Otherwise, we use a lookup table to determine the next state.
2471 else:
2472 # If the lookup table matches the current state to the next
2473 # token, we use the lookup table.
2474 if (current_state, next_token) in self._goto:
2475 self._state = self._goto[current_state, next_token]
2476 else:
2477 # Otherwise, we check if a the current state was assigned a
2478 # default.
2479 if (current_state, self.DEFAULT_TOKEN) in self._goto:
2480 self._state = self._goto[current_state, self.DEFAULT_TOKEN]
2481 # If no default has been assigned, then this parser has a logic
2482 # error.
2483 else:
2484 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
2486 if self._state == self.DONE:
2487 return False
2489 return True
2492def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
2493 """
2494 Factory for a convenience function that is used to orphan @string_leaf
2495 and then insert multiple new leaves into the same part of the node
2496 structure that @string_leaf had originally occupied.
2498 Examples:
2499 Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
2500 string_leaf.parent`. Assume the node `N` has the following
2501 original structure:
2503 Node(
2504 expr_stmt, [
2505 Leaf(NAME, 'x'),
2506 Leaf(EQUAL, '='),
2507 Leaf(STRING, '"foo"'),
2508 ]
2509 )
2511 We then run the code snippet shown below.
2512 ```
2513 insert_str_child = insert_str_child_factory(string_leaf)
2515 lpar = Leaf(token.LPAR, '(')
2516 insert_str_child(lpar)
2518 bar = Leaf(token.STRING, '"bar"')
2519 insert_str_child(bar)
2521 rpar = Leaf(token.RPAR, ')')
2522 insert_str_child(rpar)
2523 ```
2525 After which point, it follows that `string_leaf.parent is None` and
2526 the node `N` now has the following structure:
2528 Node(
2529 expr_stmt, [
2530 Leaf(NAME, 'x'),
2531 Leaf(EQUAL, '='),
2532 Leaf(LPAR, '('),
2533 Leaf(STRING, '"bar"'),
2534 Leaf(RPAR, ')'),
2535 ]
2536 )
2537 """
2538 string_parent = string_leaf.parent
2539 string_child_idx = string_leaf.remove()
2541 def insert_str_child(child: LN) -> None:
2542 nonlocal string_child_idx
2544 assert string_parent is not None
2545 assert string_child_idx is not None
2547 string_parent.insert_child(string_child_idx, child)
2548 string_child_idx += 1
2550 return insert_str_child
2553def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
2554 """
2555 Examples:
2556 ```
2557 my_list = [1, 2, 3]
2559 is_valid_index = is_valid_index_factory(my_list)
2561 assert is_valid_index(0)
2562 assert is_valid_index(2)
2564 assert not is_valid_index(3)
2565 assert not is_valid_index(-1)
2566 ```
2567 """
2569 def is_valid_index(idx: int) -> bool:
2570 """
2571 Returns:
2572 True iff @idx is positive AND seq[@idx] does NOT raise an
2573 IndexError.
2574 """
2575 return 0 <= idx < len(seq)
2577 return is_valid_index