Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/strings.py: 14%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Simple formatting on strings. Further string formatting code is in trans.py.
3"""
5import re
6import sys
7from functools import lru_cache
8from re import Match, Pattern
9from typing import Final
11from black._width_table import WIDTH_TABLE
12from black.mode import Mode, Preview
13from blib2to3.pgen2 import token
14from blib2to3.pytree import Leaf, Node
16STRING_PREFIX_CHARS: Final = "fturbFTURB" # All possible string prefix characters.
17STRING_PREFIX_RE: Final = re.compile(
18 r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL
19)
20UNICODE_ESCAPE_RE: Final = re.compile(
21 r"(?P<backslashes>\\+)(?P<body>"
22 r"(u(?P<u>[a-fA-F0-9]{4}))" # Character with 16-bit hex value xxxx
23 r"|(U(?P<U>[a-fA-F0-9]{8}))" # Character with 32-bit hex value xxxxxxxx
24 r"|(x(?P<x>[a-fA-F0-9]{2}))" # Character with hex value hh
25 r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})" # Character named name in the Unicode database
26 r")?",
27 re.VERBOSE,
28)
29# The line breaks the Python parser recognizes, and therefore the only ones that
30# may be treated as line endings inside a string. str.splitlines() also breaks
31# on form feed, vertical tab, NEL, the Unicode line and paragraph separators and
32# the C0 separators, all of which are ordinary characters of a string's value.
33# See output._splitlines_no_ff, which splits source code the same way.
34LINE_BREAK_RE: Final = re.compile(r"\r\n|[\r\n]")
37def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
38 """Replace `regex` with `replacement` twice on `original`.
40 This is used by string normalization to perform replaces on
41 overlapping matches.
42 """
43 return regex.sub(replacement, regex.sub(replacement, original))
46def has_triple_quotes(string: str) -> bool:
47 """
48 Returns:
49 True iff @string starts with three quotation characters.
50 """
51 raw_string = string.lstrip(STRING_PREFIX_CHARS)
52 return raw_string[:3] in {'"""', "'''"}
55def lines_with_leading_tabs_expanded(s: str) -> list[str]:
56 """
57 Splits string into lines and expands only leading tabs.
59 Black normalizes code indentation to four-space columns, so leading tabs in
60 docstrings need the same width to keep relative indentation stable.
61 """
62 lines = []
63 for line in LINE_BREAK_RE.split(s):
64 stripped_line = line.lstrip()
65 if not stripped_line or stripped_line == line:
66 lines.append(line)
67 else:
68 prefix_length = len(line) - len(stripped_line)
69 prefix = line[:prefix_length].expandtabs(4)
70 lines.append(prefix + stripped_line)
71 return lines
74def fix_multiline_docstring(docstring: str, prefix: str) -> str:
75 # https://peps.python.org/pep-0257/#handling-docstring-indentation
76 assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty"
77 lines = lines_with_leading_tabs_expanded(docstring)
78 # Determine minimum indentation (first line doesn't count):
79 indent = sys.maxsize
80 for line in lines[1:]:
81 stripped = line.lstrip()
82 if stripped:
83 indent = min(indent, len(line) - len(stripped))
84 # Remove indentation (first line is special):
85 trimmed = [lines[0].strip()]
86 if indent < sys.maxsize:
87 last_line_idx = len(lines) - 2
88 for i, line in enumerate(lines[1:]):
89 stripped_line = line[indent:].rstrip()
90 if stripped_line or i == last_line_idx:
91 trimmed.append(prefix + stripped_line)
92 else:
93 trimmed.append("")
94 return "\n".join(trimmed)
97def get_string_prefix(string: str) -> str:
98 """
99 Pre-conditions:
100 * assert_is_leaf_string(@string)
102 Returns:
103 @string's prefix (e.g. '', 'r', 'f', or 'rf').
104 """
105 assert_is_leaf_string(string)
107 prefix = []
108 for char in string:
109 if char in STRING_PREFIX_CHARS:
110 prefix.append(char)
111 else:
112 break
113 return "".join(prefix)
116def assert_is_leaf_string(string: str) -> None:
117 """
118 Checks the pre-condition that @string has the format that you would expect
119 of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
120 token.STRING`. A more precise description of the pre-conditions that are
121 checked are listed below.
123 Pre-conditions:
124 * @string starts with either ', ", <prefix>', or <prefix>" where
125 `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
126 * @string ends with a quote character (' or ").
128 Raises:
129 AssertionError(...) if the pre-conditions listed above are not
130 satisfied.
131 """
132 dquote_idx = string.find('"')
133 squote_idx = string.find("'")
134 if -1 in [dquote_idx, squote_idx]:
135 quote_idx = max(dquote_idx, squote_idx)
136 else:
137 quote_idx = min(squote_idx, dquote_idx)
139 assert (
140 0 <= quote_idx < len(string) - 1
141 ), f"{string!r} is missing a starting quote character (' or \")."
142 assert string[-1] in (
143 "'",
144 '"',
145 ), f"{string!r} is missing an ending quote character (' or \")."
146 assert set(string[:quote_idx]).issubset(
147 set(STRING_PREFIX_CHARS)
148 ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."
151def normalize_string_prefix(s: str, mode: Mode) -> str:
152 """Make all string prefixes lowercase."""
153 match = STRING_PREFIX_RE.match(s)
154 assert match is not None, f"failed to match string {s!r}"
155 orig_prefix = match.group(1)
156 new_prefix = (
157 orig_prefix.replace("F", "f")
158 .replace("B", "b")
159 .replace("U", "")
160 .replace("u", "")
161 )
163 if Preview.normalize_tstring_prefix in mode:
164 new_prefix = new_prefix.replace("T", "t")
165 # Python syntax guarantees max 2 prefixes and that one of them is "r"
166 if len(new_prefix) == 2 and new_prefix[0].lower() != "r":
167 new_prefix = new_prefix[::-1]
168 return f"{new_prefix}{match.group(2)}"
171# Re(gex) does actually cache patterns internally but this still improves
172# performance on a long list literal of strings by 5-9% since lru_cache's
173# caching overhead is much lower.
174@lru_cache(maxsize=64)
175def _cached_compile(pattern: str) -> Pattern[str]:
176 return re.compile(pattern)
179def _ends_with_unescaped_quote(body: str) -> bool:
180 """Does `body` end in a `"` that is not already backslash-escaped?
182 A backslash only escapes the quote when it is not itself escaped, so the run
183 of backslashes in front of the quote has to be of even length for the quote
184 to still need escaping.
185 """
186 if body[-1:] != '"':
187 return False
189 preceding = body[:-1]
190 backslashes = len(preceding) - len(preceding.rstrip("\\"))
191 return backslashes % 2 == 0
194def normalize_string_quotes(s: str) -> str:
195 """Prefer double quotes but only if it doesn't cause more escaping.
197 Adds or removes backslashes as appropriate.
198 """
199 value = s.lstrip(STRING_PREFIX_CHARS)
200 if value[:3] == '"""':
201 return s
203 elif value[:3] == "'''":
204 orig_quote = "'''"
205 new_quote = '"""'
206 elif value[0] == '"':
207 orig_quote = '"'
208 new_quote = "'"
209 else:
210 orig_quote = "'"
211 new_quote = '"'
212 first_quote_pos = s.find(orig_quote)
213 assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}"
215 prefix = s[:first_quote_pos]
216 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
217 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
218 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
219 body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]
220 if "r" in prefix.casefold():
221 if unescaped_new_quote.search(body):
222 # There's at least one unescaped new_quote in this raw string
223 # so converting is impossible
224 return s
226 # Do not introduce or remove backslashes in raw strings
227 new_body = body
228 else:
229 # remove unnecessary escapes
230 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
231 if body != new_body:
232 # Consider the string without unnecessary escapes as the original
233 body = new_body
234 s = f"{prefix}{orig_quote}{body}{orig_quote}"
235 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
236 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
238 if "f" in prefix.casefold() or "t" in prefix.casefold():
239 matches = re.findall(
240 r"""
241 (?:(?<!\{)|^)\{ # start of the string or a non-{ followed by a single {
242 ([^{].*?) # contents of the brackets except if begins with {{
243 \}(?:(?!\})|$) # A } followed by end of the string or a non-}
244 """,
245 new_body,
246 re.VERBOSE,
247 )
248 for m in matches:
249 if "\\" in str(m):
250 # Do not introduce backslashes in interpolated expressions
251 return s
253 if new_quote == '"""' and _ends_with_unescaped_quote(new_body):
254 # edge case:
255 new_body = new_body[:-1] + '\\"'
256 orig_escape_count = body.count("\\")
257 new_escape_count = new_body.count("\\")
258 if new_escape_count > orig_escape_count:
259 return s # Do not introduce more escaping
261 if new_escape_count == orig_escape_count and orig_quote == '"':
262 return s # Prefer double quotes
264 return f"{prefix}{new_quote}{new_body}{new_quote}"
267# TODO: only used in currently disabled f-string formatting
268def normalize_fstring_quotes(
269 quote: str,
270 middles: list[Leaf],
271 is_raw_fstring: bool,
272) -> tuple[list[Leaf], str]:
273 """Prefer double quotes but only if it doesn't cause more escaping.
275 Adds or removes backslashes as appropriate.
276 """
277 if quote == '"""':
278 return middles, quote
280 elif quote == "'''":
281 new_quote = '"""'
282 elif quote == '"':
283 new_quote = "'"
284 else:
285 new_quote = '"'
287 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
288 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
289 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){quote}")
290 if is_raw_fstring:
291 for middle in middles:
292 if unescaped_new_quote.search(middle.value):
293 # There's at least one unescaped new_quote in this raw string
294 # so converting is impossible
295 return middles, quote
297 # Do not introduce or remove backslashes in raw strings, just use double quote
298 return middles, '"'
300 new_segments = []
301 for middle in middles:
302 segment = middle.value
303 # remove unnecessary escapes
304 new_segment = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", segment)
305 if segment != new_segment:
306 # Consider the string without unnecessary escapes as the original
307 middle.value = new_segment
309 new_segment = sub_twice(escaped_orig_quote, rf"\1\2{quote}", new_segment)
310 new_segment = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_segment)
311 new_segments.append(new_segment)
313 if new_quote == '"""' and _ends_with_unescaped_quote(new_segments[-1]):
314 # edge case:
315 new_segments[-1] = new_segments[-1][:-1] + '\\"'
317 orig_escape_count = 0
318 new_escape_count = 0
319 for middle, new_segment in zip(middles, new_segments, strict=True):
320 orig_escape_count += middle.value.count("\\")
321 new_escape_count += new_segment.count("\\")
323 if new_escape_count > orig_escape_count:
324 return middles, quote # Do not introduce more escaping
326 if new_escape_count == orig_escape_count and quote == '"':
327 return middles, quote # Prefer double quotes
329 for middle, new_segment in zip(middles, new_segments, strict=True):
330 middle.value = new_segment
332 return middles, new_quote
335def _normalize_unicode_escape_sequences_in_text(
336 text: str, is_bytes: bool = False
337) -> str:
338 def replace(m: Match[str]) -> str:
339 groups = m.groupdict()
340 back_slashes = groups["backslashes"]
342 if groups["body"] is None or len(back_slashes) % 2 == 0:
343 return m.group(0)
345 if is_bytes and not groups["x"]:
346 # \u, \U and \N{} are not escape sequences in bytes literals
347 return m.group(0)
349 if groups["u"]:
350 # \u
351 return back_slashes + "u" + groups["u"].lower()
352 elif groups["U"]:
353 # \U
354 return back_slashes + "U" + groups["U"].lower()
355 elif groups["x"]:
356 # \x
357 return back_slashes + "x" + groups["x"].lower()
358 else:
359 assert groups["N"], f"Unexpected match: {m}"
360 # \N{}
361 return back_slashes + "N{" + groups["N"].upper() + "}"
363 return re.sub(UNICODE_ESCAPE_RE, replace, text)
366def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
367 """Replace hex codes in Unicode escape sequences with lowercase representation."""
368 text = leaf.value
369 prefix = get_string_prefix(text)
370 if "r" in prefix.lower():
371 return
373 leaf.value = _normalize_unicode_escape_sequences_in_text(
374 text, is_bytes="b" in prefix.lower()
375 )
378def _normalize_fstring_field_node(node: Leaf | Node) -> None:
379 from black.nodes import syms
381 if isinstance(node, Leaf):
382 if node.type == token.STRING:
383 normalize_unicode_escape_sequences(node)
384 return
386 if node.type in (syms.fstring, syms.tstring):
387 normalize_fstring_unicode_escape_sequences(node)
388 return
390 for child in node.children:
391 _normalize_fstring_field_node(child)
394def normalize_fstring_unicode_escape_sequences(node: Node) -> None:
395 """Replace hex codes in Unicode escape sequences with lowercase representation
396 within an f-string or t-string node."""
397 from black.nodes import syms
399 if not node.children:
400 return
402 first_child = node.children[0]
403 is_raw_fstring = (
404 isinstance(first_child, Leaf) and "r" in first_child.value.rstrip("'\"").lower()
405 )
407 for child in node.children:
408 if child.type in (token.FSTRING_MIDDLE, token.TSTRING_MIDDLE):
409 if not is_raw_fstring:
410 assert isinstance(child, Leaf)
411 child.value = _normalize_unicode_escape_sequences_in_text(child.value)
412 elif child.type in (
413 syms.fstring_replacement_field,
414 syms.tstring_replacement_field,
415 ):
416 # If this is a debug f-string (e.g. f"{foo=}"), do not touch the
417 # expression before '=', as Python preserves the exact source expression.
418 if any(
419 isinstance(c, Leaf) and c.type == token.EQUAL and c.value == "="
420 for c in child.children
421 ):
422 continue
424 for subchild in child.children:
425 _normalize_fstring_field_node(subchild)
428@lru_cache(maxsize=4096)
429def char_width(char: str) -> int:
430 """Return the width of a single character as it would be displayed in a
431 terminal or editor (which respects Unicode East Asian Width).
433 Full width characters are counted as 2, while half width characters are
434 counted as 1. Also control characters are counted as 0.
435 """
436 table = WIDTH_TABLE
437 codepoint = ord(char)
438 highest = len(table) - 1
439 lowest = 0
440 idx = highest // 2
441 while True:
442 start_codepoint, end_codepoint, width = table[idx]
443 if codepoint < start_codepoint:
444 highest = idx - 1
445 elif codepoint > end_codepoint:
446 lowest = idx + 1
447 else:
448 return 0 if width < 0 else width
449 if highest < lowest:
450 break
451 idx = (highest + lowest) // 2
452 return 1
455def str_width(line_str: str) -> int:
456 """Return the width of `line_str` as it would be displayed in a terminal
457 or editor (which respects Unicode East Asian Width).
459 You could utilize this function to determine, for example, if a string
460 is too wide to display in a terminal or editor.
461 """
462 if line_str.isascii():
463 # Fast path for a line consisting of only ASCII characters
464 return len(line_str)
465 return sum(map(char_width, line_str))
468def count_chars_in_width(line_str: str, max_width: int) -> int:
469 """Count the number of characters in `line_str` that would fit in a
470 terminal or editor of `max_width` (which respects Unicode East Asian
471 Width).
472 """
473 total_width = 0
474 for i, char in enumerate(line_str):
475 width = char_width(char)
476 if width + total_width > max_width:
477 return i
478 total_width += width
479 return len(line_str)