Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/strings.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Simple formatting on strings. Further string formatting code is in trans.py.
3"""
5import re
6import sys
7from functools import lru_cache
8from re import Match, Pattern
9from typing import Final
11from black._width_table import WIDTH_TABLE
12from blib2to3.pytree import Leaf
14STRING_PREFIX_CHARS: Final = "fturbFTURB" # All possible string prefix characters.
15STRING_PREFIX_RE: Final = re.compile(
16 r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL
17)
18UNICODE_ESCAPE_RE: Final = re.compile(
19 r"(?P<backslashes>\\+)(?P<body>"
20 r"(u(?P<u>[a-fA-F0-9]{4}))" # Character with 16-bit hex value xxxx
21 r"|(U(?P<U>[a-fA-F0-9]{8}))" # Character with 32-bit hex value xxxxxxxx
22 r"|(x(?P<x>[a-fA-F0-9]{2}))" # Character with hex value hh
23 r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})" # Character named name in the Unicode database
24 r")?",
25 re.VERBOSE,
26)
29def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
30 """Replace `regex` with `replacement` twice on `original`.
32 This is used by string normalization to perform replaces on
33 overlapping matches.
34 """
35 return regex.sub(replacement, regex.sub(replacement, original))
38def has_triple_quotes(string: str) -> bool:
39 """
40 Returns:
41 True iff @string starts with three quotation characters.
42 """
43 raw_string = string.lstrip(STRING_PREFIX_CHARS)
44 return raw_string[:3] in {'"""', "'''"}
47def lines_with_leading_tabs_expanded(s: str) -> list[str]:
48 """
49 Splits string into lines and expands only leading tabs.
51 Black normalizes code indentation to four-space columns, so leading tabs in
52 docstrings need the same width to keep relative indentation stable.
53 """
54 lines = []
55 for line in s.splitlines():
56 stripped_line = line.lstrip()
57 if not stripped_line or stripped_line == line:
58 lines.append(line)
59 else:
60 prefix_length = len(line) - len(stripped_line)
61 prefix = line[:prefix_length].expandtabs(4)
62 lines.append(prefix + stripped_line)
63 if s.endswith("\n"):
64 lines.append("")
65 return lines
68def fix_multiline_docstring(docstring: str, prefix: str) -> str:
69 # https://peps.python.org/pep-0257/#handling-docstring-indentation
70 assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty"
71 lines = lines_with_leading_tabs_expanded(docstring)
72 # Determine minimum indentation (first line doesn't count):
73 indent = sys.maxsize
74 for line in lines[1:]:
75 stripped = line.lstrip()
76 if stripped:
77 indent = min(indent, len(line) - len(stripped))
78 # Remove indentation (first line is special):
79 trimmed = [lines[0].strip()]
80 if indent < sys.maxsize:
81 last_line_idx = len(lines) - 2
82 for i, line in enumerate(lines[1:]):
83 stripped_line = line[indent:].rstrip()
84 if stripped_line or i == last_line_idx:
85 trimmed.append(prefix + stripped_line)
86 else:
87 trimmed.append("")
88 return "\n".join(trimmed)
91def get_string_prefix(string: str) -> str:
92 """
93 Pre-conditions:
94 * assert_is_leaf_string(@string)
96 Returns:
97 @string's prefix (e.g. '', 'r', 'f', or 'rf').
98 """
99 assert_is_leaf_string(string)
101 prefix = []
102 for char in string:
103 if char in STRING_PREFIX_CHARS:
104 prefix.append(char)
105 else:
106 break
107 return "".join(prefix)
110def assert_is_leaf_string(string: str) -> None:
111 """
112 Checks the pre-condition that @string has the format that you would expect
113 of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
114 token.STRING`. A more precise description of the pre-conditions that are
115 checked are listed below.
117 Pre-conditions:
118 * @string starts with either ', ", <prefix>', or <prefix>" where
119 `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
120 * @string ends with a quote character (' or ").
122 Raises:
123 AssertionError(...) if the pre-conditions listed above are not
124 satisfied.
125 """
126 dquote_idx = string.find('"')
127 squote_idx = string.find("'")
128 if -1 in [dquote_idx, squote_idx]:
129 quote_idx = max(dquote_idx, squote_idx)
130 else:
131 quote_idx = min(squote_idx, dquote_idx)
133 assert (
134 0 <= quote_idx < len(string) - 1
135 ), f"{string!r} is missing a starting quote character (' or \")."
136 assert string[-1] in (
137 "'",
138 '"',
139 ), f"{string!r} is missing an ending quote character (' or \")."
140 assert set(string[:quote_idx]).issubset(
141 set(STRING_PREFIX_CHARS)
142 ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."
145def normalize_string_prefix(s: str) -> str:
146 """Make all string prefixes lowercase."""
147 match = STRING_PREFIX_RE.match(s)
148 assert match is not None, f"failed to match string {s!r}"
149 orig_prefix = match.group(1)
150 new_prefix = (
151 orig_prefix.replace("F", "f")
152 .replace("B", "b")
153 .replace("U", "")
154 .replace("u", "")
155 )
157 # Python syntax guarantees max 2 prefixes and that one of them is "r"
158 if len(new_prefix) == 2 and new_prefix[0].lower() != "r":
159 new_prefix = new_prefix[::-1]
160 return f"{new_prefix}{match.group(2)}"
163# Re(gex) does actually cache patterns internally but this still improves
164# performance on a long list literal of strings by 5-9% since lru_cache's
165# caching overhead is much lower.
166@lru_cache(maxsize=64)
167def _cached_compile(pattern: str) -> Pattern[str]:
168 return re.compile(pattern)
171def _ends_with_unescaped_quote(body: str) -> bool:
172 """Does `body` end in a `"` that is not already backslash-escaped?
174 A backslash only escapes the quote when it is not itself escaped, so the run
175 of backslashes in front of the quote has to be of even length for the quote
176 to still need escaping.
177 """
178 if body[-1:] != '"':
179 return False
181 preceding = body[:-1]
182 backslashes = len(preceding) - len(preceding.rstrip("\\"))
183 return backslashes % 2 == 0
186def normalize_string_quotes(s: str) -> str:
187 """Prefer double quotes but only if it doesn't cause more escaping.
189 Adds or removes backslashes as appropriate.
190 """
191 value = s.lstrip(STRING_PREFIX_CHARS)
192 if value[:3] == '"""':
193 return s
195 elif value[:3] == "'''":
196 orig_quote = "'''"
197 new_quote = '"""'
198 elif value[0] == '"':
199 orig_quote = '"'
200 new_quote = "'"
201 else:
202 orig_quote = "'"
203 new_quote = '"'
204 first_quote_pos = s.find(orig_quote)
205 assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}"
207 prefix = s[:first_quote_pos]
208 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
209 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
210 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
211 body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]
212 if "r" in prefix.casefold():
213 if unescaped_new_quote.search(body):
214 # There's at least one unescaped new_quote in this raw string
215 # so converting is impossible
216 return s
218 # Do not introduce or remove backslashes in raw strings
219 new_body = body
220 else:
221 # remove unnecessary escapes
222 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
223 if body != new_body:
224 # Consider the string without unnecessary escapes as the original
225 body = new_body
226 s = f"{prefix}{orig_quote}{body}{orig_quote}"
227 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
228 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
230 if "f" in prefix.casefold() or "t" in prefix.casefold():
231 matches = re.findall(
232 r"""
233 (?:(?<!\{)|^)\{ # start of the string or a non-{ followed by a single {
234 ([^{].*?) # contents of the brackets except if begins with {{
235 \}(?:(?!\})|$) # A } followed by end of the string or a non-}
236 """,
237 new_body,
238 re.VERBOSE,
239 )
240 for m in matches:
241 if "\\" in str(m):
242 # Do not introduce backslashes in interpolated expressions
243 return s
245 if new_quote == '"""' and _ends_with_unescaped_quote(new_body):
246 # edge case:
247 new_body = new_body[:-1] + '\\"'
248 orig_escape_count = body.count("\\")
249 new_escape_count = new_body.count("\\")
250 if new_escape_count > orig_escape_count:
251 return s # Do not introduce more escaping
253 if new_escape_count == orig_escape_count and orig_quote == '"':
254 return s # Prefer double quotes
256 return f"{prefix}{new_quote}{new_body}{new_quote}"
259def normalize_fstring_quotes(
260 quote: str,
261 middles: list[Leaf],
262 is_raw_fstring: bool,
263) -> tuple[list[Leaf], str]:
264 """Prefer double quotes but only if it doesn't cause more escaping.
266 Adds or removes backslashes as appropriate.
267 """
268 if quote == '"""':
269 return middles, quote
271 elif quote == "'''":
272 new_quote = '"""'
273 elif quote == '"':
274 new_quote = "'"
275 else:
276 new_quote = '"'
278 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
279 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
280 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){quote}")
281 if is_raw_fstring:
282 for middle in middles:
283 if unescaped_new_quote.search(middle.value):
284 # There's at least one unescaped new_quote in this raw string
285 # so converting is impossible
286 return middles, quote
288 # Do not introduce or remove backslashes in raw strings, just use double quote
289 return middles, '"'
291 new_segments = []
292 for middle in middles:
293 segment = middle.value
294 # remove unnecessary escapes
295 new_segment = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", segment)
296 if segment != new_segment:
297 # Consider the string without unnecessary escapes as the original
298 middle.value = new_segment
300 new_segment = sub_twice(escaped_orig_quote, rf"\1\2{quote}", new_segment)
301 new_segment = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_segment)
302 new_segments.append(new_segment)
304 if new_quote == '"""' and _ends_with_unescaped_quote(new_segments[-1]):
305 # edge case:
306 new_segments[-1] = new_segments[-1][:-1] + '\\"'
308 orig_escape_count = 0
309 new_escape_count = 0
310 for middle, new_segment in zip(middles, new_segments, strict=True):
311 orig_escape_count += middle.value.count("\\")
312 new_escape_count += new_segment.count("\\")
314 if new_escape_count > orig_escape_count:
315 return middles, quote # Do not introduce more escaping
317 if new_escape_count == orig_escape_count and quote == '"':
318 return middles, quote # Prefer double quotes
320 for middle, new_segment in zip(middles, new_segments, strict=True):
321 middle.value = new_segment
323 return middles, new_quote
326def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
327 """Replace hex codes in Unicode escape sequences with lowercase representation."""
328 text = leaf.value
329 prefix = get_string_prefix(text)
330 if "r" in prefix.lower():
331 return
333 def replace(m: Match[str]) -> str:
334 groups = m.groupdict()
335 back_slashes = groups["backslashes"]
337 if groups["body"] is None or len(back_slashes) % 2 == 0:
338 return m.group(0)
340 if groups["u"]:
341 # \u
342 return back_slashes + "u" + groups["u"].lower()
343 elif groups["U"]:
344 # \U
345 return back_slashes + "U" + groups["U"].lower()
346 elif groups["x"]:
347 # \x
348 return back_slashes + "x" + groups["x"].lower()
349 else:
350 assert groups["N"], f"Unexpected match: {m}"
351 # \N{}
352 return back_slashes + "N{" + groups["N"].upper() + "}"
354 leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)
357@lru_cache(maxsize=4096)
358def char_width(char: str) -> int:
359 """Return the width of a single character as it would be displayed in a
360 terminal or editor (which respects Unicode East Asian Width).
362 Full width characters are counted as 2, while half width characters are
363 counted as 1. Also control characters are counted as 0.
364 """
365 table = WIDTH_TABLE
366 codepoint = ord(char)
367 highest = len(table) - 1
368 lowest = 0
369 idx = highest // 2
370 while True:
371 start_codepoint, end_codepoint, width = table[idx]
372 if codepoint < start_codepoint:
373 highest = idx - 1
374 elif codepoint > end_codepoint:
375 lowest = idx + 1
376 else:
377 return 0 if width < 0 else width
378 if highest < lowest:
379 break
380 idx = (highest + lowest) // 2
381 return 1
384def str_width(line_str: str) -> int:
385 """Return the width of `line_str` as it would be displayed in a terminal
386 or editor (which respects Unicode East Asian Width).
388 You could utilize this function to determine, for example, if a string
389 is too wide to display in a terminal or editor.
390 """
391 if line_str.isascii():
392 # Fast path for a line consisting of only ASCII characters
393 return len(line_str)
394 return sum(map(char_width, line_str))
397def count_chars_in_width(line_str: str, max_width: int) -> int:
398 """Count the number of characters in `line_str` that would fit in a
399 terminal or editor of `max_width` (which respects Unicode East Asian
400 Width).
401 """
402 total_width = 0
403 for i, char in enumerate(line_str):
404 width = char_width(char)
405 if width + total_width > max_width:
406 return i
407 total_width += width
408 return len(line_str)