Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/strings.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Simple formatting on strings. Further string formatting code is in trans.py.
3"""
5import re
6import sys
7from functools import lru_cache
8from re import Match, Pattern
9from typing import Final
11from black._width_table import WIDTH_TABLE
12from blib2to3.pytree import Leaf
14STRING_PREFIX_CHARS: Final = "fturbFTURB" # All possible string prefix characters.
15STRING_PREFIX_RE: Final = re.compile(
16 r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL
17)
18UNICODE_ESCAPE_RE: Final = re.compile(
19 r"(?P<backslashes>\\+)(?P<body>"
20 r"(u(?P<u>[a-fA-F0-9]{4}))" # Character with 16-bit hex value xxxx
21 r"|(U(?P<U>[a-fA-F0-9]{8}))" # Character with 32-bit hex value xxxxxxxx
22 r"|(x(?P<x>[a-fA-F0-9]{2}))" # Character with hex value hh
23 r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})" # Character named name in the Unicode database
24 r")?",
25 re.VERBOSE,
26)
29def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
30 """Replace `regex` with `replacement` twice on `original`.
32 This is used by string normalization to perform replaces on
33 overlapping matches.
34 """
35 return regex.sub(replacement, regex.sub(replacement, original))
38def has_triple_quotes(string: str) -> bool:
39 """
40 Returns:
41 True iff @string starts with three quotation characters.
42 """
43 raw_string = string.lstrip(STRING_PREFIX_CHARS)
44 return raw_string[:3] in {'"""', "'''"}
47def lines_with_leading_tabs_expanded(s: str) -> list[str]:
48 """
49 Splits string into lines and expands only leading tabs.
51 Black normalizes code indentation to four-space columns, so leading tabs in
52 docstrings need the same width to keep relative indentation stable.
53 """
54 lines = []
55 for line in s.splitlines():
56 stripped_line = line.lstrip()
57 if not stripped_line or stripped_line == line:
58 lines.append(line)
59 else:
60 prefix_length = len(line) - len(stripped_line)
61 prefix = line[:prefix_length].expandtabs(4)
62 lines.append(prefix + stripped_line)
63 if s.endswith("\n"):
64 lines.append("")
65 return lines
68def fix_multiline_docstring(docstring: str, prefix: str) -> str:
69 # https://peps.python.org/pep-0257/#handling-docstring-indentation
70 assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty"
71 lines = lines_with_leading_tabs_expanded(docstring)
72 # Determine minimum indentation (first line doesn't count):
73 indent = sys.maxsize
74 for line in lines[1:]:
75 stripped = line.lstrip()
76 if stripped:
77 indent = min(indent, len(line) - len(stripped))
78 # Remove indentation (first line is special):
79 trimmed = [lines[0].strip()]
80 if indent < sys.maxsize:
81 last_line_idx = len(lines) - 2
82 for i, line in enumerate(lines[1:]):
83 stripped_line = line[indent:].rstrip()
84 if stripped_line or i == last_line_idx:
85 trimmed.append(prefix + stripped_line)
86 else:
87 trimmed.append("")
88 return "\n".join(trimmed)
91def get_string_prefix(string: str) -> str:
92 """
93 Pre-conditions:
94 * assert_is_leaf_string(@string)
96 Returns:
97 @string's prefix (e.g. '', 'r', 'f', or 'rf').
98 """
99 assert_is_leaf_string(string)
101 prefix = []
102 for char in string:
103 if char in STRING_PREFIX_CHARS:
104 prefix.append(char)
105 else:
106 break
107 return "".join(prefix)
110def assert_is_leaf_string(string: str) -> None:
111 """
112 Checks the pre-condition that @string has the format that you would expect
113 of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
114 token.STRING`. A more precise description of the pre-conditions that are
115 checked are listed below.
117 Pre-conditions:
118 * @string starts with either ', ", <prefix>', or <prefix>" where
119 `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
120 * @string ends with a quote character (' or ").
122 Raises:
123 AssertionError(...) if the pre-conditions listed above are not
124 satisfied.
125 """
126 dquote_idx = string.find('"')
127 squote_idx = string.find("'")
128 if -1 in [dquote_idx, squote_idx]:
129 quote_idx = max(dquote_idx, squote_idx)
130 else:
131 quote_idx = min(squote_idx, dquote_idx)
133 assert (
134 0 <= quote_idx < len(string) - 1
135 ), f"{string!r} is missing a starting quote character (' or \")."
136 assert string[-1] in (
137 "'",
138 '"',
139 ), f"{string!r} is missing an ending quote character (' or \")."
140 assert set(string[:quote_idx]).issubset(
141 set(STRING_PREFIX_CHARS)
142 ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."
145def normalize_string_prefix(s: str) -> str:
146 """Make all string prefixes lowercase."""
147 match = STRING_PREFIX_RE.match(s)
148 assert match is not None, f"failed to match string {s!r}"
149 orig_prefix = match.group(1)
150 new_prefix = (
151 orig_prefix.replace("F", "f")
152 .replace("B", "b")
153 .replace("U", "")
154 .replace("u", "")
155 )
157 # Python syntax guarantees max 2 prefixes and that one of them is "r"
158 if len(new_prefix) == 2 and new_prefix[0].lower() != "r":
159 new_prefix = new_prefix[::-1]
160 return f"{new_prefix}{match.group(2)}"
163# Re(gex) does actually cache patterns internally but this still improves
164# performance on a long list literal of strings by 5-9% since lru_cache's
165# caching overhead is much lower.
166@lru_cache(maxsize=64)
167def _cached_compile(pattern: str) -> Pattern[str]:
168 return re.compile(pattern)
171def normalize_string_quotes(s: str) -> str:
172 """Prefer double quotes but only if it doesn't cause more escaping.
174 Adds or removes backslashes as appropriate.
175 """
176 value = s.lstrip(STRING_PREFIX_CHARS)
177 if value[:3] == '"""':
178 return s
180 elif value[:3] == "'''":
181 orig_quote = "'''"
182 new_quote = '"""'
183 elif value[0] == '"':
184 orig_quote = '"'
185 new_quote = "'"
186 else:
187 orig_quote = "'"
188 new_quote = '"'
189 first_quote_pos = s.find(orig_quote)
190 assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}"
192 prefix = s[:first_quote_pos]
193 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
194 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
195 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
196 body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]
197 if "r" in prefix.casefold():
198 if unescaped_new_quote.search(body):
199 # There's at least one unescaped new_quote in this raw string
200 # so converting is impossible
201 return s
203 # Do not introduce or remove backslashes in raw strings
204 new_body = body
205 else:
206 # remove unnecessary escapes
207 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
208 if body != new_body:
209 # Consider the string without unnecessary escapes as the original
210 body = new_body
211 s = f"{prefix}{orig_quote}{body}{orig_quote}"
212 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
213 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
215 if "f" in prefix.casefold():
216 matches = re.findall(
217 r"""
218 (?:(?<!\{)|^)\{ # start of the string or a non-{ followed by a single {
219 ([^{].*?) # contents of the brackets except if begins with {{
220 \}(?:(?!\})|$) # A } followed by end of the string or a non-}
221 """,
222 new_body,
223 re.VERBOSE,
224 )
225 for m in matches:
226 if "\\" in str(m):
227 # Do not introduce backslashes in interpolated expressions
228 return s
230 if new_quote == '"""' and new_body[-1:] == '"':
231 # edge case:
232 new_body = new_body[:-1] + '\\"'
233 orig_escape_count = body.count("\\")
234 new_escape_count = new_body.count("\\")
235 if new_escape_count > orig_escape_count:
236 return s # Do not introduce more escaping
238 if new_escape_count == orig_escape_count and orig_quote == '"':
239 return s # Prefer double quotes
241 return f"{prefix}{new_quote}{new_body}{new_quote}"
244def normalize_fstring_quotes(
245 quote: str,
246 middles: list[Leaf],
247 is_raw_fstring: bool,
248) -> tuple[list[Leaf], str]:
249 """Prefer double quotes but only if it doesn't cause more escaping.
251 Adds or removes backslashes as appropriate.
252 """
253 if quote == '"""':
254 return middles, quote
256 elif quote == "'''":
257 new_quote = '"""'
258 elif quote == '"':
259 new_quote = "'"
260 else:
261 new_quote = '"'
263 unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
264 escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
265 escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){quote}")
266 if is_raw_fstring:
267 for middle in middles:
268 if unescaped_new_quote.search(middle.value):
269 # There's at least one unescaped new_quote in this raw string
270 # so converting is impossible
271 return middles, quote
273 # Do not introduce or remove backslashes in raw strings, just use double quote
274 return middles, '"'
276 new_segments = []
277 for middle in middles:
278 segment = middle.value
279 # remove unnecessary escapes
280 new_segment = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", segment)
281 if segment != new_segment:
282 # Consider the string without unnecessary escapes as the original
283 middle.value = new_segment
285 new_segment = sub_twice(escaped_orig_quote, rf"\1\2{quote}", new_segment)
286 new_segment = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_segment)
287 new_segments.append(new_segment)
289 if new_quote == '"""' and new_segments[-1].endswith('"'):
290 # edge case:
291 new_segments[-1] = new_segments[-1][:-1] + '\\"'
293 orig_escape_count = 0
294 new_escape_count = 0
295 for middle, new_segment in zip(middles, new_segments, strict=True):
296 orig_escape_count += middle.value.count("\\")
297 new_escape_count += new_segment.count("\\")
299 if new_escape_count > orig_escape_count:
300 return middles, quote # Do not introduce more escaping
302 if new_escape_count == orig_escape_count and quote == '"':
303 return middles, quote # Prefer double quotes
305 for middle, new_segment in zip(middles, new_segments, strict=True):
306 middle.value = new_segment
308 return middles, new_quote
311def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
312 """Replace hex codes in Unicode escape sequences with lowercase representation."""
313 text = leaf.value
314 prefix = get_string_prefix(text)
315 if "r" in prefix.lower():
316 return
318 def replace(m: Match[str]) -> str:
319 groups = m.groupdict()
320 back_slashes = groups["backslashes"]
322 if groups["body"] is None or len(back_slashes) % 2 == 0:
323 return m.group(0)
325 if groups["u"]:
326 # \u
327 return back_slashes + "u" + groups["u"].lower()
328 elif groups["U"]:
329 # \U
330 return back_slashes + "U" + groups["U"].lower()
331 elif groups["x"]:
332 # \x
333 return back_slashes + "x" + groups["x"].lower()
334 else:
335 assert groups["N"], f"Unexpected match: {m}"
336 # \N{}
337 return back_slashes + "N{" + groups["N"].upper() + "}"
339 leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)
342@lru_cache(maxsize=4096)
343def char_width(char: str) -> int:
344 """Return the width of a single character as it would be displayed in a
345 terminal or editor (which respects Unicode East Asian Width).
347 Full width characters are counted as 2, while half width characters are
348 counted as 1. Also control characters are counted as 0.
349 """
350 table = WIDTH_TABLE
351 codepoint = ord(char)
352 highest = len(table) - 1
353 lowest = 0
354 idx = highest // 2
355 while True:
356 start_codepoint, end_codepoint, width = table[idx]
357 if codepoint < start_codepoint:
358 highest = idx - 1
359 elif codepoint > end_codepoint:
360 lowest = idx + 1
361 else:
362 return 0 if width < 0 else width
363 if highest < lowest:
364 break
365 idx = (highest + lowest) // 2
366 return 1
369def str_width(line_str: str) -> int:
370 """Return the width of `line_str` as it would be displayed in a terminal
371 or editor (which respects Unicode East Asian Width).
373 You could utilize this function to determine, for example, if a string
374 is too wide to display in a terminal or editor.
375 """
376 if line_str.isascii():
377 # Fast path for a line consisting of only ASCII characters
378 return len(line_str)
379 return sum(map(char_width, line_str))
382def count_chars_in_width(line_str: str, max_width: int) -> int:
383 """Count the number of characters in `line_str` that would fit in a
384 terminal or editor of `max_width` (which respects Unicode East Asian
385 Width).
386 """
387 total_width = 0
388 for i, char in enumerate(line_str):
389 width = char_width(char)
390 if width + total_width > max_width:
391 return i
392 total_width += width
393 return len(line_str)