Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Early detection of escape-sequence-based encodings (ISO-2022, HZ-GB-2312, UTF-7).
3These encodings use ESC (0x1B), tilde (~), or plus (+) sequences to switch
4character sets. They must be detected before binary detection (ESC is a control
5byte) and before ASCII detection (HZ-GB-2312 and UTF-7 use only printable ASCII
6bytes plus their respective shift markers).
8Note: ``from __future__ import annotations`` is intentionally omitted because
9this module is compiled with mypyc, which does not support PEP 563 string
10annotations.
11"""
13from chardet._utils import EVIDENCE_CAP_BYTES, count_deleted, decodes_without_error
14from chardet.pipeline import DETERMINISTIC_CONFIDENCE, DetectionResult
16# Byte values legal inside an HZ-GB-2312 ``~{...~}`` region.
17_HZ_GB_BYTES: bytes = bytes(range(0x21, 0x7F))
20def _has_valid_hz_regions(data: bytes, max_start: int, max_end: int) -> bool:
21 """Check that at least one ~{...~} region contains valid GB2312 byte pairs.
23 In HZ-GB-2312 GB mode, characters are encoded as pairs of bytes in the
24 0x21-0x7E range. We require at least one region with a non-empty, even-
25 length run of such bytes.
27 *max_start* bounds where a region may open and *max_end* where it may
28 close. Keeping them separate is what lets a region that opens inside
29 the evidence window close beyond it: bounding both at the same offset
30 would cut a straddling region and hide the only escape evidence a
31 document has (ADR-0006).
32 """
33 start = 0
34 while True:
35 begin = data.find(b"~{", start)
36 if begin == -1 or begin >= max_start:
37 return False
38 end = data.find(b"~}", begin + 2, max_end)
39 if end == -1:
40 return False
41 region = data[begin + 2 : end]
42 # Must be non-empty, even length, and all bytes in GB2312 range.
43 # The range test is a C-level scan rather than a Python loop: a
44 # region may now run past the evidence window, and per-byte Python
45 # work proportional to it is exactly what the window exists to
46 # prevent.
47 if (
48 len(region) >= 2
49 and len(region) % 2 == 0
50 and count_deleted(region, _HZ_GB_BYTES) == len(region)
51 ):
52 return True
53 start = end + 2
56# Base64 alphabet used inside UTF-7 shifted sequences (+<Base64>-)
57_B64_CHARS: bytes = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
58_UTF7_BASE64: frozenset[int] = frozenset(_B64_CHARS)
60# Uppercase ASCII letters (A-Z), used by Guard C in _has_valid_utf7_sequences.
61_B64_UPPERCASE: frozenset[int] = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
63# Lookup table mapping each Base64 byte to its 6-bit value (0-63).
64_B64_DECODE: dict[int, int] = {c: i for i, c in enumerate(_B64_CHARS)}
66# Consecutive base64 characters before a '+' that mark it as part of a
67# base64 stream rather than a UTF-7 shift.
68_EMBEDDED_B64_RUN: int = 4
71def _is_valid_utf7_b64(b64_bytes: bytes) -> bool:
72 """Check if base64 bytes decode to valid UTF-16BE with correct padding.
74 A valid UTF-7 shifted sequence must:
75 1. Contain at least 3 Base64 characters (18 bits, enough for one 16-bit
76 UTF-16 code unit).
77 2. Have zero-valued trailing padding bits (the unused low bits of the last
78 Base64 sextet after the last complete 16-bit code unit).
79 3. Decode to valid UTF-16BE — no lone surrogates.
81 This rejects accidental ``+<alphanum>-`` patterns found in URLs, MIME
82 boundaries, hex-encoded hashes (e.g. SHA-1 git refs), and other ASCII data.
84 The caller (``_has_valid_utf7_sequences``) already checks ``b64_len >= 3``
85 before calling this function, so *b64_bytes* is always at least 3 bytes.
86 """
87 n = len(b64_bytes)
88 total_bits = n * 6
89 # Check that padding bits (trailing bits after last complete code unit)
90 # are zero.
91 padding_bits = total_bits % 16
92 if padding_bits > 0:
93 last_val = _B64_DECODE[b64_bytes[-1]]
94 # The low `padding_bits` of the last sextet must be zero
95 mask = (1 << padding_bits) - 1
96 if last_val & mask:
97 return False
98 # Decode the base64 to raw bytes and validate as UTF-16BE.
99 # Lone surrogates (unpaired 0xD800-0xDFFF code units) are illegal in
100 # well-formed UTF-16 and cannot appear in real UTF-7 text. This catches
101 # hex-encoded hashes and other accidental base64-like sequences.
102 # The bit buffer keeps only the bits not yet emitted: left to grow, it
103 # becomes an integer as wide as the run, and shifting it per sextet
104 # makes the decode quadratic in the run length.
105 num_bytes = total_bits // 8
106 raw = bytearray(num_bytes)
107 bit_buf = 0
108 bit_count = 0
109 out_idx = 0
110 for c in b64_bytes:
111 bit_buf = (bit_buf << 6) | _B64_DECODE[c]
112 bit_count += 6
113 if bit_count >= 8:
114 bit_count -= 8
115 raw[out_idx] = (bit_buf >> bit_count) & 0xFF
116 bit_buf &= (1 << bit_count) - 1
117 out_idx += 1
118 prev_high = False
119 for i in range(0, num_bytes - 1, 2):
120 code_unit = (raw[i] << 8) | raw[i + 1]
121 if 0xD800 <= code_unit <= 0xDBFF: # high surrogate
122 if prev_high:
123 return False # consecutive high surrogates
124 prev_high = True
125 elif 0xDC00 <= code_unit <= 0xDFFF: # low surrogate
126 if not prev_high:
127 return False # lone low surrogate
128 prev_high = False
129 else:
130 if prev_high:
131 return False # high surrogate not followed by low surrogate
132 prev_high = False
133 return not prev_high
136def _is_embedded_in_base64(data: bytes, pos: int) -> bool:
137 """Return True if the ``+`` at *pos* is embedded in a base64 stream.
139 Walks backward from *pos*, skipping CR/LF, and counts consecutive base64
140 characters (including ``=`` for padding). If 4 or more are found, the
141 ``+`` is likely part of a PEM certificate, email attachment, or similar
142 base64 blob rather than a real UTF-7 shift character.
144 The walk stops at the fourth character rather than running to the start
145 of the run. Only the ``>= 4`` verdict is used, and newlines are
146 skipped rather than ending the walk, so without the early exit a
147 line-wrapped base64 blob — a PEM file, a MIME attachment, exactly what
148 this guard exists to recognize — makes each ``+`` rescan everything
149 before it, and the caller quadratic in the size of the blob.
150 """
151 b64_with_pad: frozenset[int] = _UTF7_BASE64 | frozenset(b"=")
152 count = 0
153 i = pos - 1
154 while i >= 0:
155 b = data[i]
156 if b in {0x0A, 0x0D}: # skip newlines
157 i -= 1
158 continue
159 if b in b64_with_pad:
160 count += 1
161 if count >= _EMBEDDED_B64_RUN:
162 return True
163 i -= 1
164 else:
165 break
166 return False
169def _single_unit(b64_data: bytes) -> int:
170 """Decode the first (only) UTF-16 code unit of a single-unit block."""
171 return (
172 (_B64_DECODE[b64_data[0]] << 12)
173 | (_B64_DECODE[b64_data[1]] << 6)
174 | _B64_DECODE[b64_data[2]]
175 ) >> 2
178def _plausible_lone_unit(unit: int) -> bool:
179 """Script ranges where a lone shifted character plausibly occurs."""
180 return (
181 0x0080 <= unit <= 0x07FF # Latin supp. .. Arabic
182 or 0x0E00 <= unit <= 0x0FFF # Thai, Lao, Tibetan
183 or 0x2000 <= unit <= 0x2BFF # punctuation .. arrows (em dash, euro)
184 or 0x3000 <= unit <= 0x30FF # CJK punctuation, kana
185 or 0x4E00 <= unit <= 0x9FFF # CJK unified
186 or 0xAC00 <= unit <= 0xD7A3 # Hangul
187 or 0xFF00 <= unit <= 0xFFEF # fullwidth forms
188 )
191def _has_valid_utf7_sequences(data: bytes, max_start: int, max_end: int) -> bool:
192 """Check that *data* contains at least one valid UTF-7 shifted sequence.
194 A valid shifted sequence is ``+<base64 chars>`` terminated by either an
195 explicit ``-`` or any non-Base64 character (per RFC 2152). The base64
196 portion must decode to valid UTF-16BE with correct zero-padding bits.
197 The sequence ``+-`` is a literal plus sign and is **not** counted.
199 *max_start* bounds where a shift may occur and *max_end* how far its
200 base64 run may reach, so a run that begins inside the evidence window
201 can finish beyond it (ADR-0006). A run that outruns *max_end* is
202 skipped rather than judged on the part that fits: the padding and
203 surrogate checks read a cut run as a different run, and accepting on
204 evidence we did not see is the one direction this must never fail in.
205 """
206 start = 0
207 limit = min(len(data), max_end)
208 while True:
209 shift_pos = data.find(ord("+"), start)
210 if shift_pos == -1 or shift_pos >= max_start:
211 return False
212 pos = shift_pos + 1 # skip the '+'
213 # +- is a literal plus, not a shifted sequence
214 if pos < len(data) and data[pos] == ord("-"):
215 start = pos + 1
216 continue
217 # Guard A: '+' as the first base64 character encodes PUA code points
218 # (U+F800-U+FBFC) which never appear in real text. This catches
219 # patterns like "C++20" and "++row". Skip past ALL consecutive '+'
220 # characters so the next '+' in a run like ``++`` or ``+++`` is not
221 # re-examined as a new shift character.
222 if pos < len(data) and data[pos] == ord("+"):
223 while pos < len(data) and data[pos] == ord("+"):
224 pos += 1
225 start = pos
226 continue
227 # Guard B: if the '+' is embedded in a base64 stream (PEM, email
228 # attachment, etc.), it's not a real UTF-7 shift character.
229 if _is_embedded_in_base64(data, shift_pos):
230 start = pos
231 continue
232 # Collect consecutive Base64 characters
233 i = pos
234 while i < limit and data[i] in _UTF7_BASE64:
235 i += 1
236 if i == limit and limit < len(data) and data[i] in _UTF7_BASE64:
237 # The run outruns the bound — skip it (see the docstring).
238 start = i
239 continue
240 b64_len = i - pos
241 b64_data = data[pos:i]
242 # Guard C: reject base64 blocks with no uppercase letters.
243 # UTF-7 encodes UTF-16BE code points, and the high byte for virtually
244 # every script (Latin Extended, Cyrillic, Arabic, CJK, …) produces
245 # uppercase base64 characters. Sequences without any uppercase like
246 # "row", "foo", "pos" (variable names / English words) or "100", "99"
247 # (digit runs) are almost always ASCII text that accidentally follows a
248 # '+'. Out of 71,510 real UTF-7 base64 blocks in the test corpus, only
249 # 4 lack uppercase letters (0.006%).
250 #
251 # NOTE: this must test for the *absence of uppercase letters*, not
252 # ``bytes.islower()``. ``b"100".islower()`` is ``False`` (there are no
253 # cased characters at all), so an all-digit run like ``+100`` would slip
254 # through an ``islower()`` guard and be misdetected as UTF-7 (issue #371).
255 if b64_len >= 3 and not any(b in _B64_UPPERCASE for b in b64_data):
256 start = i
257 continue
258 # Accept if base64 content is valid UTF-16BE (padding bits check
259 # prevents false positives). Terminator can be '-', any non-Base64
260 # byte, or end of data — all per RFC 2152.
261 if b64_len >= 3 and _is_valid_utf7_b64(b64_data):
262 # Guard D: a block encoding a *single* code unit — with or
263 # without a dash terminator — must decode into a script range
264 # where a lone shifted character plausibly occurs. The corpus's
265 # single-unit blocks live in Latin/Greek/Cyrillic/Hebrew/Arabic
266 # supplements, Thai, general punctuation through arrows (em
267 # dashes and ellipses dominate), CJK, kana, Hangul, and
268 # fullwidth forms. An accidental uppercase run like "+LAY"
269 # (issue #371) decodes to U+2C06, Glagolitic — no genuine lone
270 # block in 149 corpus files lands in such a range. Multi-unit
271 # blocks are untouched: accidental ASCII does not survive the
272 # padding and surrogate checks for long runs.
273 if (b64_len * 6) // 16 == 1:
274 unit = _single_unit(b64_data)
275 if not _plausible_lone_unit(unit):
276 start = i
277 continue
278 return True
279 start = max(pos, i)
282def _iso2022_variant(data: bytes) -> tuple[str, str] | None:
283 """Name the ISO-2022 variant whose designations appear in *data*.
285 :returns: ``(encoding, language)`` or ``None`` when no designation is
286 present.
287 """
288 # ISO-2022-JP-2004: JIS X 0213 designations are unique to this variant.
289 # Python's codec reads the 2004 plane-1 (Q) and plane-2 (P) designations
290 # but not the 2000 plane-1 one (O), so an O stream fails the decode gate
291 # below and the other stages answer; naming it here would hand the
292 # caller an encoding that raises.
293 if b"\x1b$(O" in data or b"\x1b$(P" in data or b"\x1b$(Q" in data:
294 return "iso2022_jp_2004", "ja"
296 # ISO-2022-JP-EXT: JIS X 0201 Kana designation is unique to this variant.
297 if b"\x1b(I" in data:
298 return "iso2022_jp_ext", "ja"
300 # ISO-2022-JP base: JIS X 0208/0201/0212 designations.
301 if (
302 b"\x1b$B" in data
303 or b"\x1b$@" in data
304 or b"\x1b(J" in data
305 or b"\x1b$(D" in data # JIS X 0212-1990 (JP-1/JP-2/JP-EXT)
306 ):
307 # SI/SO (0x0E / 0x0F) shift controls -> JP-EXT
308 if b"\x0e" in data and b"\x0f" in data:
309 return "iso2022_jp_ext", "ja"
310 # Default to JP-2: a strict superset of JP and JP-1 that
311 # decodes all base sequences correctly.
312 return "iso2022_jp_2", "ja"
314 # ISO-2022-KR: ESC sequence for KS C 5601
315 if b"\x1b$)C" in data:
316 return "iso2022_kr", "ko"
318 return None
321def detect_escape_encoding(data: bytes) -> DetectionResult | None:
322 """Detect ISO-2022, HZ-GB-2312, and UTF-7 from escape/tilde/plus sequences.
324 Every answer this stage returns decodes the whole window: a designation
325 is three bytes that ordinary text can contain (``log: ESC $ B is the
326 escape``), and a genuine ISO-2022 or HZ stream can still carry a bad
327 byte past the header. The gate costs one decode per hit only.
329 :param data: The raw byte data to examine.
330 :returns: A :class:`DetectionResult` if an escape encoding is found, or ``None``.
331 """
332 has_esc = b"\x1b" in data
333 has_tilde = b"~" in data
334 has_plus = b"+" in data
336 if not has_esc and not has_tilde and not has_plus:
337 return None
339 if has_esc:
340 variant = _iso2022_variant(data)
341 if variant is not None and decodes_without_error(data, variant[0]):
342 return DetectionResult(
343 encoding=variant[0],
344 confidence=DETERMINISTIC_CONFIDENCE,
345 language=variant[1],
346 )
348 # Bounds for the deep validators below. They walk candidate sites in
349 # Python loops whose pathological case is *rejection* — a large tilde-
350 # or plus-heavy file that is neither HZ nor UTF-7 — so where a sequence
351 # may *begin* converges on the evidence cap (ADR-0006). Where it may
352 # *end* is a separate bound, one further window on, so a sequence that
353 # opens just inside the evidence window is judged whole rather than cut
354 # by the boundary and read as malformed. Nothing plausible sits
355 # between the two: a single escape run a quarter of a megabyte long is
356 # not text. Gates that decide whether an answer is *true* stay
357 # exhaustive over the window.
358 max_start = min(len(data), EVIDENCE_CAP_BYTES)
359 max_end = min(len(data), 2 * EVIDENCE_CAP_BYTES)
361 # HZ-GB-2312: tilde escapes for GB2312
362 # Require valid GB2312 byte pairs (0x21-0x7E range) between ~{ and ~}
363 # markers.
364 if (
365 has_tilde
366 and b"~{" in data
367 and b"~}" in data
368 and _has_valid_hz_regions(data, max_start, max_end)
369 and decodes_without_error(data, "hz")
370 ):
371 return DetectionResult(
372 encoding="hz",
373 confidence=DETERMINISTIC_CONFIDENCE,
374 language="zh",
375 )
377 # UTF-7: plus-sign shifts into Base64-encoded Unicode.
378 # UTF-7 is a 7-bit encoding (RFC 2152): every byte must be in 0x00-0x7F,
379 # checked over the whole window (isascii is exactly that predicate, at C
380 # scan speed). The buffer must also *decode* as UTF-7: tabular ASCII
381 # like "|16847+|" contains "+|", which is illegal (a shift must be
382 # followed by base64 or "-"), so the decode gate kills the
383 # delimited-data false-positive class outright while genuine UTF-7 —
384 # which real encoders emit as valid streams — always passes. The
385 # decoder fails fast on the first bad sequence.
386 #
387 # The decode gate keeps the *whole* window: it is what makes a utf-7
388 # answer true, and this stage returns it at deterministic confidence.
389 # Capping it would let an illegal shift past the cap pass as utf-7 that
390 # the caller's own ``decode`` then rejects.
391 #
392 # It runs *after* the bounded sequence validator, which is the reverse
393 # of the obvious order. A decode gate normally fails fast, but plain
394 # ASCII is valid UTF-7, so on the common case — a large ASCII file
395 # containing a '+' anywhere — it succeeds over every byte instead, at
396 # about a second per 272 MiB. The validator settles the same files
397 # from bounded evidence, leaving the full decode for data that already
398 # looks like UTF-7. Both are pure predicates, so the order changes
399 # only cost.
400 if (
401 has_plus
402 and data.isascii()
403 and _has_valid_utf7_sequences(data, max_start, max_end)
404 and decodes_without_error(data, "utf-7")
405 ):
406 return DetectionResult(
407 encoding="utf-7",
408 confidence=DETERMINISTIC_CONFIDENCE,
409 language=None,
410 )
412 return None