Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Early detection of escape-sequence-based encodings (ISO-2022, HZ-GB-2312, UTF-7).
3These encodings use ESC (0x1B), tilde (~), or plus (+) sequences to switch
4character sets. They must be detected before binary detection (ESC is a control
5byte) and before ASCII detection (HZ-GB-2312 and UTF-7 use only printable ASCII
6bytes plus their respective shift markers).
8Note: ``from __future__ import annotations`` is intentionally omitted because
9this module is compiled with mypyc, which does not support PEP 563 string
10annotations.
11"""
13from chardet._utils import EVIDENCE_CAP_BYTES, count_deleted, decodes_without_error
14from chardet.pipeline import DETERMINISTIC_CONFIDENCE, DetectionResult
16# Byte values legal inside an HZ-GB-2312 ``~{...~}`` region.
17_HZ_GB_BYTES: bytes = bytes(range(0x21, 0x7F))
20def _has_valid_hz_regions(data: bytes, max_start: int, max_end: int) -> bool:
21 """Check that at least one ~{...~} region contains valid GB2312 byte pairs.
23 In HZ-GB-2312 GB mode, characters are encoded as pairs of bytes in the
24 0x21-0x7E range. We require at least one region with a non-empty, even-
25 length run of such bytes.
27 *max_start* bounds where a region may open and *max_end* where it may
28 close. Keeping them separate is what lets a region that opens inside
29 the evidence window close beyond it: bounding both at the same offset
30 would cut a straddling region and hide the only escape evidence a
31 document has (ADR-0006).
32 """
33 start = 0
34 while True:
35 begin = data.find(b"~{", start)
36 if begin == -1 or begin >= max_start:
37 return False
38 end = data.find(b"~}", begin + 2, max_end)
39 if end == -1:
40 return False
41 region = data[begin + 2 : end]
42 # Must be non-empty, even length, and all bytes in GB2312 range.
43 # The range test is a C-level scan rather than a Python loop: a
44 # region may now run past the evidence window, and per-byte Python
45 # work proportional to it is exactly what the window exists to
46 # prevent.
47 if (
48 len(region) >= 2
49 and len(region) % 2 == 0
50 and count_deleted(region, _HZ_GB_BYTES) == len(region)
51 ):
52 return True
53 start = end + 2
56# Base64 alphabet used inside UTF-7 shifted sequences (+<Base64>-)
57_B64_CHARS: bytes = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
58_UTF7_BASE64: frozenset[int] = frozenset(_B64_CHARS)
60# Uppercase ASCII letters (A-Z), used by Guard C in _has_valid_utf7_sequences.
61_B64_UPPERCASE: frozenset[int] = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
63# Lookup table mapping each Base64 byte to its 6-bit value (0-63).
64_B64_DECODE: dict[int, int] = {c: i for i, c in enumerate(_B64_CHARS)}
66# Consecutive base64 characters before a '+' that mark it as part of a
67# base64 stream rather than a UTF-7 shift.
68_EMBEDDED_B64_RUN: int = 4
71def _is_valid_utf7_b64(b64_bytes: bytes) -> bool:
72 """Check if base64 bytes decode to valid UTF-16BE with correct padding.
74 A valid UTF-7 shifted sequence must:
75 1. Contain at least 3 Base64 characters (18 bits, enough for one 16-bit
76 UTF-16 code unit).
77 2. Have zero-valued trailing padding bits (the unused low bits of the last
78 Base64 sextet after the last complete 16-bit code unit).
79 3. Decode to valid UTF-16BE — no lone surrogates.
81 This rejects accidental ``+<alphanum>-`` patterns found in URLs, MIME
82 boundaries, hex-encoded hashes (e.g. SHA-1 git refs), and other ASCII data.
84 The caller (``_has_valid_utf7_sequences``) already checks ``b64_len >= 3``
85 before calling this function, so *b64_bytes* is always at least 3 bytes.
86 """
87 n = len(b64_bytes)
88 total_bits = n * 6
89 # Check that padding bits (trailing bits after last complete code unit)
90 # are zero.
91 padding_bits = total_bits % 16
92 if padding_bits > 0:
93 last_val = _B64_DECODE[b64_bytes[-1]]
94 # The low `padding_bits` of the last sextet must be zero
95 mask = (1 << padding_bits) - 1
96 if last_val & mask:
97 return False
98 # Decode the base64 to raw bytes and validate as UTF-16BE.
99 # Lone surrogates (unpaired 0xD800-0xDFFF code units) are illegal in
100 # well-formed UTF-16 and cannot appear in real UTF-7 text. This catches
101 # hex-encoded hashes and other accidental base64-like sequences.
102 num_bytes = total_bits // 8
103 raw = bytearray(num_bytes)
104 bit_buf = 0
105 bit_count = 0
106 out_idx = 0
107 for c in b64_bytes:
108 bit_buf = (bit_buf << 6) | _B64_DECODE[c]
109 bit_count += 6
110 if bit_count >= 8:
111 bit_count -= 8
112 raw[out_idx] = (bit_buf >> bit_count) & 0xFF
113 out_idx += 1
114 prev_high = False
115 for i in range(0, num_bytes - 1, 2):
116 code_unit = (raw[i] << 8) | raw[i + 1]
117 if 0xD800 <= code_unit <= 0xDBFF: # high surrogate
118 if prev_high:
119 return False # consecutive high surrogates
120 prev_high = True
121 elif 0xDC00 <= code_unit <= 0xDFFF: # low surrogate
122 if not prev_high:
123 return False # lone low surrogate
124 prev_high = False
125 else:
126 if prev_high:
127 return False # high surrogate not followed by low surrogate
128 prev_high = False
129 return not prev_high
132def _is_embedded_in_base64(data: bytes, pos: int) -> bool:
133 """Return True if the ``+`` at *pos* is embedded in a base64 stream.
135 Walks backward from *pos*, skipping CR/LF, and counts consecutive base64
136 characters (including ``=`` for padding). If 4 or more are found, the
137 ``+`` is likely part of a PEM certificate, email attachment, or similar
138 base64 blob rather than a real UTF-7 shift character.
140 The walk stops at the fourth character rather than running to the start
141 of the run. Only the ``>= 4`` verdict is used, and newlines are
142 skipped rather than ending the walk, so without the early exit a
143 line-wrapped base64 blob — a PEM file, a MIME attachment, exactly what
144 this guard exists to recognize — makes each ``+`` rescan everything
145 before it, and the caller quadratic in the size of the blob.
146 """
147 b64_with_pad: frozenset[int] = _UTF7_BASE64 | frozenset(b"=")
148 count = 0
149 i = pos - 1
150 while i >= 0:
151 b = data[i]
152 if b in {0x0A, 0x0D}: # skip newlines
153 i -= 1
154 continue
155 if b in b64_with_pad:
156 count += 1
157 if count >= _EMBEDDED_B64_RUN:
158 return True
159 i -= 1
160 else:
161 break
162 return False
165def _single_unit(b64_data: bytes) -> int:
166 """Decode the first (only) UTF-16 code unit of a single-unit block."""
167 return (
168 (_B64_DECODE[b64_data[0]] << 12)
169 | (_B64_DECODE[b64_data[1]] << 6)
170 | _B64_DECODE[b64_data[2]]
171 ) >> 2
174def _plausible_lone_unit(unit: int) -> bool:
175 """Script ranges where a lone shifted character plausibly occurs."""
176 return (
177 0x0080 <= unit <= 0x07FF # Latin supp. .. Arabic
178 or 0x0E00 <= unit <= 0x0FFF # Thai, Lao, Tibetan
179 or 0x2000 <= unit <= 0x2BFF # punctuation .. arrows (em dash, euro)
180 or 0x3000 <= unit <= 0x30FF # CJK punctuation, kana
181 or 0x4E00 <= unit <= 0x9FFF # CJK unified
182 or 0xAC00 <= unit <= 0xD7A3 # Hangul
183 or 0xFF00 <= unit <= 0xFFEF # fullwidth forms
184 )
187def _has_valid_utf7_sequences(data: bytes, max_start: int, max_end: int) -> bool:
188 """Check that *data* contains at least one valid UTF-7 shifted sequence.
190 A valid shifted sequence is ``+<base64 chars>`` terminated by either an
191 explicit ``-`` or any non-Base64 character (per RFC 2152). The base64
192 portion must decode to valid UTF-16BE with correct zero-padding bits.
193 The sequence ``+-`` is a literal plus sign and is **not** counted.
195 *max_start* bounds where a shift may occur and *max_end* how far its
196 base64 run may reach, so a run that begins inside the evidence window
197 can finish beyond it (ADR-0006). A run that outruns *max_end* is
198 skipped rather than judged on the part that fits: the padding and
199 surrogate checks read a cut run as a different run, and accepting on
200 evidence we did not see is the one direction this must never fail in.
201 """
202 start = 0
203 limit = min(len(data), max_end)
204 while True:
205 shift_pos = data.find(ord("+"), start)
206 if shift_pos == -1 or shift_pos >= max_start:
207 return False
208 pos = shift_pos + 1 # skip the '+'
209 # +- is a literal plus, not a shifted sequence
210 if pos < len(data) and data[pos] == ord("-"):
211 start = pos + 1
212 continue
213 # Guard A: '+' as the first base64 character encodes PUA code points
214 # (U+F800-U+FBFC) which never appear in real text. This catches
215 # patterns like "C++20" and "++row". Skip past ALL consecutive '+'
216 # characters so the next '+' in a run like ``++`` or ``+++`` is not
217 # re-examined as a new shift character.
218 if pos < len(data) and data[pos] == ord("+"):
219 while pos < len(data) and data[pos] == ord("+"):
220 pos += 1
221 start = pos
222 continue
223 # Guard B: if the '+' is embedded in a base64 stream (PEM, email
224 # attachment, etc.), it's not a real UTF-7 shift character.
225 if _is_embedded_in_base64(data, shift_pos):
226 start = pos
227 continue
228 # Collect consecutive Base64 characters
229 i = pos
230 while i < limit and data[i] in _UTF7_BASE64:
231 i += 1
232 if i == limit and limit < len(data) and data[i] in _UTF7_BASE64:
233 # The run outruns the bound — skip it (see the docstring).
234 start = i
235 continue
236 b64_len = i - pos
237 b64_data = data[pos:i]
238 # Guard C: reject base64 blocks with no uppercase letters.
239 # UTF-7 encodes UTF-16BE code points, and the high byte for virtually
240 # every script (Latin Extended, Cyrillic, Arabic, CJK, …) produces
241 # uppercase base64 characters. Sequences without any uppercase like
242 # "row", "foo", "pos" (variable names / English words) or "100", "99"
243 # (digit runs) are almost always ASCII text that accidentally follows a
244 # '+'. Out of 71,510 real UTF-7 base64 blocks in the test corpus, only
245 # 4 lack uppercase letters (0.006%).
246 #
247 # NOTE: this must test for the *absence of uppercase letters*, not
248 # ``bytes.islower()``. ``b"100".islower()`` is ``False`` (there are no
249 # cased characters at all), so an all-digit run like ``+100`` would slip
250 # through an ``islower()`` guard and be misdetected as UTF-7 (issue #371).
251 if b64_len >= 3 and not any(b in _B64_UPPERCASE for b in b64_data):
252 start = i
253 continue
254 # Accept if base64 content is valid UTF-16BE (padding bits check
255 # prevents false positives). Terminator can be '-', any non-Base64
256 # byte, or end of data — all per RFC 2152.
257 if b64_len >= 3 and _is_valid_utf7_b64(b64_data):
258 # Guard D: a block encoding a *single* code unit — with or
259 # without a dash terminator — must decode into a script range
260 # where a lone shifted character plausibly occurs. The corpus's
261 # single-unit blocks live in Latin/Greek/Cyrillic/Hebrew/Arabic
262 # supplements, Thai, general punctuation through arrows (em
263 # dashes and ellipses dominate), CJK, kana, Hangul, and
264 # fullwidth forms. An accidental uppercase run like "+LAY"
265 # (issue #371) decodes to U+2C06, Glagolitic — no genuine lone
266 # block in 149 corpus files lands in such a range. Multi-unit
267 # blocks are untouched: accidental ASCII does not survive the
268 # padding and surrogate checks for long runs.
269 if (b64_len * 6) // 16 == 1:
270 unit = _single_unit(b64_data)
271 if not _plausible_lone_unit(unit):
272 start = i
273 continue
274 return True
275 start = max(pos, i)
278def detect_escape_encoding(data: bytes) -> DetectionResult | None:
279 """Detect ISO-2022, HZ-GB-2312, and UTF-7 from escape/tilde/plus sequences.
281 :param data: The raw byte data to examine.
282 :returns: A :class:`DetectionResult` if an escape encoding is found, or ``None``.
283 """
284 has_esc = b"\x1b" in data
285 has_tilde = b"~" in data
286 has_plus = b"+" in data
288 if not has_esc and not has_tilde and not has_plus:
289 return None
291 if has_esc:
292 # ISO-2022-JP-2004: JIS X 0213 designations are unique to this variant.
293 if b"\x1b$(O" in data or b"\x1b$(P" in data or b"\x1b$(Q" in data:
294 return DetectionResult(
295 encoding="iso2022_jp_2004",
296 confidence=DETERMINISTIC_CONFIDENCE,
297 language="ja",
298 )
300 # ISO-2022-JP-EXT: JIS X 0201 Kana designation is unique to this variant.
301 if b"\x1b(I" in data:
302 return DetectionResult(
303 encoding="iso2022_jp_ext",
304 confidence=DETERMINISTIC_CONFIDENCE,
305 language="ja",
306 )
308 # ISO-2022-JP base: JIS X 0208/0201/0212 designations.
309 if (
310 b"\x1b$B" in data
311 or b"\x1b$@" in data
312 or b"\x1b(J" in data
313 or b"\x1b$(D" in data # JIS X 0212-1990 (JP-1/JP-2/JP-EXT)
314 ):
315 # SI/SO (0x0E / 0x0F) shift controls -> JP-EXT
316 if b"\x0e" in data and b"\x0f" in data:
317 return DetectionResult(
318 encoding="iso2022_jp_ext",
319 confidence=DETERMINISTIC_CONFIDENCE,
320 language="ja",
321 )
322 # Default to JP-2: a strict superset of JP and JP-1 that
323 # decodes all base sequences correctly.
324 return DetectionResult(
325 encoding="iso2022_jp_2",
326 confidence=DETERMINISTIC_CONFIDENCE,
327 language="ja",
328 )
330 # ISO-2022-KR: ESC sequence for KS C 5601
331 if b"\x1b$)C" in data:
332 return DetectionResult(
333 encoding="iso2022_kr",
334 confidence=DETERMINISTIC_CONFIDENCE,
335 language="ko",
336 )
338 # Bounds for the deep validators below. They walk candidate sites in
339 # Python loops whose pathological case is *rejection* — a large tilde-
340 # or plus-heavy file that is neither HZ nor UTF-7 — so where a sequence
341 # may *begin* converges on the evidence cap (ADR-0006). Where it may
342 # *end* is a separate bound, one further window on, so a sequence that
343 # opens just inside the evidence window is judged whole rather than cut
344 # by the boundary and read as malformed. Nothing plausible sits
345 # between the two: a single escape run a quarter of a megabyte long is
346 # not text. Gates that decide whether an answer is *true* stay
347 # exhaustive over the window.
348 max_start = min(len(data), EVIDENCE_CAP_BYTES)
349 max_end = min(len(data), 2 * EVIDENCE_CAP_BYTES)
351 # HZ-GB-2312: tilde escapes for GB2312
352 # Require valid GB2312 byte pairs (0x21-0x7E range) between ~{ and ~}
353 # markers.
354 if (
355 has_tilde
356 and b"~{" in data
357 and b"~}" in data
358 and _has_valid_hz_regions(data, max_start, max_end)
359 ):
360 return DetectionResult(
361 encoding="hz",
362 confidence=DETERMINISTIC_CONFIDENCE,
363 language="zh",
364 )
366 # UTF-7: plus-sign shifts into Base64-encoded Unicode.
367 # UTF-7 is a 7-bit encoding (RFC 2152): every byte must be in 0x00-0x7F,
368 # checked over the whole window (isascii is exactly that predicate, at C
369 # scan speed). The buffer must also *decode* as UTF-7: tabular ASCII
370 # like "|16847+|" contains "+|", which is illegal (a shift must be
371 # followed by base64 or "-"), so the decode gate kills the
372 # delimited-data false-positive class outright while genuine UTF-7 —
373 # which real encoders emit as valid streams — always passes. The
374 # decoder fails fast on the first bad sequence.
375 #
376 # The decode gate keeps the *whole* window: it is what makes a utf-7
377 # answer true, and this stage returns it at deterministic confidence.
378 # Capping it would let an illegal shift past the cap pass as utf-7 that
379 # the caller's own ``decode`` then rejects.
380 #
381 # It runs *after* the bounded sequence validator, which is the reverse
382 # of the obvious order. A decode gate normally fails fast, but plain
383 # ASCII is valid UTF-7, so on the common case — a large ASCII file
384 # containing a '+' anywhere — it succeeds over every byte instead, at
385 # about a second per 272 MiB. The validator settles the same files
386 # from bounded evidence, leaving the full decode for data that already
387 # looks like UTF-7. Both are pure predicates, so the order changes
388 # only cost.
389 if (
390 has_plus
391 and data.isascii()
392 and _has_valid_utf7_sequences(data, max_start, max_end)
393 and decodes_without_error(data, "utf-7")
394 ):
395 return DetectionResult(
396 encoding="utf-7",
397 confidence=DETERMINISTIC_CONFIDENCE,
398 language=None,
399 )
401 return None