Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Early detection of escape-sequence-based encodings (ISO-2022, HZ-GB-2312, UTF-7).
3These encodings use ESC (0x1B), tilde (~), or plus (+) sequences to switch
4character sets. They must be detected before binary detection (ESC is a control
5byte) and before ASCII detection (HZ-GB-2312 and UTF-7 use only printable ASCII
6bytes plus their respective shift markers).
8Note: ``from __future__ import annotations`` is intentionally omitted because
9this module is compiled with mypyc, which does not support PEP 563 string
10annotations.
11"""
13from chardet.pipeline import DETERMINISTIC_CONFIDENCE, DetectionResult
16def _has_valid_hz_regions(data: bytes) -> bool:
17 """Check that at least one ~{...~} region contains valid GB2312 byte pairs.
19 In HZ-GB-2312 GB mode, characters are encoded as pairs of bytes in the
20 0x21-0x7E range. We require at least one region with a non-empty, even-
21 length run of such bytes.
22 """
23 start = 0
24 while True:
25 begin = data.find(b"~{", start)
26 if begin == -1:
27 return False
28 end = data.find(b"~}", begin + 2)
29 if end == -1:
30 return False
31 region = data[begin + 2 : end]
32 # Must be non-empty, even length, and all bytes in GB2312 range
33 if (
34 len(region) >= 2
35 and len(region) % 2 == 0
36 and all(0x21 <= b <= 0x7E for b in region)
37 ):
38 return True
39 start = end + 2
42# Base64 alphabet used inside UTF-7 shifted sequences (+<Base64>-)
43_B64_CHARS: bytes = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
44_UTF7_BASE64: frozenset[int] = frozenset(_B64_CHARS)
46# Uppercase ASCII letters (A-Z), used by Guard C in _has_valid_utf7_sequences.
47_B64_UPPERCASE: frozenset[int] = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
49# Lookup table mapping each Base64 byte to its 6-bit value (0-63).
50_B64_DECODE: dict[int, int] = {c: i for i, c in enumerate(_B64_CHARS)}
53def _is_valid_utf7_b64(b64_bytes: bytes) -> bool:
54 """Check if base64 bytes decode to valid UTF-16BE with correct padding.
56 A valid UTF-7 shifted sequence must:
57 1. Contain at least 3 Base64 characters (18 bits, enough for one 16-bit
58 UTF-16 code unit).
59 2. Have zero-valued trailing padding bits (the unused low bits of the last
60 Base64 sextet after the last complete 16-bit code unit).
61 3. Decode to valid UTF-16BE — no lone surrogates.
63 This rejects accidental ``+<alphanum>-`` patterns found in URLs, MIME
64 boundaries, hex-encoded hashes (e.g. SHA-1 git refs), and other ASCII data.
66 The caller (``_has_valid_utf7_sequences``) already checks ``b64_len >= 3``
67 before calling this function, so *b64_bytes* is always at least 3 bytes.
68 """
69 n = len(b64_bytes)
70 total_bits = n * 6
71 # Check that padding bits (trailing bits after last complete code unit)
72 # are zero.
73 padding_bits = total_bits % 16
74 if padding_bits > 0:
75 last_val = _B64_DECODE[b64_bytes[-1]]
76 # The low `padding_bits` of the last sextet must be zero
77 mask = (1 << padding_bits) - 1
78 if last_val & mask:
79 return False
80 # Decode the base64 to raw bytes and validate as UTF-16BE.
81 # Lone surrogates (unpaired 0xD800-0xDFFF code units) are illegal in
82 # well-formed UTF-16 and cannot appear in real UTF-7 text. This catches
83 # hex-encoded hashes and other accidental base64-like sequences.
84 num_bytes = total_bits // 8
85 raw = bytearray(num_bytes)
86 bit_buf = 0
87 bit_count = 0
88 out_idx = 0
89 for c in b64_bytes:
90 bit_buf = (bit_buf << 6) | _B64_DECODE[c]
91 bit_count += 6
92 if bit_count >= 8:
93 bit_count -= 8
94 raw[out_idx] = (bit_buf >> bit_count) & 0xFF
95 out_idx += 1
96 prev_high = False
97 for i in range(0, num_bytes - 1, 2):
98 code_unit = (raw[i] << 8) | raw[i + 1]
99 if 0xD800 <= code_unit <= 0xDBFF: # high surrogate
100 if prev_high:
101 return False # consecutive high surrogates
102 prev_high = True
103 elif 0xDC00 <= code_unit <= 0xDFFF: # low surrogate
104 if not prev_high:
105 return False # lone low surrogate
106 prev_high = False
107 else:
108 if prev_high:
109 return False # high surrogate not followed by low surrogate
110 prev_high = False
111 return not prev_high
114def _is_embedded_in_base64(data: bytes, pos: int) -> bool:
115 """Return True if the ``+`` at *pos* is embedded in a base64 stream.
117 Walks backward from *pos*, skipping CR/LF, and counts consecutive base64
118 characters (including ``=`` for padding). If 4 or more are found, the
119 ``+`` is likely part of a PEM certificate, email attachment, or similar
120 base64 blob rather than a real UTF-7 shift character.
121 """
122 b64_with_pad: frozenset[int] = _UTF7_BASE64 | frozenset(b"=")
123 count = 0
124 i = pos - 1
125 while i >= 0:
126 b = data[i]
127 if b in {0x0A, 0x0D}: # skip newlines
128 i -= 1
129 continue
130 if b in b64_with_pad:
131 count += 1
132 i -= 1
133 else:
134 break
135 return count >= 4
138def _has_valid_utf7_sequences(data: bytes) -> bool:
139 """Check that *data* contains at least one valid UTF-7 shifted sequence.
141 A valid shifted sequence is ``+<base64 chars>`` terminated by either an
142 explicit ``-`` or any non-Base64 character (per RFC 2152). The base64
143 portion must decode to valid UTF-16BE with correct zero-padding bits.
144 The sequence ``+-`` is a literal plus sign and is **not** counted.
145 """
146 start = 0
147 while True:
148 shift_pos = data.find(ord("+"), start)
149 if shift_pos == -1:
150 return False
151 pos = shift_pos + 1 # skip the '+'
152 # +- is a literal plus, not a shifted sequence
153 if pos < len(data) and data[pos] == ord("-"):
154 start = pos + 1
155 continue
156 # Guard A: '+' as the first base64 character encodes PUA code points
157 # (U+F800-U+FBFC) which never appear in real text. This catches
158 # patterns like "C++20" and "++row". Skip past ALL consecutive '+'
159 # characters so the next '+' in a run like ``++`` or ``+++`` is not
160 # re-examined as a new shift character.
161 if pos < len(data) and data[pos] == ord("+"):
162 while pos < len(data) and data[pos] == ord("+"):
163 pos += 1
164 start = pos
165 continue
166 # Guard B: if the '+' is embedded in a base64 stream (PEM, email
167 # attachment, etc.), it's not a real UTF-7 shift character.
168 if _is_embedded_in_base64(data, shift_pos):
169 start = pos
170 continue
171 # Collect consecutive Base64 characters
172 i = pos
173 while i < len(data) and data[i] in _UTF7_BASE64:
174 i += 1
175 b64_len = i - pos
176 b64_data = data[pos:i]
177 # Guard C: reject base64 blocks with no uppercase letters.
178 # UTF-7 encodes UTF-16BE code points, and the high byte for virtually
179 # every script (Latin Extended, Cyrillic, Arabic, CJK, …) produces
180 # uppercase base64 characters. Sequences without any uppercase like
181 # "row", "foo", "pos" (variable names / English words) or "100", "99"
182 # (digit runs) are almost always ASCII text that accidentally follows a
183 # '+'. Out of 71,510 real UTF-7 base64 blocks in the test corpus, only
184 # 4 lack uppercase letters (0.006%).
185 #
186 # NOTE: this must test for the *absence of uppercase letters*, not
187 # ``bytes.islower()``. ``b"100".islower()`` is ``False`` (there are no
188 # cased characters at all), so an all-digit run like ``+100`` would slip
189 # through an ``islower()`` guard and be misdetected as UTF-7 (issue #371).
190 if b64_len >= 3 and not any(b in _B64_UPPERCASE for b in b64_data):
191 start = i
192 continue
193 # Accept if base64 content is valid UTF-16BE (padding bits check
194 # prevents false positives). Terminator can be '-', any non-Base64
195 # byte, or end of data — all per RFC 2152.
196 if b64_len >= 3 and _is_valid_utf7_b64(b64_data):
197 return True
198 start = max(pos, i)
201def detect_escape_encoding(data: bytes) -> DetectionResult | None:
202 """Detect ISO-2022, HZ-GB-2312, and UTF-7 from escape/tilde/plus sequences.
204 :param data: The raw byte data to examine.
205 :returns: A :class:`DetectionResult` if an escape encoding is found, or ``None``.
206 """
207 has_esc = b"\x1b" in data
208 has_tilde = b"~" in data
209 has_plus = b"+" in data
211 if not has_esc and not has_tilde and not has_plus:
212 return None
214 if has_esc:
215 # ISO-2022-JP-2004: JIS X 0213 designations are unique to this variant.
216 if b"\x1b$(O" in data or b"\x1b$(P" in data or b"\x1b$(Q" in data:
217 return DetectionResult(
218 encoding="iso2022_jp_2004",
219 confidence=DETERMINISTIC_CONFIDENCE,
220 language="ja",
221 )
223 # ISO-2022-JP-EXT: JIS X 0201 Kana designation is unique to this variant.
224 if b"\x1b(I" in data:
225 return DetectionResult(
226 encoding="iso2022_jp_ext",
227 confidence=DETERMINISTIC_CONFIDENCE,
228 language="ja",
229 )
231 # ISO-2022-JP base: JIS X 0208/0201/0212 designations.
232 if (
233 b"\x1b$B" in data
234 or b"\x1b$@" in data
235 or b"\x1b(J" in data
236 or b"\x1b$(D" in data # JIS X 0212-1990 (JP-1/JP-2/JP-EXT)
237 ):
238 # SI/SO (0x0E / 0x0F) shift controls -> JP-EXT
239 if b"\x0e" in data and b"\x0f" in data:
240 return DetectionResult(
241 encoding="iso2022_jp_ext",
242 confidence=DETERMINISTIC_CONFIDENCE,
243 language="ja",
244 )
245 # Default to JP-2: a strict superset of JP and JP-1 that
246 # decodes all base sequences correctly.
247 return DetectionResult(
248 encoding="iso2022_jp_2",
249 confidence=DETERMINISTIC_CONFIDENCE,
250 language="ja",
251 )
253 # ISO-2022-KR: ESC sequence for KS C 5601
254 if b"\x1b$)C" in data:
255 return DetectionResult(
256 encoding="iso2022_kr",
257 confidence=DETERMINISTIC_CONFIDENCE,
258 language="ko",
259 )
261 # HZ-GB-2312: tilde escapes for GB2312
262 # Require valid GB2312 byte pairs (0x21-0x7E range) between ~{ and ~} markers.
263 if has_tilde and b"~{" in data and b"~}" in data and _has_valid_hz_regions(data):
264 return DetectionResult(
265 encoding="hz",
266 confidence=DETERMINISTIC_CONFIDENCE,
267 language="zh",
268 )
270 # UTF-7: plus-sign shifts into Base64-encoded Unicode.
271 # UTF-7 is a 7-bit encoding (RFC 2152): every byte must be in 0x00-0x7F.
272 # Data with any byte > 0x7F cannot be UTF-7.
273 if has_plus and max(data) < 0x80 and _has_valid_utf7_sequences(data):
274 return DetectionResult(
275 encoding="utf-7",
276 confidence=DETERMINISTIC_CONFIDENCE,
277 language=None,
278 )
280 return None