Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1a+: UTF-16/UTF-32 detection for data without BOM.
3This stage runs after BOM detection but before binary detection.
4UTF-16 and UTF-32 encoded text contains characteristic null-byte patterns
5that would otherwise cause binary detection to reject the data.
7Note: ``from __future__ import annotations`` is intentionally omitted because
8this module is compiled with mypyc, which does not support PEP 563 string
9annotations.
10"""
12import codecs
13import unicodedata
15from chardet.pipeline import ASCII_TEXT_BYTES, DETERMINISTIC_CONFIDENCE, DetectionResult
17# How many bytes to sample for pattern analysis
18_SAMPLE_SIZE = 4096
20# Minimum bytes needed for reliable pattern detection
21_MIN_BYTES_UTF32 = 16 # 4 full code units
22_MIN_BYTES_UTF16 = 10 # 5 full code units
24# Minimum fraction of null bytes in the expected position for UTF-16.
25# CJK-heavy UTF-16 text (Chinese, Japanese, Korean) can have as few as
26# ~4.5% null bytes in the expected position, since CJK codepoints have
27# non-zero high bytes. The validation step (decode + text quality check)
28# prevents false positives from binary files at this lower threshold.
29_UTF16_MIN_NULL_FRACTION = 0.03
31# Minimum text-quality score to accept a UTF-16 candidate when both
32# endiannesses show null-byte patterns. A score of 0.5 corresponds to
33# roughly 50% letters with no ASCII bonus (or ~40% with whitespace
34# present) — sufficient to distinguish real text from coincidental byte
35# patterns.
36_MIN_TEXT_QUALITY = 0.5
38# Quality margin a byte order with the weaker null signal must win by to
39# override the null pattern's choice. Keeps noisy near-ties (e.g. two
40# alphabetic decodings of similar letter density) on the side the null
41# evidence favors, while letting a clear gap (real CJK vs its byte-swapped
42# scatter across the BMP) flip the answer.
43_QUALITY_TIE_MARGIN = 0.05
45# Minimum fraction of printable characters for a decoded sample to be
46# considered text rather than binary data.
47_MIN_PRINTABLE_FRACTION = 0.7
49# Maximum null fraction (in the candidate null-byte position) below which
50# the data is checked for a null-separator pattern. If the null fraction
51# is below this AND all non-null bytes are printable ASCII, the candidate
52# is rejected as a null-separator false positive rather than real UTF-16.
53#
54# When every non-null byte is printable ASCII, the two readings are easy
55# to tell apart by how many code units carry a null. UTF-16 text made of
56# ASCII characters puts a null in every unit, and the only thing that
57# lowers the fraction is a stretch that fell out of phase (the corpus has
58# a UTF-16-BE file with one such stretch, at 0.865). Null-separated
59# records (``find -print0``, ``git ls-tree -z``) put a null in at most
60# half the units once records are two characters or longer: three-byte
61# ``abc\0`` records on one parity give exactly 0.5, and longer records
62# give less. Single-character records are byte-identical to UTF-16 and
63# are read as UTF-16. 0.75 sits between the two shapes.
64_NULL_SEPARATOR_MAX_FRACTION = 0.75
66# ASCII_TEXT_BYTES plus the null byte — used by the null-separator guard
67# to check whether non-null bytes are all printable ASCII.
68_NULL_SEPARATOR_ALLOWED: bytes = b"\x00" + ASCII_TEXT_BYTES
71def _is_null_separator_pattern(data: bytes, null_frac: float) -> bool:
72 """Return True if the data looks like ASCII with null byte separators.
74 :param data: The raw byte sample to examine.
75 :param null_frac: The positional null fraction for this UTF-16 candidate
76 (i.e. fraction of null bytes in even positions for BE, or odd positions
77 for LE) — not the total null fraction across all bytes.
79 Checks two conditions:
80 1. The positional null fraction is below ``_NULL_SEPARATOR_MAX_FRACTION``
81 2. Every non-null byte is printable ASCII or common whitespace
83 When both conditions are met, the nulls are likely field separators
84 (e.g. ``find -print0``), not UTF-16 encoding artifacts.
85 """
86 if null_frac >= _NULL_SEPARATOR_MAX_FRACTION:
87 return False
88 return not data.translate(None, _NULL_SEPARATOR_ALLOWED)
91def detect_utf1632_patterns(data: bytes) -> DetectionResult | None:
92 """Detect UTF-32 or UTF-16 encoding from null-byte patterns.
94 UTF-32 is checked before UTF-16 since UTF-32 patterns are more specific.
96 :param data: The raw byte data to examine.
97 :returns: A :class:`DetectionResult` if a strong pattern is found, or ``None``.
98 """
99 sample = data[:_SAMPLE_SIZE]
101 if len(sample) < _MIN_BYTES_UTF16:
102 return None
104 # Check UTF-32 first (more specific pattern)
105 result = _check_utf32(sample)
106 if result is not None:
107 return result
109 # Then check UTF-16
110 return _check_utf16(sample)
113def _check_utf32(data: bytes) -> DetectionResult | None:
114 """Check for UTF-32 encoding based on 4-byte unit structure.
116 For valid Unicode (U+0000 to U+10FFFF = 0x0010FFFF):
117 - UTF-32-BE: the first byte of each 4-byte unit is always 0x00
118 - UTF-32-LE: the last byte of each 4-byte unit is always 0x00
120 For BMP characters (U+0000 to U+FFFF), additionally:
121 - UTF-32-BE: the second byte is also 0x00
122 - UTF-32-LE: the third byte is also 0x00
123 """
124 # Trim to a multiple of 4 bytes (like _check_utf16 trims to even length)
125 trimmed_len = len(data) - (len(data) % 4)
126 if trimmed_len < _MIN_BYTES_UTF32:
127 return None
128 data = data[:trimmed_len]
130 num_units = trimmed_len // 4
132 # UTF-32-BE: first byte of each 4-byte unit must be 0x00
133 be_first_null = sum(1 for i in range(0, len(data), 4) if data[i] == 0)
134 # Second byte is 0x00 for BMP characters (the vast majority of text)
135 be_second_null = sum(1 for i in range(0, len(data), 4) if data[i + 1] == 0)
137 if be_first_null == num_units and be_second_null / num_units > 0.5:
138 try:
139 text = data.decode("utf-32-be")
140 if _looks_like_text(text):
141 return DetectionResult(
142 encoding="utf-32-be",
143 confidence=DETERMINISTIC_CONFIDENCE,
144 language=None,
145 )
146 except UnicodeDecodeError:
147 pass
149 # UTF-32-LE: last byte of each 4-byte unit must be 0x00
150 le_last_null = sum(1 for i in range(3, len(data), 4) if data[i] == 0)
151 # Third byte is 0x00 for BMP characters
152 le_third_null = sum(1 for i in range(2, len(data), 4) if data[i] == 0)
154 if le_last_null == num_units and le_third_null / num_units > 0.5:
155 try:
156 text = data.decode("utf-32-le")
157 if _looks_like_text(text):
158 return DetectionResult(
159 encoding="utf-32-le",
160 confidence=DETERMINISTIC_CONFIDENCE,
161 language=None,
162 )
163 except UnicodeDecodeError:
164 pass
166 return None
169def _check_utf16(data: bytes) -> DetectionResult | None:
170 """Check for UTF-16 via null-byte patterns in alternating positions.
172 UTF-16 encodes each BMP character as two bytes. For characters whose
173 code-point high byte is 0x00 (Latin, digits, basic punctuation, many
174 control structures), one of the two bytes in each unit will be a null.
175 Even for non-Latin scripts (Arabic, CJK, Cyrillic, etc.) a significant
176 fraction of code units still contain at least one null byte.
178 Non-UTF-16 single-byte encodings never contain null bytes, so even a
179 small null-byte fraction in alternating positions is a strong signal.
181 The null pattern only establishes that the data is UTF-16-like. Byte
182 order is always chosen by decoding both ways and comparing text-quality
183 scores, with the null signal breaking near-ties (see below).
184 """
185 sample_len = min(len(data), _SAMPLE_SIZE)
186 sample_len -= sample_len % 2
187 if sample_len < _MIN_BYTES_UTF16: # pragma: no cover - caller checks length
188 return None
190 num_units = sample_len // 2
192 # Count null bytes in even positions (UTF-16-BE high byte for ASCII)
193 be_null_count = sum(1 for i in range(0, sample_len, 2) if data[i] == 0)
194 # Count null bytes in odd positions (UTF-16-LE high byte for ASCII)
195 le_null_count = sum(1 for i in range(1, sample_len, 2) if data[i] == 0)
197 be_frac = be_null_count / num_units
198 le_frac = le_null_count / num_units
200 le_qualified = (
201 le_frac >= _UTF16_MIN_NULL_FRACTION
202 and not _is_null_separator_pattern(data[:sample_len], le_frac)
203 )
204 be_qualified = (
205 be_frac >= _UTF16_MIN_NULL_FRACTION
206 and not _is_null_separator_pattern(data[:sample_len], be_frac)
207 )
209 if not (le_qualified or be_qualified):
210 return None
212 # The null-byte pattern only establishes that the data is UTF-16-like;
213 # it cannot be trusted to pick the byte order on its own. In pure-CJK
214 # text with no ASCII at all, the only null bytes come from the *low*
215 # byte of characters like U+4E00, which sit in the opposite parity
216 # position and vote for the swapped byte order. Decode both ways and
217 # let text quality decide. Sides are visited in null-signal order and
218 # a challenger must win by a clear margin, so noisy near-ties keep the
219 # answer the null pattern chose.
220 sides = [("utf-16-le", le_frac, le_qualified), ("utf-16-be", be_frac, be_qualified)]
221 if be_frac > le_frac:
222 sides.reverse()
224 best_encoding: str | None = None
225 best_quality = -2.0
226 best_qualified = False
227 viable = 0
228 qualified_side_decoded = False
230 for encoding, _frac, qualified in sides:
231 # final=False: a surrogate pair cut at the sample's end is not an error.
232 decoder = codecs.getincrementaldecoder(encoding)()
233 try:
234 text = decoder.decode(data[:sample_len], final=False)
235 except UnicodeDecodeError:
236 continue
237 if qualified:
238 qualified_side_decoded = True
239 if not _looks_like_text(text):
240 continue
241 viable += 1
242 quality = _text_quality(text)
243 if quality > best_quality + (_QUALITY_TIE_MARGIN if viable > 1 else 0.0):
244 best_quality = quality
245 best_encoding = encoding
246 best_qualified = qualified
248 if best_encoding is None:
249 return None
251 if best_qualified:
252 # The null pattern and the quality comparison agree. A sole viable
253 # side keeps the old single-candidate behavior (no quality floor);
254 # a contested choice must look like real text on its own merits.
255 accepted = viable == 1 or best_quality >= _MIN_TEXT_QUALITY
256 else:
257 # A side that failed the null check can win only when a
258 # null-qualified side actually decoded (so the quality comparison
259 # was a fair fight, e.g. real CJK beating its byte-swapped scatter)
260 # and the winner clears the quality floor. If the null-favored
261 # side could not even decode, the data is corrupt in the only byte
262 # order the evidence supports: report nothing rather than the swap.
263 accepted = qualified_side_decoded and best_quality >= _MIN_TEXT_QUALITY
264 if accepted:
265 return DetectionResult(
266 encoding=best_encoding,
267 confidence=DETERMINISTIC_CONFIDENCE,
268 language=None,
269 )
271 return None
274def _looks_like_text(text: str) -> bool:
275 """Quick check: is decoded text mostly printable characters."""
276 if not text:
277 return False
278 sample = text[:500]
279 printable = sum(1 for c in sample if c.isprintable() or c in "\n\r\t")
280 return printable / len(sample) > _MIN_PRINTABLE_FRACTION
283def _text_quality(text: str, limit: int = 500) -> float:
284 """Score how much *text* looks like real human-readable content.
286 Returns a score in the range [-1.0, ~1.6). Higher values indicate
287 more natural text. The practical maximum is 1.5 for all-ASCII-letter
288 input (1.6 approaches as sample size grows with all ASCII letters plus
289 whitespace). A score of -1.0 means the content is almost certainly not
290 valid text (too many control characters or combining marks).
292 Scoring factors:
294 * Base score: ratio of Unicode letters (category ``L*``) to sample length.
295 * ASCII bonus: additional 0.5x weight for ASCII letters. This is the
296 primary signal for disambiguating endianness — correct decoding of
297 Latin-heavy text produces ASCII letters, wrong decoding produces CJK.
298 * Space bonus: +0.1 when the sample contains at least one whitespace
299 character and is longer than 20 characters.
300 * Rejection: returns -1.0 if >10% control characters or >20% combining
301 marks (category ``M*``).
302 """
303 sample = text[:limit]
304 n = len(sample)
305 if n == 0: # pragma: no cover - callers always pass non-empty text
306 return -1.0
308 letters = 0
309 marks = 0
310 spaces = 0
311 controls = 0
312 ascii_letters = 0
314 for c in sample:
315 cat = unicodedata.category(c)
316 if cat[0] == "L":
317 letters += 1
318 if ord(c) < 128:
319 ascii_letters += 1
320 elif cat[0] == "M":
321 marks += 1
322 elif cat == "Zs" or c in "\n\r\t":
323 spaces += 1
324 elif cat[0] == "C":
325 controls += 1
327 # Reject data with many control characters or combining marks
328 if controls / n > 0.1:
329 return -1.0
330 if marks / n > 0.2:
331 return -1.0
333 score = letters / n
334 # ASCII letters strongly indicate correct endianness
335 score += (ascii_letters / n) * 0.5
336 # Real text usually contains some whitespace
337 if n > 20 and spaces > 0:
338 score += 0.1
340 return score