Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1a+: UTF-16/UTF-32 detection for data without BOM.
3This stage runs after BOM detection but before binary detection.
4UTF-16 and UTF-32 encoded text contains characteristic null-byte patterns
5that would otherwise cause binary detection to reject the data.
7Note: ``from __future__ import annotations`` is intentionally omitted because
8this module is compiled with mypyc, which does not support PEP 563 string
9annotations.
10"""
12import unicodedata
14from chardet.pipeline import ASCII_TEXT_BYTES, DETERMINISTIC_CONFIDENCE, DetectionResult
16# How many bytes to sample for pattern analysis
17_SAMPLE_SIZE = 4096
19# Minimum bytes needed for reliable pattern detection
20_MIN_BYTES_UTF32 = 16 # 4 full code units
21_MIN_BYTES_UTF16 = 10 # 5 full code units
23# Minimum fraction of null bytes in the expected position for UTF-16.
24# CJK-heavy UTF-16 text (Chinese, Japanese, Korean) can have as few as
25# ~4.5% null bytes in the expected position, since CJK codepoints have
26# non-zero high bytes. The validation step (decode + text quality check)
27# prevents false positives from binary files at this lower threshold.
28_UTF16_MIN_NULL_FRACTION = 0.03
30# Minimum text-quality score to accept a UTF-16 candidate when both
31# endiannesses show null-byte patterns. A score of 0.5 corresponds to
32# roughly 50% letters with no ASCII bonus (or ~40% with whitespace
33# present) — sufficient to distinguish real text from coincidental byte
34# patterns.
35_MIN_TEXT_QUALITY = 0.5
37# Quality margin a byte order with the weaker null signal must win by to
38# override the null pattern's choice. Keeps noisy near-ties (e.g. two
39# alphabetic decodings of similar letter density) on the side the null
40# evidence favors, while letting a clear gap (real CJK vs its byte-swapped
41# scatter across the BMP) flip the answer.
42_QUALITY_TIE_MARGIN = 0.05
44# Minimum fraction of printable characters for a decoded sample to be
45# considered text rather than binary data.
46_MIN_PRINTABLE_FRACTION = 0.7
48# Maximum null fraction (in the candidate null-byte position) below which
49# the data is checked for a null-separator pattern. If the null fraction
50# is below this AND all non-null bytes are printable ASCII, the candidate
51# is rejected as a null-separator false positive rather than real UTF-16.
52# Real Latin UTF-16 has ~50% nulls; CJK UTF-16 has fewer but non-ASCII
53# non-null bytes. 15% is generous — separator data is typically 1-5%.
54_NULL_SEPARATOR_MAX_FRACTION = 0.15
56# ASCII_TEXT_BYTES plus the null byte — used by the null-separator guard
57# to check whether non-null bytes are all printable ASCII.
58_NULL_SEPARATOR_ALLOWED: bytes = b"\x00" + ASCII_TEXT_BYTES
61def _is_null_separator_pattern(data: bytes, null_frac: float) -> bool:
62 """Return True if the data looks like ASCII with null byte separators.
64 :param data: The raw byte sample to examine.
65 :param null_frac: The positional null fraction for this UTF-16 candidate
66 (i.e. fraction of null bytes in even positions for BE, or odd positions
67 for LE) — not the total null fraction across all bytes.
69 Checks two conditions:
70 1. The positional null fraction is below ``_NULL_SEPARATOR_MAX_FRACTION``
71 2. Every non-null byte is printable ASCII or common whitespace
73 When both conditions are met, the nulls are likely field separators
74 (e.g. ``find -print0``), not UTF-16 encoding artifacts.
75 """
76 if null_frac >= _NULL_SEPARATOR_MAX_FRACTION:
77 return False
78 return not data.translate(None, _NULL_SEPARATOR_ALLOWED)
81def detect_utf1632_patterns(data: bytes) -> DetectionResult | None:
82 """Detect UTF-32 or UTF-16 encoding from null-byte patterns.
84 UTF-32 is checked before UTF-16 since UTF-32 patterns are more specific.
86 :param data: The raw byte data to examine.
87 :returns: A :class:`DetectionResult` if a strong pattern is found, or ``None``.
88 """
89 sample = data[:_SAMPLE_SIZE]
91 if len(sample) < _MIN_BYTES_UTF16:
92 return None
94 # Check UTF-32 first (more specific pattern)
95 result = _check_utf32(sample)
96 if result is not None:
97 return result
99 # Then check UTF-16
100 return _check_utf16(sample)
103def _check_utf32(data: bytes) -> DetectionResult | None:
104 """Check for UTF-32 encoding based on 4-byte unit structure.
106 For valid Unicode (U+0000 to U+10FFFF = 0x0010FFFF):
107 - UTF-32-BE: the first byte of each 4-byte unit is always 0x00
108 - UTF-32-LE: the last byte of each 4-byte unit is always 0x00
110 For BMP characters (U+0000 to U+FFFF), additionally:
111 - UTF-32-BE: the second byte is also 0x00
112 - UTF-32-LE: the third byte is also 0x00
113 """
114 # Trim to a multiple of 4 bytes (like _check_utf16 trims to even length)
115 trimmed_len = len(data) - (len(data) % 4)
116 if trimmed_len < _MIN_BYTES_UTF32:
117 return None
118 data = data[:trimmed_len]
120 num_units = trimmed_len // 4
122 # UTF-32-BE: first byte of each 4-byte unit must be 0x00
123 be_first_null = sum(1 for i in range(0, len(data), 4) if data[i] == 0)
124 # Second byte is 0x00 for BMP characters (the vast majority of text)
125 be_second_null = sum(1 for i in range(0, len(data), 4) if data[i + 1] == 0)
127 if be_first_null == num_units and be_second_null / num_units > 0.5:
128 try:
129 text = data.decode("utf-32-be")
130 if _looks_like_text(text):
131 return DetectionResult(
132 encoding="utf-32-be",
133 confidence=DETERMINISTIC_CONFIDENCE,
134 language=None,
135 )
136 except UnicodeDecodeError:
137 pass
139 # UTF-32-LE: last byte of each 4-byte unit must be 0x00
140 le_last_null = sum(1 for i in range(3, len(data), 4) if data[i] == 0)
141 # Third byte is 0x00 for BMP characters
142 le_third_null = sum(1 for i in range(2, len(data), 4) if data[i] == 0)
144 if le_last_null == num_units and le_third_null / num_units > 0.5:
145 try:
146 text = data.decode("utf-32-le")
147 if _looks_like_text(text):
148 return DetectionResult(
149 encoding="utf-32-le",
150 confidence=DETERMINISTIC_CONFIDENCE,
151 language=None,
152 )
153 except UnicodeDecodeError:
154 pass
156 return None
159def _check_utf16(data: bytes) -> DetectionResult | None:
160 """Check for UTF-16 via null-byte patterns in alternating positions.
162 UTF-16 encodes each BMP character as two bytes. For characters whose
163 code-point high byte is 0x00 (Latin, digits, basic punctuation, many
164 control structures), one of the two bytes in each unit will be a null.
165 Even for non-Latin scripts (Arabic, CJK, Cyrillic, etc.) a significant
166 fraction of code units still contain at least one null byte.
168 Non-UTF-16 single-byte encodings never contain null bytes, so even a
169 small null-byte fraction in alternating positions is a strong signal.
171 The null pattern only establishes that the data is UTF-16-like. Byte
172 order is always chosen by decoding both ways and comparing text-quality
173 scores, with the null signal breaking near-ties (see below).
174 """
175 sample_len = min(len(data), _SAMPLE_SIZE)
176 sample_len -= sample_len % 2
177 if sample_len < _MIN_BYTES_UTF16: # pragma: no cover - caller checks length
178 return None
180 num_units = sample_len // 2
182 # Count null bytes in even positions (UTF-16-BE high byte for ASCII)
183 be_null_count = sum(1 for i in range(0, sample_len, 2) if data[i] == 0)
184 # Count null bytes in odd positions (UTF-16-LE high byte for ASCII)
185 le_null_count = sum(1 for i in range(1, sample_len, 2) if data[i] == 0)
187 be_frac = be_null_count / num_units
188 le_frac = le_null_count / num_units
190 le_qualified = (
191 le_frac >= _UTF16_MIN_NULL_FRACTION
192 and not _is_null_separator_pattern(data[:sample_len], le_frac)
193 )
194 be_qualified = (
195 be_frac >= _UTF16_MIN_NULL_FRACTION
196 and not _is_null_separator_pattern(data[:sample_len], be_frac)
197 )
199 if not (le_qualified or be_qualified):
200 return None
202 # The null-byte pattern only establishes that the data is UTF-16-like;
203 # it cannot be trusted to pick the byte order on its own. In pure-CJK
204 # text with no ASCII at all, the only null bytes come from the *low*
205 # byte of characters like U+4E00, which sit in the opposite parity
206 # position and vote for the swapped byte order. Decode both ways and
207 # let text quality decide. Sides are visited in null-signal order and
208 # a challenger must win by a clear margin, so noisy near-ties keep the
209 # answer the null pattern chose.
210 sides = [("utf-16-le", le_frac, le_qualified), ("utf-16-be", be_frac, be_qualified)]
211 if be_frac > le_frac:
212 sides.reverse()
214 best_encoding: str | None = None
215 best_quality = -2.0
216 best_qualified = False
217 viable = 0
218 qualified_side_decoded = False
220 for encoding, _frac, qualified in sides:
221 try:
222 text = data[:sample_len].decode(encoding)
223 except UnicodeDecodeError:
224 continue
225 if qualified:
226 qualified_side_decoded = True
227 if not _looks_like_text(text):
228 continue
229 viable += 1
230 quality = _text_quality(text)
231 if quality > best_quality + (_QUALITY_TIE_MARGIN if viable > 1 else 0.0):
232 best_quality = quality
233 best_encoding = encoding
234 best_qualified = qualified
236 if best_encoding is None:
237 return None
239 if best_qualified:
240 # The null pattern and the quality comparison agree. A sole viable
241 # side keeps the old single-candidate behavior (no quality floor);
242 # a contested choice must look like real text on its own merits.
243 accepted = viable == 1 or best_quality >= _MIN_TEXT_QUALITY
244 else:
245 # A side that failed the null check can win only when a
246 # null-qualified side actually decoded (so the quality comparison
247 # was a fair fight, e.g. real CJK beating its byte-swapped scatter)
248 # and the winner clears the quality floor. If the null-favored
249 # side could not even decode, the data is corrupt in the only byte
250 # order the evidence supports: report nothing rather than the swap.
251 accepted = qualified_side_decoded and best_quality >= _MIN_TEXT_QUALITY
252 if accepted:
253 return DetectionResult(
254 encoding=best_encoding,
255 confidence=DETERMINISTIC_CONFIDENCE,
256 language=None,
257 )
259 return None
262def _looks_like_text(text: str) -> bool:
263 """Quick check: is decoded text mostly printable characters."""
264 if not text:
265 return False
266 sample = text[:500]
267 printable = sum(1 for c in sample if c.isprintable() or c in "\n\r\t")
268 return printable / len(sample) > _MIN_PRINTABLE_FRACTION
271def _text_quality(text: str, limit: int = 500) -> float:
272 """Score how much *text* looks like real human-readable content.
274 Returns a score in the range [-1.0, ~1.6). Higher values indicate
275 more natural text. The practical maximum is 1.5 for all-ASCII-letter
276 input (1.6 approaches as sample size grows with all ASCII letters plus
277 whitespace). A score of -1.0 means the content is almost certainly not
278 valid text (too many control characters or combining marks).
280 Scoring factors:
282 * Base score: ratio of Unicode letters (category ``L*``) to sample length.
283 * ASCII bonus: additional 0.5x weight for ASCII letters. This is the
284 primary signal for disambiguating endianness — correct decoding of
285 Latin-heavy text produces ASCII letters, wrong decoding produces CJK.
286 * Space bonus: +0.1 when the sample contains at least one whitespace
287 character and is longer than 20 characters.
288 * Rejection: returns -1.0 if >10% control characters or >20% combining
289 marks (category ``M*``).
290 """
291 sample = text[:limit]
292 n = len(sample)
293 if n == 0: # pragma: no cover - callers always pass non-empty text
294 return -1.0
296 letters = 0
297 marks = 0
298 spaces = 0
299 controls = 0
300 ascii_letters = 0
302 for c in sample:
303 cat = unicodedata.category(c)
304 if cat[0] == "L":
305 letters += 1
306 if ord(c) < 128:
307 ascii_letters += 1
308 elif cat[0] == "M":
309 marks += 1
310 elif cat == "Zs" or c in "\n\r\t":
311 spaces += 1
312 elif cat[0] == "C":
313 controls += 1
315 # Reject data with many control characters or combining marks
316 if controls / n > 0.1:
317 return -1.0
318 if marks / n > 0.2:
319 return -1.0
321 score = letters / n
322 # ASCII letters strongly indicate correct endianness
323 score += (ascii_letters / n) * 0.5
324 # Real text usually contains some whitespace
325 if n > 20 and spaces > 0:
326 score += 0.1
328 return score