Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Confusion group resolution for similar single-byte encodings.
3At runtime, loads pre-computed distinguishing byte maps from confusion.bin
4and uses them to resolve statistical scoring ties between similar encodings.
6Build-time computation (``compute_confusion_groups``, ``compute_distinguishing_maps``,
7``serialize_confusion_data``) lives in ``scripts/confusion_training.py``.
9Note: ``from __future__ import annotations`` is intentionally omitted because
10this module is compiled with mypyc, which does not support PEP 563 string
11annotations.
12"""
14import functools
15import importlib.resources
16import struct
17import unicodedata
18import warnings
20from chardet.models import (
21 ART_LANGUAGE,
22 BigramProfile,
23 get_enc_index,
24 get_idf_weights,
25 score_with_profile,
26)
27from chardet.pipeline import DetectionResult
28from chardet.registry import lookup_encoding
30# Type alias for the distinguishing map structure:
31# Maps (enc_a, enc_b) -> (distinguishing_byte_set, {byte_val: (cat_a, cat_b)})
32DistinguishingMaps = dict[
33 tuple[str, str],
34 tuple[frozenset[int], dict[int, tuple[str, str]]],
35]
37# uint8 -> Unicode general category, inverse of the mapping in
38# scripts/confusion_training.py used at serialization time.
39_INT_TO_CATEGORY: dict[int, str] = {
40 0: "Lu",
41 1: "Ll",
42 2: "Lt",
43 3: "Lm",
44 4: "Lo",
45 5: "Mn",
46 6: "Mc",
47 7: "Me",
48 8: "Nd",
49 9: "Nl",
50 10: "No",
51 11: "Pc",
52 12: "Pd",
53 13: "Ps",
54 14: "Pe",
55 15: "Pi",
56 16: "Pf",
57 17: "Po",
58 18: "Sm",
59 19: "Sc",
60 20: "Sk",
61 21: "So",
62 22: "Zs",
63 23: "Zl",
64 24: "Zp",
65 25: "Cc",
66 26: "Cf",
67 27: "Cs",
68 28: "Co",
69 29: "Cn",
70}
72#: Inverse mapping for serialization. Public because the build side
73#: (``scripts/confusion_training.py``) must encode categories with the
74#: exact map this module decodes them with.
75CATEGORY_TO_INT: dict[str, int] = {v: k for k, v in _INT_TO_CATEGORY.items()}
78def deserialize_confusion_data_from_bytes(data: bytes) -> DistinguishingMaps:
79 """Load confusion group data from raw bytes.
81 :param data: The raw binary content of a confusion.bin file.
82 :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs.
83 """
84 result: DistinguishingMaps = {}
85 offset = 0
86 (num_pairs,) = struct.unpack_from("!H", data, offset)
87 offset += 2
89 for _ in range(num_pairs):
90 (name_a_len,) = struct.unpack_from("!B", data, offset)
91 offset += 1
92 name_a = data[offset : offset + name_a_len].decode("utf-8")
93 offset += name_a_len
95 (name_b_len,) = struct.unpack_from("!B", data, offset)
96 offset += 1
97 name_b = data[offset : offset + name_b_len].decode("utf-8")
98 offset += name_b_len
100 (num_diffs,) = struct.unpack_from("!B", data, offset)
101 offset += 1
103 diff_bytes_list: list[int] = []
104 categories: dict[int, tuple[str, str]] = {}
105 for _ in range(num_diffs):
106 bv, cat_a_int, cat_b_int = struct.unpack_from("!BBB", data, offset)
107 offset += 3
108 diff_bytes_list.append(bv)
109 categories[bv] = (
110 _INT_TO_CATEGORY.get(cat_a_int, "Cn"),
111 _INT_TO_CATEGORY.get(cat_b_int, "Cn"),
112 )
113 result[(name_a, name_b)] = (frozenset(diff_bytes_list), categories)
115 return result
118@functools.cache
119def load_confusion_data() -> DistinguishingMaps:
120 """Load confusion group data from the bundled confusion.bin file.
122 :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs.
123 """
124 ref = importlib.resources.files("chardet.models").joinpath("confusion.bin")
125 raw = ref.read_bytes()
126 if not raw:
127 warnings.warn(
128 "chardet confusion.bin is empty — confusion resolution disabled; "
129 "reinstall chardet to fix",
130 RuntimeWarning,
131 stacklevel=2,
132 )
133 return {}
134 try:
135 raw_maps = deserialize_confusion_data_from_bytes(raw)
136 except (struct.error, UnicodeDecodeError) as e:
137 msg = f"corrupt confusion.bin: {e}"
138 raise ValueError(msg) from e
139 # Normalize keys to canonical codec names so pipeline output matches.
140 normalized: DistinguishingMaps = {}
141 for (a, b), value in raw_maps.items():
142 norm_a = lookup_encoding(a) or a
143 norm_b = lookup_encoding(b) or b
144 normalized[(norm_a, norm_b)] = value
145 return normalized
148# Unicode general category preference scores for voting resolution.
149# Higher scores indicate more linguistically meaningful characters.
150_CATEGORY_PREFERENCE: dict[str, int] = {
151 "Lu": 10,
152 "Ll": 10,
153 "Lt": 10,
154 "Lm": 9,
155 "Lo": 9,
156 "Nd": 8,
157 "Nl": 7,
158 "No": 7,
159 "Pc": 6,
160 "Pd": 6,
161 "Ps": 6,
162 "Pe": 6,
163 "Pi": 6,
164 "Pf": 6,
165 "Po": 6,
166 "Sc": 5,
167 "Sm": 5,
168 "Sk": 4,
169 "So": 4,
170 "Zs": 3,
171 "Zl": 3,
172 "Zp": 3,
173 "Cf": 2,
174 "Cc": 1,
175 "Co": 1,
176 "Cs": 0,
177 "Cn": 0,
178 "Mn": 5,
179 "Mc": 5,
180 "Me": 5,
181}
184# Preference assigned to a letter reading whose context makes it an
185# implausible word member — below every punctuation and symbol category.
186_IMPLAUSIBLE_LETTER_PREFERENCE = 2
188# Vote margin at which category voting overrides the bigram rescore. Two
189# context-decisive occurrences (letter-vs-punctuation with the word-shape
190# rule fired: 2 x (6 - 2)) clear it; a lone punctuation-vs-punctuation
191# reading (margin 1) never does. Raising this threshold is not safe: the
192# EBCDIC record suite depends on a decisive margin of 12 (three
193# occurrences) to hold off the rescore's max-over-variants bias.
194_DECISIVE_VOTE_MARGIN = 8
196# Minimum number of distinct demotion-earning occurrences for a vote to be
197# decisive. A single occurrence can reach margin 8 on its own (a
198# plausible-letter reading at preference 10 against an implausible-letter
199# reading demoted to 2), and one byte of context must never outrank the
200# rescore's model evidence.
201_DECISIVE_MIN_EVENTS = 2
203# Cap on distinguishing-byte occurrences examined per pair. Sparse by
204# nature; the cap only bounds pathological inputs.
205_MAX_VOTE_OCCURRENCES = 256
207# Density at which the focused-profile scan stops paying off. Below one
208# distinguishing byte per this many input bytes, locating the hits with a
209# C-level scan beats walking every byte in Python; above it, the set of
210# start indices costs more than the straight loop it replaces. Only 62 of
211# 2,170 rescore calls over the test corpus are that dense.
212_DENSE_HIT_DIVISOR = 4
215@functools.cache
216def _letter_case_table(encoding: str) -> bytes:
217 """256-entry table: 0 = non-letter, 1 = uppercase letter, 2 = other letter.
219 Combining marks count as letters: in decomposed text (Vietnamese under
220 windows-1258) a base letter's neighbor is its diacritic, which is
221 word-internal, not a word boundary. Whitespace deliberately counts as
222 a plain non-letter: exempting space-adjacent letters from the
223 isolated-letter demotion (to spare one-letter words like French ``à``)
224 was tried and falsified by the accuracy suite — Irish/Finnish po files
225 and the EBCDIC record set depend on space-adjacent demotions, so the
226 cross-family decisive-override gate handles the ``à`` failure mode
227 instead.
228 """
229 table = bytearray(256)
230 for b in range(256):
231 try:
232 ch = bytes([b]).decode(encoding)
233 except UnicodeDecodeError:
234 continue
235 # Stateful codecs can decode a byte to zero characters (utf-7's
236 # ``+`` opens a base64 run and yields ``""``), and category()
237 # rejects anything but a single character.
238 if len(ch) != 1:
239 continue
240 cat = unicodedata.category(ch)
241 if cat == "Lu":
242 table[b] = 1
243 elif cat[0] == "L" or cat in ("Mn", "Mc"):
244 table[b] = 2
245 return bytes(table)
248def _context_preference(cat: str, left: int, right: int, case_table: bytes) -> int:
249 """Preference for reading a byte as *cat*, adjusted for word shape.
251 A letter reading only deserves its high preference when its neighbors
252 make it look like part of a word under the same encoding: a letter with
253 no letter neighbors is quoted/isolated punctuation in disguise, and a
254 lowercase letter immediately followed by an uppercase one is not a word
255 shape any of the supported languages produce.
256 """
257 pref = _CATEGORY_PREFERENCE.get(cat, 0)
258 if cat[0] != "L":
259 return pref
260 left_kind = case_table[left]
261 right_kind = case_table[right]
262 if left_kind == 0 and right_kind == 0:
263 return _IMPLAUSIBLE_LETTER_PREFERENCE
264 if cat == "Ll" and right_kind == 1:
265 return _IMPLAUSIBLE_LETTER_PREFERENCE
266 return pref
269def _vote_with_margin(
270 data: bytes,
271 enc_a: str,
272 enc_b: str,
273 diff_bytes: frozenset[int],
274 categories: dict[int, tuple[str, str]],
275) -> tuple[str | None, int, int, int]:
276 """Context-aware category voting.
278 Returns ``(winner, margin, demotion_margin, demotion_events)``.
280 For each occurrence of a distinguishing byte, compare the two
281 encodings' readings: Unicode category preference, adjusted for word
282 shape (see :func:`_context_preference`). The reading that makes more
283 linguistic sense of the byte *in its context* collects the vote;
284 occurrences vote independently, so repeated evidence counts.
286 ``demotion_margin`` counts only the winner's votes earned where the
287 *losing* side's letter reading was word-shape-implausible — evidence
288 against an impossible reading, which is far stronger than the naive
289 letters-beat-symbols preference. ``demotion_events`` counts how many
290 distinct occurrences contributed to it, so callers can tell repeated
291 evidence from one loud byte.
292 """
293 # Delete every non-distinguishing byte value in one C-level translate
294 # pass; what survives is exactly the distinguishing bytes present.
295 # Equivalent to ``frozenset(data) & diff_bytes`` but ~5x faster, since
296 # that would hash every byte of the (up to max_bytes) input.
297 non_diff, _ = _pair_byte_tables(diff_bytes)
298 relevant = frozenset(data.translate(None, non_diff))
299 if not relevant:
300 return None, 0, 0, 0
301 table_a = _letter_case_table(enc_a)
302 table_b = _letter_case_table(enc_b)
303 votes_a = 0
304 votes_b = 0
305 demotion_a = 0
306 demotion_b = 0
307 events_a = 0
308 events_b = 0
309 end = len(data) - 1
310 for bv in relevant:
311 cat_a, cat_b = categories[bv]
312 needle = bytes((bv,))
313 pos = data.find(needle)
314 examined = 0
315 while pos >= 0 and examined < _MAX_VOTE_OCCURRENCES:
316 left = data[pos - 1] if pos > 0 else 0
317 right = data[pos + 1] if pos < end else 0
318 pref_a = _context_preference(cat_a, left, right, table_a)
319 pref_b = _context_preference(cat_b, left, right, table_b)
320 # A letter reading beating a *punctuation* reading on naive
321 # preference alone is not evidence: punctuation of every
322 # category legitimately borders letters (delimiters, hyphens,
323 # brackets, apostrophes), so the letter interpretation is
324 # never the only plausible one. A letter beating a *symbol*
325 # reading still counts — box-drawing or dingbats inside a
326 # word is not a shape prose produces.
327 if pref_a > pref_b:
328 if cat_a[0] == "L" and cat_b[0] == "P":
329 pass
330 else:
331 votes_a += pref_a - pref_b
332 if cat_b[0] == "L" and pref_b == _IMPLAUSIBLE_LETTER_PREFERENCE:
333 demotion_a += pref_a - pref_b
334 events_a += 1
335 elif pref_b > pref_a:
336 if cat_b[0] == "L" and cat_a[0] == "P":
337 pass
338 else:
339 votes_b += pref_b - pref_a
340 if cat_a[0] == "L" and pref_a == _IMPLAUSIBLE_LETTER_PREFERENCE:
341 demotion_b += pref_b - pref_a
342 events_b += 1
343 examined += 1
344 pos = data.find(needle, pos + 1)
345 if votes_a > votes_b:
346 return enc_a, votes_a - votes_b, demotion_a, events_a
347 if votes_b > votes_a:
348 return enc_b, votes_b - votes_a, demotion_b, events_b
349 return None, 0, 0, 0
352def confusion_pair_winner(
353 data: bytes,
354 enc_x: str,
355 enc_y: str,
356 languages: frozenset[str] = frozenset(),
357) -> str | None:
358 """Return the byte-evidence winner between two encodings, or ``None``.
360 Mirrors the in-band pairwise rule of :func:`resolve_confusion_groups`
361 (decisive demotion vote, else bigram rescore, else category vote) for
362 callers outside the ranked-results scan — e.g. the classic-Mac
363 line-ending promotion, whose platform prior must not override
364 distinguishing-byte evidence. Returns ``None`` when the pair has no
365 distinguishing map or the evidence is inconclusive.
367 *languages* must carry what the two results being compared report, or
368 the mirror breaks: the rescore would arbitrate the same pair under a
369 different rule than the confusion stage just did, and a veto built on
370 that answer can reverse a promotion the stage had settled.
371 """
372 maps = load_confusion_data()
373 pair_key = _find_pair_key(maps, enc_x, enc_y)
374 if pair_key is None:
375 return None
376 diff_bytes, categories = maps[pair_key]
377 enc_a, enc_b = pair_key
378 cat_winner, _margin, demotion_margin, demotion_events = _vote_with_margin(
379 data, enc_a, enc_b, diff_bytes, categories
380 )
381 if (
382 cat_winner is not None
383 and demotion_margin >= _DECISIVE_VOTE_MARGIN
384 and demotion_events >= _DECISIVE_MIN_EVENTS
385 and len(diff_bytes) < _CROSS_FAMILY_MIN_DIFFS
386 ):
387 return cat_winner
388 bigram_winner = resolve_by_bigram_rescore(data, enc_a, enc_b, diff_bytes, languages)
389 if len(diff_bytes) >= _CROSS_FAMILY_MIN_DIFFS:
390 # Cross-family pairs: corroboration required (see the strict rule
391 # in resolve_confusion_groups).
392 if bigram_winner is not None and bigram_winner == cat_winner:
393 return bigram_winner
394 return None
395 return bigram_winner if bigram_winner is not None else cat_winner
398def resolve_by_category_voting(
399 data: bytes,
400 enc_a: str,
401 enc_b: str,
402 diff_bytes: frozenset[int],
403 categories: dict[int, tuple[str, str]],
404) -> str | None:
405 """Resolve between two encodings using context-aware category voting.
407 :returns: The winning encoding name, or ``None`` if tied.
408 """
409 winner, _margin, _demotion, _events = _vote_with_margin(
410 data, enc_a, enc_b, diff_bytes, categories
411 )
412 return winner
415@functools.cache
416def _pair_byte_tables(diff_bytes: frozenset[int]) -> tuple[bytes, bytes]:
417 """Return ``(non_diff_delete, membership)`` byte tables for a pair.
419 ``non_diff_delete`` holds every byte value *not* in *diff_bytes* (for
420 ``bytes.translate`` deletion) and ``membership`` is a 256-entry table
421 with 1 at each distinguishing byte (native indexing under mypyc, where
422 frozenset probes are boxed). Cached because *diff_bytes* comes from
423 the fixed per-pair confusion maps loaded once per process.
424 """
425 member = bytearray(256)
426 for b in diff_bytes:
427 member[b] = 1
428 non_diff = bytes(b for b in range(256) if not member[b])
429 return non_diff, bytes(member)
432#: Pseudo-language for models trained on data with no linguistic content
433#: (ANSI art / box drawing). Not a language, so language-fairness rules
434#: do not apply to it. Canonically defined beside the model tables.
435_ART_LANGUAGE = ART_LANGUAGE
438@functools.cache
439def _modelled_languages(enc: str) -> frozenset[str]:
440 """Languages *enc* has a bigram model for."""
441 return frozenset(
442 lang for lang, _, _ in get_enc_index().get(enc, []) if lang is not None
443 )
446def _comparable_languages(
447 enc_a: str,
448 enc_b: str,
449 languages: frozenset[str],
450) -> frozenset[str] | None:
451 """Languages to score *enc_a* and *enc_b* under, or ``None`` for all.
453 ``None`` means *unrestricted* — score every variant, the original
454 max-over-models comparison. It is not an abstention.
456 An encoding should not win on language coverage the other side lacks.
457 On a Hungarian document the u-double-acute bytes score 0.014 against
458 iso8859-2's *Czech* model (which reads them as a common r-hacek) and
459 only 0.006 against iso8859-16's Hungarian one, so max-over-variants
460 hands Hungarian text to the Czech reading. Dropping the languages
461 only one side models removes that particular unfairness.
463 This is deliberately a narrow rule, and two tempting generalisations
464 were measured and rejected against the accuracy suite:
466 * Restricting to *languages* themselves rather than the shared set
467 costs 12 tests. As a consequence the restriction is a no-op for
468 pairs whose language coverage already matches (67 of the 236
469 confusion pairs), which is accepted — those pairs have no coverage
470 asymmetry to correct in the first place.
471 * Restricting when *languages* is not wholly inside the shared set
472 costs 6 tests: cp1125 models only Ukrainian, so a Belarusian cp866
473 document shares just ``uk`` with it, and scoring the *right*
474 encoding under the *wrong* language loses to cp1125. Falling back
475 to unrestricted is also what keeps a Vietnamese windows-1258
476 document, whose rival cp1252 models no Vietnamese, resolving
477 correctly. The cost is that pairs modelling disjoint languages
478 (koi8-r/koi8-u, mac-roman/mac-turkish) never restrict at all —
479 about 20% of calls over the corpus.
480 """
481 if not languages:
482 return None
483 langs_a = _modelled_languages(enc_a)
484 langs_b = _modelled_languages(enc_b)
485 shared = langs_a & langs_b
486 if not languages <= shared:
487 return None
488 # The art pseudo-language is not a language, so the fairness argument
489 # above does not reach it: cp437 is the only encoding carrying a zxx
490 # model, which means a plain intersection would strip box-drawing
491 # evidence from every restricted rescore it takes part in. Keeping it
492 # for whichever side has it preserves the art protection the module
493 # maintains elsewhere (see the zxx guard in resolve_confusion_groups).
494 if _ART_LANGUAGE in langs_a or _ART_LANGUAGE in langs_b:
495 return shared | {_ART_LANGUAGE}
496 return shared
499def _best_variant_score(
500 profile: BigramProfile,
501 index: dict[str, list[tuple[str | None, bytes, str]]],
502 enc: str,
503 languages: frozenset[str] | None,
504) -> float:
505 """Return the best bigram score for *enc*, restricted to *languages*.
507 *languages* of ``None`` means every variant, the unrestricted
508 max-over-models comparison. The ``default`` guards a caller-supplied
509 set that names no variant of *enc*; note that scoring 0.0 hands the
510 comparison to the rival rather than abstaining, so callers wanting an
511 abstention must not rely on it.
512 """
513 variants = index.get(enc)
514 if not variants:
515 return 0.0
516 return max(
517 (
518 score_with_profile(profile, model, model_key)
519 for lang, model, model_key in variants
520 if languages is None or lang in languages
521 ),
522 default=0.0,
523 )
526def build_focused_profile(
527 data: bytes, diff_bytes: frozenset[int]
528) -> BigramProfile | None:
529 """Build the bigram profile of *data* restricted to *diff_bytes* context.
531 The profile holds only bigrams where at least one byte is in
532 *diff_bytes*, weighted by IDF like the full-input profile, so scoring a
533 model against it asks how well that model explains the bytes two
534 encodings read differently, and nothing else. Returns ``None`` when
535 *data* cannot form a bigram or contains no distinguishing byte.
537 :param data: The raw byte data to examine.
538 :param diff_bytes: Byte values where the two encodings differ.
539 """
540 if len(data) < 2:
541 return None
543 # C-level prefilter: if no distinguishing byte occurs anywhere, the
544 # focused profile below would be empty — skip the per-byte loop.
545 # Deleting the *non*-distinguishing bytes leaves a result that is tiny
546 # (usually empty) rather than a near-full copy of the input.
547 non_diff, is_diff = _pair_byte_tables(diff_bytes)
548 hits = len(data.translate(None, non_diff))
549 if not hits:
550 return None
552 idf = get_idf_weights()
553 freq: dict[int, int] = {}
554 limit = len(data) - 1
555 if hits * _DENSE_HIT_DIVISOR < len(data):
556 # Sparse case, which is nearly all of them: locate the hits with
557 # bytes.find rather than walking every byte. Each hit at position
558 # p belongs to the bigrams starting at p-1 and p; collecting start
559 # indices counts a bigram whose bytes are both distinguishing once,
560 # exactly as the dense scan below does.
561 starts: set[int] = set()
562 for bv in frozenset(data.translate(None, non_diff)):
563 needle = bytes((bv,))
564 pos = data.find(needle)
565 while pos >= 0:
566 if pos:
567 starts.add(pos - 1)
568 if pos < limit:
569 starts.add(pos)
570 pos = data.find(needle, pos + 1)
571 for i in starts:
572 idx = (data[i] << 8) | data[i + 1]
573 freq[idx] = freq.get(idx, 0) + idf[idx]
574 else:
575 for i in range(limit):
576 b1 = data[i]
577 b2 = data[i + 1]
578 if not (is_diff[b1] | is_diff[b2]):
579 continue
580 idx = (b1 << 8) | b2
581 freq[idx] = freq.get(idx, 0) + idf[idx]
583 # Unreachable: the hit prefilter above already returned when no
584 # distinguishing byte occurs, and with len(data) >= 2 every hit byte
585 # belongs to at least one bigram, so both scan paths put a key in
586 # ``freq`` (a zero IDF weight still creates the key). Kept so a future
587 # scan-path change cannot hand an empty profile to the scorer.
588 if not freq: # pragma: no cover
589 return None
591 return BigramProfile.from_weighted_freq(freq)
594def resolve_by_bigram_rescore(
595 data: bytes,
596 enc_a: str,
597 enc_b: str,
598 diff_bytes: frozenset[int],
599 languages: frozenset[str] = frozenset(),
600) -> str | None:
601 """Resolve between two encodings by re-scoring only distinguishing bigrams.
603 Builds a focused bigram profile containing only bigrams where at least one
604 byte is a distinguishing byte, then scores both encodings under the
605 languages they can be compared in (see :func:`_comparable_languages`).
607 There is no abstention path here: when the pair has no comparable
608 language the comparison widens to every variant rather than declining
609 to answer. That is why a Danish mac-roman/mac-turkish document, whose
610 pair models disjoint languages, is still decided by the Turkish model
611 the rescore has no business consulting — the caller's category vote is
612 better placed on such evidence, and overriding this to abstain was
613 measured as costing more accuracy than it recovers.
615 :param data: The raw byte data to examine.
616 :param enc_a: First encoding name.
617 :param enc_b: Second encoding name.
618 :param diff_bytes: Byte values where the two encodings differ.
619 :param languages: Languages the ranked results report for this pair;
620 empty when the caller has no ranking to draw on, which scores
621 every variant.
622 :returns: The winning encoding name, or ``None`` if tied or if no
623 distinguishing byte occurs in *data*.
624 """
625 profile = build_focused_profile(data, diff_bytes)
626 if profile is None:
627 return None
629 comparable = _comparable_languages(enc_a, enc_b, languages)
631 index = get_enc_index()
632 best_a = _best_variant_score(profile, index, enc_a, comparable)
633 best_b = _best_variant_score(profile, index, enc_b, comparable)
635 if best_a > best_b:
636 return enc_a
637 if best_b > best_a:
638 return enc_b
639 return None
642@functools.cache
643def differing_high_bytes(enc_a: str, enc_b: str) -> frozenset[int]:
644 """Byte values >= 0x80 that *enc_a* and *enc_b* decode to different text.
646 The distinguishing set for a pair the confusion maps do not cover,
647 computed from the codecs themselves. A byte only one side can decode
648 counts as differing. Bytes below 0x80 are left out: the callers ask
649 about high-byte evidence, and every single-byte Latin family agrees
650 on ASCII anyway.
651 """
652 out: set[int] = set()
653 for b in range(0x80, 0x100):
654 raw = bytes((b,))
655 try:
656 text_a: str | None = raw.decode(enc_a)
657 except UnicodeDecodeError:
658 text_a = None
659 try:
660 text_b: str | None = raw.decode(enc_b)
661 except UnicodeDecodeError:
662 text_b = None
663 if text_a != text_b:
664 out.add(b)
665 return frozenset(out)
668@functools.cache
669def _pair_categories(
670 enc_a: str, enc_b: str, diff_bytes: frozenset[int]
671) -> dict[int, tuple[str, str]]:
672 """Unicode general categories of each distinguishing byte under both encodings.
674 The confusion maps carry this table for their own pairs; callers that
675 arbitrate a pair the maps do not cover (the niche-Latin demotion's
676 candidate against its swap target) build it here. A byte one side
677 cannot decode reads as unassigned (``Cn``), which the vote treats as
678 the least plausible reading of all.
679 """
680 table: dict[int, tuple[str, str]] = {}
681 for b in diff_bytes:
682 cats = []
683 for enc in (enc_a, enc_b):
684 try:
685 ch = bytes([b]).decode(enc)
686 except UnicodeDecodeError:
687 cats.append("Cn")
688 continue
689 cats.append(unicodedata.category(ch) if len(ch) == 1 else "Cn")
690 table[b] = (cats[0], cats[1])
691 return table
694def arbitrate_distinguishing_bytes( # noqa: PLR0913
695 data: bytes,
696 enc_a: str,
697 enc_b: str,
698 diff_bytes: frozenset[int],
699 *,
700 languages_a: frozenset[str] | None,
701 languages_b: frozenset[str] | None,
702) -> str | None:
703 """Decide a pair on its distinguishing bytes: model rescore, then context.
705 The in-band rule of :func:`resolve_confusion_groups` for a pair the
706 confusion maps do not cover, with one difference: each side is scored
707 under the languages the caller names for it rather than the shared
708 set. The bigram rescore decides when the models have an opinion; when
709 they score the distinguishing bigrams equally (usually both at zero,
710 a byte neither model has seen in that context), the category vote
711 reads word shape instead, so a letter between letters still beats a
712 superscript between letters. Returns ``None`` when neither step can
713 tell the two apart.
715 :param data: The raw byte data to examine.
716 :param enc_a: First encoding name.
717 :param enc_b: Second encoding name.
718 :param diff_bytes: Byte values where the two encodings differ.
719 :param languages_a: Variants of *enc_a* to score; ``None`` for all.
720 :param languages_b: Variants of *enc_b* to score; ``None`` for all.
721 """
722 profile = build_focused_profile(data, diff_bytes)
723 if profile is not None:
724 index = get_enc_index()
725 best_a = _best_variant_score(profile, index, enc_a, languages_a)
726 best_b = _best_variant_score(profile, index, enc_b, languages_b)
727 if best_a > best_b:
728 return enc_a
729 if best_b > best_a:
730 return enc_b
731 winner, _margin, _demotion, _events = _vote_with_margin(
732 data, enc_a, enc_b, diff_bytes, _pair_categories(enc_a, enc_b, diff_bytes)
733 )
734 return winner
737def _find_pair_key(
738 maps: DistinguishingMaps,
739 enc_a: str,
740 enc_b: str,
741) -> tuple[str, str] | None:
742 """Find the canonical key for a pair of encodings in the confusion maps."""
743 if (enc_a, enc_b) in maps:
744 return (enc_a, enc_b)
745 if (enc_b, enc_a) in maps:
746 return (enc_b, enc_a)
747 return None
750# Pairs whose distinguishing set is at least this large come from the
751# cross-family tier of the pair generator (byte-similar siblings differ at
752# most at 51 positions under its 0.80 similarity floor). Cross-family
753# pairs arbitrate wholesale-different byte tables, where the rescore alone
754# is a coin flip whenever the distinguishing evidence in the data is
755# sparse — so these pairs require vote/rescore corroboration even for
756# in-band near-ties.
757_CROSS_FAMILY_MIN_DIFFS = 52
759#: Maximum confidence gap from the top result for candidates beyond
760#: position 1 to participate in confusion resolution. Public because it
761#: is a contract fact: the pruning contract in
762#: :mod:`chardet.pipeline.postprocess` composes it into the floor that
763#: statistical pruning must score exactly.
764CONFUSION_BAND = 0.005
766#: Minimum confidence, as a fraction of the top result's, for out-of-band
767#: candidates to participate in the strict tier of confusion resolution.
768#: Confusion siblings can score far apart in absolute terms while the
769#: statistical ranking among them is still noise (EBCDIC record data), so
770#: the strict tier extends beyond the band — but only for challengers with
771#: corroborated evidence (vote and bigram agreement, or a decisive
772#: demotion-driven vote). Public: a contract fact, see
773#: :data:`CONFUSION_BAND`.
774CONFUSION_FLOOR_RATIO = 0.5
776#: The strict tier only opens when the top confidence is below this value.
777#: A low absolute confidence means no model explains the data, so the
778#: ranking among confusion siblings is noise and corroborated byte-level
779#: evidence may overturn it. A confident top means the statistics are
780#: working; overriding them from far down the ranking does more harm than
781#: good (correlated vote/rescore errors across the many near-scoring
782#: Latin encodings). Public: a contract fact, see :data:`CONFUSION_BAND`.
783STRICT_TIER_MAX_CONF = 0.2
786def resolve_confusion_groups(
787 data: bytes,
788 results: list[DetectionResult],
789) -> list[DetectionResult]:
790 """Resolve confusion between similar encodings in the top results.
792 Checks the top result against each candidate within a confidence band.
793 Always checks position 1 (preserving original top-2 behavior); for
794 positions 2+ only checks within the band. Uses bigram re-scoring
795 with category voting as fallback.
797 :param data: The raw byte data to examine.
798 :param results: Detection results sorted by confidence descending.
799 :returns: A reordered list of :class:`DetectionResult` with the winner first.
800 """
801 if len(results) < 2:
802 return results
804 top = results[0]
805 if top.encoding is None:
806 return results
807 # An art-model win (the zxx pseudo-language: no linguistic content) is
808 # not up for linguistic-plausibility review — voting and rescoring both
809 # reason about prose, which box-drawing data is not. Narrowing this to
810 # rescore-only review was tried and rejected: under era filtering a
811 # prose sibling can tie the art model exactly (cp850-en vs cp437-zxx on
812 # a real artpack file) and the flickery diff-focused rescore then
813 # dethrones genuine art, while the case the review would rescue —
814 # box-drawing strong enough to outrank dominant prose statistically,
815 # yet weaker than it in the diff-byte rescore — is a knife-edge regime
816 # the statistical ranking already resolves whenever prose dominates.
817 if top.language == _ART_LANGUAGE:
818 return results
820 maps = load_confusion_data()
821 top_conf = top.confidence
822 floor = top_conf * CONFUSION_FLOOR_RATIO
824 champion_idx = 0
825 champion = top
826 # Tracked alongside ``champion`` as a narrowed ``str``. Rebinding
827 # ``champion`` on promotion widens ``.encoding`` back to ``str | None``
828 # for the type checker, and both values it can hold — ``top`` and a
829 # promoted ``candidate`` — are None-checked before they get here.
830 champion_enc = top.encoding
831 for i in range(1, len(results)):
832 candidate = results[i]
833 if candidate.encoding is None:
834 continue
835 # Position 1 and band members use the original in-band rules;
836 # candidates between the band and the floor enter the strict tier,
837 # which only opens when the statistics have failed outright.
838 in_band = i == 1 or top_conf - candidate.confidence <= CONFUSION_BAND
839 if not in_band and (
840 top_conf >= STRICT_TIER_MAX_CONF or candidate.confidence < floor
841 ):
842 break
844 pair_key = _find_pair_key(maps, champion_enc, candidate.encoding)
845 if pair_key is None:
846 continue
848 diff_bytes, categories = maps[pair_key]
849 enc_a, enc_b = pair_key
851 cat_winner, _vote_margin, demotion_margin, demotion_events = _vote_with_margin(
852 data, enc_a, enc_b, diff_bytes, categories
853 )
854 # A demotion-driven vote outranks the bigram rescore: those votes
855 # were earned where the opposing reading was a word-shape-impossible
856 # letter (lowercase jammed between digits or capitals), which is
857 # stronger evidence than the rescore's prose-typicality priors. A
858 # vote won on the naive letters-beat-symbols preference defers
859 # to the rescore's model evidence. Decisiveness requires repeated
860 # evidence: one occurrence can reach the margin on its own, and a
861 # single byte of context must never outrank the models.
862 winner: str | None
863 # The decisive-demotion override exists because *sibling* models
864 # are too similar for the rescore to arbitrate — a premise that
865 # only holds within-family. Cross-family models differ wholesale,
866 # so there the rescore is at its most informative and the vote's
867 # linguistic priors at their least reliable (a French ``à`` read
868 # as a footnote dagger collects huge demotion margins): cross-
869 # family pairs always require corroboration instead.
870 if (
871 cat_winner is not None
872 and demotion_margin >= _DECISIVE_VOTE_MARGIN
873 and demotion_events >= _DECISIVE_MIN_EVENTS
874 and len(diff_bytes) < _CROSS_FAMILY_MIN_DIFFS
875 ):
876 winner = cat_winner
877 else:
878 # When both results read the document as the same language,
879 # the rescore compares them in it rather than under whichever
880 # language happens to like the distinguishing bytes most.
881 langs = frozenset(
882 lang
883 for lang in (champion.language, candidate.language)
884 if lang is not None
885 )
886 bigram_winner = resolve_by_bigram_rescore(
887 data, enc_a, enc_b, diff_bytes, langs
888 )
889 if in_band and len(diff_bytes) < _CROSS_FAMILY_MIN_DIFFS:
890 winner = bigram_winner if bigram_winner is not None else cat_winner
891 # Strict rule (out-of-band candidates, and cross-family pairs
892 # even in-band): overturning the ranking needs corroboration —
893 # the vote and the rescore must agree. (The decisive-demotion
894 # case is handled above.)
895 elif bigram_winner is not None and bigram_winner == cat_winner:
896 winner = bigram_winner
897 else:
898 winner = None
900 if winner is None or winner != candidate.encoding:
901 continue
902 if in_band:
903 # In-band promotion: trust it and stop, preserving the
904 # original single-promotion behavior for near-ties.
905 promoted = DetectionResult(
906 candidate.encoding,
907 top_conf,
908 candidate.language,
909 candidate.mime_type,
910 )
911 rest = [r for j, r in enumerate(results) if j != i]
912 return [promoted, *rest]
913 # Strict-tier promotion: the new champion must defend against
914 # the remaining candidates (king-of-the-hill), because the
915 # correct member of a clique may rank below another sibling
916 # that also beats the current champion. Known limitation: the
917 # scan is single-pass, so a higher-ranked candidate skipped
918 # earlier for lack of a pair with the then-champion is never
919 # revisited against the new one — accepted, since the original
920 # top-anchored code could not arbitrate those either.
921 champion_idx = i
922 champion = candidate
923 champion_enc = candidate.encoding
925 if champion_idx == 0:
926 return results
928 # Give the promoted candidate the top result's confidence so the
929 # promotion survives any downstream confidence-based sort.
930 promoted = DetectionResult(
931 champion.encoding,
932 top_conf,
933 champion.language,
934 champion.mime_type,
935 )
936 rest = [r for j, r in enumerate(results) if j != champion_idx]
937 return [promoted, *rest]