Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 13: post-processing rank corrections.
3After statistical scoring produces a ranked list of candidates, a chain
4of rank corrections fixes up the ranking when bigrams alone are
5insufficient — see :func:`postprocess_results` for the order. The steps:
6dead-heat priors (superset preference, era prevalence), rare-language
7arbitration (ADR-0005), confusion-group resolution (delegated to
8:mod:`chardet.pipeline.confusion`), niche Latin demotion, KOI8-T
9promotion, classic-Mac line-ending promotion, and last of all the
10decode-safety flip, which hands a winner whose only multi-byte evidence
11is an undecodable trailing sequence to the best rival that can decode
12the caller's complete input.
14Note: ``from __future__ import annotations`` is intentionally omitted because
15this module is compiled with mypyc, which does not support PEP 563 string
16annotations.
17"""
19from chardet._utils import (
20 dangling_tail_with_ascii_prefix,
21 decodes_without_error,
22)
23from chardet.models import ART_LANGUAGE, RARE_LANGUAGES, get_enc_index
24from chardet.output_names import decodes_under_public_name
25from chardet.pipeline import DetectionResult
26from chardet.pipeline.confusion import (
27 CONFUSION_BAND,
28 CONFUSION_FLOOR_RATIO,
29 STRICT_TIER_MAX_CONF,
30 _comparable_languages,
31 arbitrate_distinguishing_bytes,
32 confusion_pair_winner,
33 differing_high_bytes,
34 resolve_confusion_groups,
35)
36from chardet.registry import CJK_SUPERSETS, REGISTRY
38# Common Western Latin encodings that share the iso-8859-1 character
39# repertoire for the byte values where iso-8859-10 is indistinguishable.
40# Used as swap targets when demoting iso-8859-10 — we prefer these over
41# iso-8859-10, but do not want to accidentally promote an unrelated encoding
42# (e.g. windows-1254).
43_COMMON_LATIN_ENCODINGS: frozenset[str] = frozenset(
44 {
45 "iso8859-1",
46 "iso8859-15",
47 "cp1252",
48 }
49)
51# Bytes where iso-8859-10 decodes to a different character than iso-8859-1.
52# Computed programmatically via:
53# {b for b in range(0x80, 0x100)
54# if bytes([b]).decode('iso-8859-10') != bytes([b]).decode('iso-8859-1')}
55_ISO_8859_10_DISTINGUISHING: frozenset[int] = frozenset(
56 {
57 0xA1,
58 0xA2,
59 0xA3,
60 0xA4,
61 0xA5,
62 0xA6,
63 0xA8,
64 0xA9,
65 0xAA,
66 0xAB,
67 0xAC,
68 0xAE,
69 0xAF,
70 0xB1,
71 0xB2,
72 0xB3,
73 0xB4,
74 0xB5,
75 0xB6,
76 0xB8,
77 0xB9,
78 0xBA,
79 0xBB,
80 0xBC,
81 0xBD,
82 0xBE,
83 0xBF,
84 0xC0,
85 0xC7,
86 0xC8,
87 0xCA,
88 0xCC,
89 0xD1,
90 0xD2,
91 0xD7,
92 0xD9,
93 0xE0,
94 0xE7,
95 0xE8,
96 0xEA,
97 0xEC,
98 0xF1,
99 0xF2,
100 0xF7,
101 0xF9,
102 0xFF,
103 }
104)
106# Bytes where iso-8859-14 decodes to a different character than iso-8859-1.
107# Computed programmatically via:
108# {b for b in range(0x80, 0x100)
109# if bytes([b]).decode('iso-8859-14') != bytes([b]).decode('iso-8859-1')}
110_ISO_8859_14_DISTINGUISHING: frozenset[int] = frozenset(
111 {
112 0xA1,
113 0xA2,
114 0xA4,
115 0xA5,
116 0xA6,
117 0xA8,
118 0xAA,
119 0xAB,
120 0xAC,
121 0xAF,
122 0xB0,
123 0xB1,
124 0xB2,
125 0xB3,
126 0xB4,
127 0xB5,
128 0xB7,
129 0xB8,
130 0xB9,
131 0xBA,
132 0xBB,
133 0xBC,
134 0xBD,
135 0xBE,
136 0xBF,
137 0xD0,
138 0xD7,
139 0xDE,
140 0xF0,
141 0xF7,
142 0xFE,
143 }
144)
146# Bytes where windows-1254 has Turkish-specific characters that differ from
147# windows-1252. Windows-1254 differs from windows-1252 at 8 byte positions.
148# Two (0x8E, 0x9E) are undefined in Windows-1254 but defined in Windows-1252;
149# these are excluded here because undefined bytes are not useful for
150# identifying Turkish text. The remaining six positions map to
151# Turkish-specific letters and are the primary distinguishing signal.
152_WINDOWS_1254_DISTINGUISHING: frozenset[int] = frozenset(
153 {0xD0, 0xDD, 0xDE, 0xF0, 0xFD, 0xFE}
154)
156# Bytes where HP-Roman8 maps to lowercase accented letters but ISO-8859-1
157# maps to uppercase letters. Real HP-Roman8 text (from HP-UX terminals)
158# contains these bytes; data misdetected as HP-Roman8 typically does not.
159# {b for b in range(0x80, 0x100)
160# if (unicodedata.category(bytes([b]).decode('hp-roman8')) == 'Ll'
161# and unicodedata.category(bytes([b]).decode('iso-8859-1')) == 'Lu')}
162_HP_ROMAN8_DISTINGUISHING: frozenset[int] = frozenset(
163 {
164 0xC0,
165 0xC1,
166 0xC2,
167 0xC3,
168 0xC4,
169 0xC5,
170 0xC6,
171 0xC7,
172 0xC8,
173 0xC9,
174 0xCA,
175 0xCB,
176 0xCC,
177 0xCD,
178 0xCE,
179 0xCF,
180 0xD1,
181 0xD4,
182 0xD5,
183 0xD6,
184 0xD9,
185 0xDD,
186 0xDE,
187 }
188)
190# Encodings that are often false positives when their distinguishing bytes
191# are absent. Keyed by encoding name -> frozenset of byte values where
192# that encoding differs from iso-8859-1 (or windows-1252 in the case of
193# windows-1254).
194_DEMOTION_CANDIDATES: dict[str, frozenset[int]] = {
195 "iso8859-10": _ISO_8859_10_DISTINGUISHING,
196 "iso8859-14": _ISO_8859_14_DISTINGUISHING,
197 "cp1254": _WINDOWS_1254_DISTINGUISHING,
198 "hp-roman8": _HP_ROMAN8_DISTINGUISHING,
199}
201# Bytes where KOI8-T maps to Tajik-specific Cyrillic letters but KOI8-R
202# maps to box-drawing characters. Presence of any of these bytes is strong
203# evidence for KOI8-T over KOI8-R.
204_KOI8_T_DISTINGUISHING: frozenset[int] = frozenset(
205 {0x80, 0x81, 0x83, 0x8A, 0x8C, 0x8D, 0x8E, 0x90, 0xA1, 0xA2, 0xA5, 0xB5}
206)
209# Deletion tables for bytes.translate: length changes iff a distinguishing
210# byte occurs in the data. A translate scan runs at C speed where the
211# equivalent generator expression iterates the whole (up to max_bytes) input
212# with boxed set-membership tests. All distinguishing bytes are > 0x7F, so
213# the old per-byte high-bit filter is subsumed by set membership.
214_DEMOTION_DELETE: dict[str, bytes] = {
215 enc: bytes(byte_set) for enc, byte_set in _DEMOTION_CANDIDATES.items()
216}
217_KOI8_T_DELETE: bytes = bytes(_KOI8_T_DISTINGUISHING)
220def _should_demote(data: bytes, top: DetectionResult, target: DetectionResult) -> bool:
221 """Return True if *top*, a demotion candidate, has no byte evidence over *target*.
223 Callers guarantee ``top.encoding`` is in :data:`_DEMOTION_CANDIDATES`.
224 Two questions, cheapest first. Does *data* contain any byte the
225 candidate decodes differently from ISO-8859-1? If not, the data is
226 equally valid under both encodings, nothing at the byte level favors
227 the candidate, and it is demoted.
229 If such bytes are present, do they favor the candidate? Presence
230 alone is symmetric evidence: a Windows-1252 file whose only non-ASCII
231 letter is an ``Ö`` carries 0xD6, which HP-Roman8 reads as ``ø``, so
232 both candidates "contain" the byte and the question is which reading
233 holds up. That is a confusion-style arbitration between the candidate
234 and its swap target on the distinguishing bytes alone (see
235 :func:`~chardet.pipeline.confusion.arbitrate_distinguishing_bytes`),
236 each side scored under the variant that actually won its slot. The
237 models decide when they can: one Welsh ``ŵ`` keeps ISO-8859-14 because
238 the Welsh model knows that bigram and the Windows-1252 reading is a
239 ``ð`` no Welsh model has seen. When the models are silent, word shape
240 decides: a Kven ``đ`` keeps a Finnish file on ISO-8859-10 because a
241 letter between letters beats the superscript ``¹`` Windows-1252 reads
242 there. A lone ``Ö`` mid-word decides nothing either way, and the more
243 prevalent encoding takes the evidence-free tie.
245 The arbitration is only asked when the candidate's lead over the swap
246 target is within :data:`~chardet.pipeline.confusion.CONFUSION_BAND`.
247 A win by more than the band was decided on the full statistics, and
248 re-litigating it on a handful of bytes is neither sound nor free: the
249 scan is Python level, and mainstream Turkish text tops as windows-1254
250 with hundreds of distinguishing bytes and a lead of 0.2.
251 """
252 encoding = top.encoding or ""
253 if len(data.translate(None, _DEMOTION_DELETE[encoding])) == len(data):
254 return True
255 if top.confidence - target.confidence > CONFUSION_BAND:
256 return False
257 winner = arbitrate_distinguishing_bytes(
258 data,
259 encoding,
260 target.encoding or "",
261 _DEMOTION_CANDIDATES[encoding],
262 languages_a=None if top.language is None else frozenset((top.language,)),
263 languages_b=None if target.language is None else frozenset((target.language,)),
264 )
265 return winner != encoding
268def _swap_target(candidates: list[DetectionResult]) -> DetectionResult:
269 """Pick the common Latin candidate that replaces a demoted top.
271 Among the candidates within :data:`_DEAD_HEAT_EPSILON` of the
272 highest-scoring one, era prevalence chooses (windows-1252 over
273 iso-8859-1): inside that band the confidence order is noise, the very
274 premise of the demotion. A candidate trailing the best common Latin
275 by more than the epsilon lost to it on real evidence and stays put.
276 Equal era ranks (iso-8859-1 against iso-8859-15, both legacy ISO) keep
277 confidence order, since ``min`` returns the first of equals and the
278 candidates arrive ranked.
279 """
280 lead_conf = max(r.confidence for r in candidates)
281 in_band = [r for r in candidates if lead_conf - r.confidence <= _DEAD_HEAT_EPSILON]
282 return min(in_band, key=lambda r: _era_rank(r.encoding or ""))
285def _demote_niche_latin(
286 data: bytes,
287 results: list[DetectionResult],
288) -> list[DetectionResult]:
289 """Demote a niche Latin top that its distinguishing bytes do not support.
291 Some bigram models (iso-8859-10, iso-8859-14, windows-1254, hp-roman8)
292 can win on data that contains only bytes shared with the common Western
293 Latin encodings, or on a lone shared byte the models cannot arbitrate.
294 When :func:`_should_demote` finds no byte-level evidence for the
295 winning encoding, promote the swap target :func:`_swap_target` picks
296 among the common Latin candidates and push the demoted encoding to
297 last.
299 The demoted entries take the confidence of the candidate they now sit
300 behind. Rank position alone does not survive the trip out to callers:
301 ``detect_all`` re-sorts by confidence, and a stable sort hands an entry
302 that kept the top score its old place back.
304 :param data: The raw byte data the results were produced from.
305 :param results: A list of :class:`DetectionResult` ranked by confidence.
306 :returns: A new list (or the same list) with the demotion applied.
307 """
308 if len(results) < 2 or results[0].encoding not in _DEMOTION_CANDIDATES:
309 return results
310 candidates = [r for r in results[1:] if r.encoding in _COMMON_LATIN_ENCODINGS]
311 if not candidates:
312 return results
313 target = _swap_target(candidates)
314 if not _should_demote(data, results[0], target):
315 return results
316 demoted_encoding = results[0].encoding
317 top_conf = results[0].confidence
318 promoted = DetectionResult(
319 target.encoding, top_conf, target.language, target.mime_type
320 )
321 others = [x for x in results if x.encoding != demoted_encoding and x is not target]
322 tail_conf = others[-1].confidence if others else top_conf
323 demoted_entries = [
324 DetectionResult(
325 x.encoding,
326 min(x.confidence, tail_conf),
327 x.language,
328 x.mime_type,
329 )
330 for x in results
331 if x.encoding == demoted_encoding
332 ]
333 return [promoted, *others, *demoted_entries]
336def _promote_koi8t(
337 data: bytes,
338 results: list[DetectionResult],
339) -> list[DetectionResult]:
340 """Promote KOI8-T over KOI8-R when Tajik-specific bytes are present.
342 KOI8-T and KOI8-R share the entire 0xC0-0xFF Cyrillic letter block,
343 making statistical discrimination difficult. However, KOI8-T maps 12
344 bytes in 0x80-0xBF to Tajik-specific Cyrillic letters where KOI8-R has
345 box-drawing characters. If any of these bytes appear, KOI8-T is the
346 better match.
347 """
348 if not results or results[0].encoding != "koi8-r":
349 return results
350 # Check if KOI8-T is anywhere in the results
351 koi8t_idx = next((i for i, r in enumerate(results) if r.encoding == "koi8-t"), None)
352 if koi8t_idx is None:
353 return results
354 # Check for Tajik-specific bytes
355 if len(data.translate(None, _KOI8_T_DELETE)) != len(data):
356 return _promote_to_top(results, koi8t_idx)
357 return results
360# Confidence gap below which two candidates are a statistical dead heat:
361# their scores differ only through model-norm noise on bigrams that carry no
362# real evidence (observed dead heats sit within ~1e-5; genuinely decided
363# rankings lead by >= 2e-3).
364_DEAD_HEAT_EPSILON = 1e-4
366# On a dead heat between an encoding and its Windows superset, prefer the
367# superset: it decodes everything the base encoding does, so it is never a
368# worse answer when the statistics cannot separate them. The pairs are
369# the registry's ``CJK_SUPERSETS``, shared with ``markup``'s promotion:
370# reaching the ISO/Windows pairs would be an accuracy change to measure on
371# its own (ADR-0007).
373# Confidence band for the classic-Mac line-ending promotion. Wider than the
374# dead-heat epsilon because bare-\r line endings are decisive platform
375# evidence, not just a prior. Structurally the confusion band: retuning
376# ``CONFUSION_BAND`` carries this promotion's reach with it, keeping the
377# band inside ``_CORRECTION_REACH`` so pruning always scores what it scans.
378_CR_MAC_BAND = CONFUSION_BAND
380# Minimum number of \r line endings before the classic-Mac promotion fires.
381_CR_MAC_MIN_LINES = 3
383# EncodingEra.LEGACY_MAC — value inlined to avoid importing the enum into
384# this mypyc-compiled hot path for a single constant.
385_LEGACY_MAC_ERA = 4
388# Cap on the data scanned for high-byte bigram evidence — matches the
389# window statistical scoring uses (``orchestrator._STAT_SCORE_MAX_BYTES``),
390# so the evidence check sees the same bytes the scores were computed from.
391_EVIDENCE_SCAN_MAX_BYTES = 16384
394def _era_rank(encoding: str) -> int:
395 """Return the lowest era bit for *encoding* (lower = more prevalent today)."""
396 info = REGISTRY.get(encoding)
397 if info is None:
398 return 1 << 30
399 era = int(info.era)
400 return era & -era
403def _has_high_byte_evidence(data: bytes, encoding: str, language: "str | None") -> bool:
404 """Return True if *encoding*'s winning model weights a high-byte bigram present in *data*.
406 A candidate whose model assigns zero weight to every non-ASCII bigram in
407 the data earned its statistical score purely from ASCII bigrams — noise
408 that cannot distinguish encodings. Only the variant that actually won
409 (*language*) counts: another language's variant having weight for those
410 bytes says nothing about why *this* result is on top. Only called on
411 dead heats, so the Python-level scan of the (capped) data is off the
412 hot path.
413 """
414 variants = get_enc_index().get(encoding)
415 if not variants:
416 return False
417 window = data[:_EVIDENCE_SCAN_MAX_BYTES]
418 seen: set[int] = set()
419 prev = window[0]
420 for i in range(1, len(window)):
421 b = window[i]
422 if prev >= 0x80 or b >= 0x80:
423 seen.add((prev << 8) | b)
424 prev = b
425 if not seen:
426 return False
427 for lang, table, _key in variants:
428 if language is not None and lang != language:
429 continue
430 for idx in seen:
431 if table[idx]:
432 return True
433 return False
436def _prefer_prevalent_on_dead_heat(
437 data: bytes,
438 results: list[DetectionResult],
439) -> list[DetectionResult]:
440 """Break statistical dead heats in favor of the more prevalent era.
442 When several encodings score within :data:`_DEAD_HEAT_EPSILON` of the
443 top result, the ranking among them is mostly an artifact of
444 ASCII-bigram noise. Promote the candidate from the most prevalent era
445 (modern web > legacy ISO > Mac > regional > DOS > mainframe) so
446 evidence-free dead heats resolve to the likeliest real-world answer.
448 A top result whose models carry no weight for any high-byte bigram in
449 the data has no evidence at all and yields outright. One whose models
450 do weight an observed bigram is not thereby safe: an English file with
451 one capital ``É`` ranks MacRoman first because the MacRoman model
452 reads 0xC9 as the ellipsis English text is full of, a lead worth
453 2e-5. Such a top is arbitrated against the prevalent candidate on the
454 bytes the two read differently, under the languages the two can be
455 compared in (see :func:`~chardet.pipeline.confusion.arbitrate_distinguishing_bytes`
456 and confusion's ``_comparable_languages``): the Windows-1252 French
457 model knows ``École`` even when the English one does not, while its
458 Icelandic model may not read a Welsh ``dŵr`` as ``dðr`` against an
459 encoding that models no Icelandic. The prevalent candidate is
460 promoted only when it wins outright; a tie keeps the top, so an
461 ISO-8859-1 result tied with Windows-1252 on data without C1 bytes
462 stays where the statistics put it. Genuine MacRoman text never
463 reaches the arbitration, since its hundreds of distinguishing bytes
464 put Windows-1252 far outside the band.
465 """
466 top = results[0] if results else None
467 if top is None or top.encoding is None or len(results) < 2:
468 return results
469 best_idx = 0
470 best_rank = _era_rank(top.encoding)
471 for i in range(1, len(results)):
472 r = results[i]
473 if r.encoding is None:
474 continue
475 if top.confidence - r.confidence > _DEAD_HEAT_EPSILON:
476 break
477 rank = _era_rank(r.encoding)
478 if rank < best_rank:
479 best_rank = rank
480 best_idx = i
481 if best_idx == 0:
482 return results
483 if not _has_high_byte_evidence(data, top.encoding, top.language):
484 return _promote_to_top(results, best_idx)
485 rival = results[best_idx].encoding or ""
486 comparable = _comparable_languages(
487 top.encoding,
488 rival,
489 frozenset(
490 lang
491 for lang in (top.language, results[best_idx].language)
492 if lang is not None
493 ),
494 )
495 winner = arbitrate_distinguishing_bytes(
496 data,
497 top.encoding,
498 rival,
499 differing_high_bytes(top.encoding, rival),
500 languages_a=comparable,
501 languages_b=comparable,
502 )
503 if winner == rival:
504 return _promote_to_top(results, best_idx)
505 return results
508def _promote_to_top(results: list[DetectionResult], i: int) -> list[DetectionResult]:
509 """Move ``results[i]`` to the top, carrying the current top confidence."""
510 r = results[i]
511 promoted = DetectionResult(
512 r.encoding, results[0].confidence, r.language, r.mime_type
513 )
514 rest = [x for j, x in enumerate(results) if j != i]
515 return [promoted, *rest]
518def _promote_superset_on_dead_heat(
519 data: bytes,
520 results: list[DetectionResult],
521) -> list[DetectionResult]:
522 """Promote a Windows superset over its base encoding on a dead heat."""
523 top = results[0] if results else None
524 if top is None or top.encoding is None or len(results) < 2:
525 return results
526 superset = CJK_SUPERSETS.get(top.encoding)
527 if superset is None:
528 return results
529 for i in range(1, len(results)):
530 r = results[i]
531 if top.confidence - r.confidence > _DEAD_HEAT_EPSILON:
532 break
533 if r.encoding == superset and decodes_without_error(data, superset):
534 return _promote_to_top(results, i)
535 return results
538# Languages whose (language, encoding) variants never accumulated a
539# measurable legacy document population. Not a judgment about the
540# languages — a graded prior about legacy-era *bytes*: iso8859-14 (Latin-8,
541# the Celtic code page) was standardized in 1998 but Celtic text lived in
542# latin-1 and moved to UTF-8; this project's wild-page mining has found no
543# native specimen, and web surveys place the encoding at noise level. The
544# one documented genuine niche — Irish gettext .po catalogues that declare
545# ISO-8859-14 (Scannell's vim/gettext translations, in the test suite) —
546# measures safely outside the arbitration gate: genuine Celtic text has no
547# prevalent-language rival anywhere near it. Revision protocol per
548# ADR-0005: any new genuine specimen goes into test-data and forces a
549# re-audit of the set. The set itself is :data:`chardet.models.RARE_LANGUAGES`
550# — one definition, used by this gate and by the language fill's thin-margin
551# band, so the two can never drift apart. Membership changes happen there.
553# Maximum lead over the best prevalent-language candidate for a
554# rare-language winner to count as a coin flip rather than evidence.
555_RARE_ARBITRATION_MARGIN = 0.02
557# Maximum absolute confidence for arbitration to apply: genuine
558# rare-language text scores well above this even when short, so the gate
559# only opens in the evidence-free zone.
560_RARE_ARBITRATION_MAX_CONFIDENCE = 0.15
563def _arbitrate_rare_language(
564 results: list[DetectionResult],
565) -> list[DetectionResult]:
566 """Demote a rare-language winner that leads a prevalent rival by a coin flip.
568 Fires only when the winner's language is in
569 :data:`~chardet.models.RARE_LANGUAGES`, its absolute confidence is
570 inside the evidence-free zone, and a prevalent-language candidate sits
571 within :data:`_RARE_ARBITRATION_MARGIN`. Genuine rare-language text fails
572 both gates: even short files score confidently, and their entire
573 neighborhood is same-language variants.
574 """
575 top = results[0] if results else None
576 if (
577 top is None
578 or top.encoding is None
579 or top.language not in RARE_LANGUAGES
580 or top.confidence >= _RARE_ARBITRATION_MAX_CONFIDENCE
581 or len(results) < 2
582 ):
583 return results
584 for i in range(1, len(results)):
585 r = results[i]
586 if top.confidence - r.confidence > _RARE_ARBITRATION_MARGIN:
587 break
588 if r.encoding is None or r.language is None:
589 continue
590 if r.language not in RARE_LANGUAGES:
591 return _promote_to_top(results, i)
592 return results
595def _promote_mac_on_cr_line_endings(
596 data: bytes,
597 results: list[DetectionResult],
598) -> list[DetectionResult]:
599 r"""Promote a classic-Mac candidate when line endings are bare ``\r``.
601 Classic Mac OS is the only platform that terminated lines with a lone
602 carriage return, so data with several ``\r`` bytes and no ``\n`` is
603 near-certainly Mac-era text. When a LEGACY_MAC candidate scores within
604 :data:`_CR_MAC_BAND` of a non-Mac top result, promote it — unless the
605 pair has a distinguishing-byte map and the byte-level evidence says the
606 current top wins: a platform prior must not overturn direct evidence
607 that confusion resolution may have just used to establish the top.
608 """
609 top = results[0] if results else None
610 if top is None or top.encoding is None or len(results) < 2:
611 return results
612 if _era_rank(top.encoding) == _LEGACY_MAC_ERA:
613 return results
614 # An art-model win is not up for prose-based review: the pairwise
615 # veto below reasons about word shapes and prose bigrams, which
616 # box-drawing data is not, and old ANSI art legitimately carries
617 # bare-CR line endings.
618 if top.language == ART_LANGUAGE:
619 return results
620 if data.find(b"\n") >= 0 or data.count(b"\r") < _CR_MAC_MIN_LINES:
621 return results
622 for i in range(1, len(results)):
623 r = results[i]
624 if top.confidence - r.confidence > _CR_MAC_BAND:
625 break
626 if r.encoding is not None and _era_rank(r.encoding) == _LEGACY_MAC_ERA:
627 # Pass both languages so the veto arbitrates this pair under
628 # the same rule confusion resolution just applied to it.
629 langs = frozenset(
630 lang for lang in (top.language, r.language) if lang is not None
631 )
632 if (
633 confusion_pair_winner(data, top.encoding, r.encoding, langs)
634 == top.encoding
635 ):
636 # Byte-level evidence says the top beats the best-ranked
637 # Mac candidate: stop entirely rather than letting a
638 # lower-ranked sibling take the promotion just because it
639 # has no distinguishing-byte map to be checked against.
640 break
641 return _promote_to_top(results, i)
642 return results
645def _prefer_decodable_on_tie(
646 data: bytes,
647 results: list[DetectionResult],
648 *,
649 input_truncated: bool,
650) -> list[DetectionResult]:
651 """Promote a strictly decoding rival over a winner with no real evidence.
653 Byte-validity filtering runs incremental decoders with ``final=False``,
654 tolerating an incomplete multi-byte sequence at the end because detection
655 input is often a prefix of a larger whole. When chardet examined the
656 caller's *entire* input, that tolerance can hand back an encoding the
657 caller's very next ``data.decode()`` will reject --- a four-byte
658 ``iso-8859-1`` word ending in ``0xE1`` detected as utf-8 (issue #380).
660 Fires only when *data* is the whole of what the caller handed over ---
661 *input_truncated* is False --- the tail can actually hold a
662 dangling sequence (a high byte in the final four, multi-byte winner),
663 and the winner's tolerant decode is **non-empty pure ASCII** --- its
664 only multi-byte evidence is the dangling tail itself. An empty
665 tolerant decode (the whole input is one clipped sequence) is zero
666 evidence, not ASCII evidence, and disqualifies the flip. The
667 best-ranked rival that decodes the input completely under both its
668 internal and public names (``compat_names=True``, the default, can
669 remap to a strictly narrower codec: ``euc_jis_2004`` is reported as
670 ``EUC-JP``) then takes the top slot, regardless of the
671 confidence gap: an all-ASCII-evidence winner detected nothing the
672 rival did not also detect, and any statistical lead it holds comes
673 from ASCII bigrams the rival matched equally well. The scan sees the
674 ranking as given, which under ``full_ranking=False`` is pruned; if no
675 listed rival decodes, the winner stands (measured across a 648-case
676 accent-final sweep, the pruned ranking always carried a decodable
677 rival).
679 The pure-ASCII condition is what makes the unconditional flip safe. A
680 short mid-character CJK cut has a correct answer that cannot decode the
681 input --- flipping it to whichever single-byte codec happens to decode
682 the bytes trades a right answer for a wrong one, and a 5-40 byte sweep
683 measured exactly that under a gap-based rule (34 correct CJK answers
684 lost, Big5 becoming cp1125). Such a winner has decoded real multi-byte
685 characters and keeps its ranking; with the pure-ASCII condition in
686 place, the same sweep measures zero lost answers at any gap.
687 """
688 top = results[0] if results else None
689 if (
690 input_truncated
691 or top is None
692 or top.encoding is None
693 or len(results) < 2
694 # A dangling multi-byte tail needs a high byte among the final
695 # bytes (empty data trivially has none). No cheaper winner gate
696 # exists: the registry's is_multibyte_cjk means CJK-style structural
697 # multibyte and is False for utf-8, the main deferring codec. The
698 # helper below is a single tolerant decode; validity already ran
699 # the same decode once per candidate, so this adds at most one
700 # more, and only for high-byte-tailed winners.
701 or not any(b >= 0x80 for b in data[-4:])
702 or not dangling_tail_with_ascii_prefix(data, top.encoding)
703 ):
704 return results
705 for i in range(1, len(results)):
706 r = results[i]
707 if r.encoding is None or not decodes_under_public_name(
708 data, r.encoding, complete=True
709 ):
710 continue
711 return _promote_to_top(results, i)
712 return results
715# ---------------------------------------------------------------------------
716# The pruning contract: what statistical pruning must score exactly
717# ---------------------------------------------------------------------------
719#: How far below the running second-best score a candidate can sit and still
720#: be examined by a rank correction: rare-language arbitration reads margins
721#: up to ``_RARE_ARBITRATION_MARGIN`` from the top, and confusion resolution
722#: examines the band, kept with a 2x cushion for float noise.
723_CORRECTION_REACH = _RARE_ARBITRATION_MARGIN + 2 * CONFUSION_BAND
726def scoring_floor(top1: float, top2: float) -> float:
727 """Return the score below which the rank corrections cannot examine a candidate.
729 One half of the pruning contract statistical scoring consumes: given
730 the running top two encoding scores, every candidate at or above this
731 floor must carry its exact full-ranking score, or a correction could
732 fire on an understated (or missing) entry and ``detect()`` would
733 diverge from the unpruned full ranking. The floor trails the
734 second-best score by the corrections' reach; while the top is low
735 enough for confusion resolution's strict tier to open, it extends down
736 to that tier's floor, because a strict-tier promotion may raise any
737 candidate above the tier floor into position 0 before the other
738 corrections evaluate their triggers.
740 One correction sits deliberately outside this floor: the decode-safety
741 flip (:func:`_prefer_decodable_on_tie`) scans the ranking *as given*,
742 including pruned-path tails whose entries may be understated or absent
743 — a measured trade-off documented in its own docstring, not a floor
744 violation to fix by widening the reach.
745 """
746 floor = top2 - _CORRECTION_REACH
747 if top1 < STRICT_TIER_MAX_CONF:
748 floor = min(floor, top1 * CONFUSION_FLOOR_RATIO)
749 return floor
752def forced_encodings(near_top: list[str]) -> list[str]:
753 """Return the encodings the corrections look up by name, given the near-top set.
755 The other half of the pruning contract: :func:`postprocess_results`
756 inspects some encodings wherever they rank (the common Western Latin
757 trio for niche Latin demotion, KOI8-T for the KOI8-R promotion), so
758 pruning must score every variant of these whenever a trigger encoding
759 sits at or above the :func:`scoring_floor` — confusion resolution may
760 promote any such candidate to the top before those triggers are
761 evaluated.
762 """
763 forced: list[str] = []
764 if any(e in _DEMOTION_CANDIDATES for e in near_top):
765 forced.extend(_COMMON_LATIN_ENCODINGS)
766 if "koi8-r" in near_top:
767 forced.append("koi8-t")
768 return forced
771def postprocess_results(
772 data: bytes,
773 results: list[DetectionResult],
774 *,
775 input_truncated: bool = False,
776) -> list[DetectionResult]:
777 """Apply rank corrections to the statistically scored results.
779 Steps run in sequence, weakest evidence first: dead-heat priors
780 (superset preference, era prevalence), then confusion-group resolution,
781 niche Latin demotion, and KOI8-T promotion (byte-level evidence), and
782 finally the classic-Mac line-ending promotion (platform evidence that
783 should override the priors). The decode-safety tiebreak runs last of
784 all: whatever the ranking settled on, a winner that cannot decode the
785 caller's complete input, and whose own evidence is nothing but the
786 undecodable tail, yields to the best-ranked rival that can decode it.
788 :param data: The raw byte data the results were produced from.
789 :param results: A list of :class:`DetectionResult` ranked by confidence.
790 :param input_truncated: True when *data* is a chardet-made slice rather
791 than the caller's whole input --- the ``max_bytes`` slice, the
792 evidence-cap slice, or ``UniversalDetector``'s buffer cap. Any of
793 them means the bytes here are not the whole story, so the
794 decode-safety tiebreak, which reasons about what the caller will
795 decode, stands down.
796 :returns: A new list (or the same list) with rank corrections applied.
797 """
798 results = _promote_superset_on_dead_heat(data, results)
799 results = _prefer_prevalent_on_dead_heat(data, results)
800 results = _arbitrate_rare_language(results)
801 results = resolve_confusion_groups(data, results)
802 results = _demote_niche_latin(data, results)
803 results = _promote_koi8t(data, results)
804 results = _promote_mac_on_cr_line_endings(data, results)
805 return _prefer_decodable_on_tie(data, results, input_truncated=input_truncated)