Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 13: post-processing rank corrections.
3After statistical scoring produces a ranked list of candidates, a chain
4of rank corrections fixes up the ranking when bigrams alone are
5insufficient — see :func:`postprocess_results` for the order. The steps:
6dead-heat priors (superset preference, era prevalence), rare-language
7arbitration (ADR-0005), confusion-group resolution (delegated to
8:mod:`chardet.pipeline.confusion`), niche Latin demotion, KOI8-T
9promotion, classic-Mac line-ending promotion, and last of all the
10decode-safety flip, which hands a winner whose only multi-byte evidence
11is an undecodable trailing sequence to the best rival that can decode
12the caller's complete input.
14Note: ``from __future__ import annotations`` is intentionally omitted because
15this module is compiled with mypyc, which does not support PEP 563 string
16annotations.
17"""
19from chardet._utils import (
20 dangling_tail_with_ascii_prefix,
21 decodes_completely,
22 decodes_without_error,
23)
24from chardet.models import ART_LANGUAGE, RARE_LANGUAGES, get_enc_index
25from chardet.output_names import _COMPAT_NAMES
26from chardet.pipeline import DetectionResult
27from chardet.pipeline.confusion import (
28 CONFUSION_BAND,
29 CONFUSION_FLOOR_RATIO,
30 STRICT_TIER_MAX_CONF,
31 _comparable_languages,
32 arbitrate_distinguishing_bytes,
33 confusion_pair_winner,
34 differing_high_bytes,
35 resolve_confusion_groups,
36)
37from chardet.registry import REGISTRY
39# Common Western Latin encodings that share the iso-8859-1 character
40# repertoire for the byte values where iso-8859-10 is indistinguishable.
41# Used as swap targets when demoting iso-8859-10 — we prefer these over
42# iso-8859-10, but do not want to accidentally promote an unrelated encoding
43# (e.g. windows-1254).
44_COMMON_LATIN_ENCODINGS: frozenset[str] = frozenset(
45 {
46 "iso8859-1",
47 "iso8859-15",
48 "cp1252",
49 }
50)
52# Bytes where iso-8859-10 decodes to a different character than iso-8859-1.
53# Computed programmatically via:
54# {b for b in range(0x80, 0x100)
55# if bytes([b]).decode('iso-8859-10') != bytes([b]).decode('iso-8859-1')}
56_ISO_8859_10_DISTINGUISHING: frozenset[int] = frozenset(
57 {
58 0xA1,
59 0xA2,
60 0xA3,
61 0xA4,
62 0xA5,
63 0xA6,
64 0xA8,
65 0xA9,
66 0xAA,
67 0xAB,
68 0xAC,
69 0xAE,
70 0xAF,
71 0xB1,
72 0xB2,
73 0xB3,
74 0xB4,
75 0xB5,
76 0xB6,
77 0xB8,
78 0xB9,
79 0xBA,
80 0xBB,
81 0xBC,
82 0xBD,
83 0xBE,
84 0xBF,
85 0xC0,
86 0xC7,
87 0xC8,
88 0xCA,
89 0xCC,
90 0xD1,
91 0xD2,
92 0xD7,
93 0xD9,
94 0xE0,
95 0xE7,
96 0xE8,
97 0xEA,
98 0xEC,
99 0xF1,
100 0xF2,
101 0xF7,
102 0xF9,
103 0xFF,
104 }
105)
107# Bytes where iso-8859-14 decodes to a different character than iso-8859-1.
108# Computed programmatically via:
109# {b for b in range(0x80, 0x100)
110# if bytes([b]).decode('iso-8859-14') != bytes([b]).decode('iso-8859-1')}
111_ISO_8859_14_DISTINGUISHING: frozenset[int] = frozenset(
112 {
113 0xA1,
114 0xA2,
115 0xA4,
116 0xA5,
117 0xA6,
118 0xA8,
119 0xAA,
120 0xAB,
121 0xAC,
122 0xAF,
123 0xB0,
124 0xB1,
125 0xB2,
126 0xB3,
127 0xB4,
128 0xB5,
129 0xB7,
130 0xB8,
131 0xB9,
132 0xBA,
133 0xBB,
134 0xBC,
135 0xBD,
136 0xBE,
137 0xBF,
138 0xD0,
139 0xD7,
140 0xDE,
141 0xF0,
142 0xF7,
143 0xFE,
144 }
145)
147# Bytes where windows-1254 has Turkish-specific characters that differ from
148# windows-1252. Windows-1254 differs from windows-1252 at 8 byte positions.
149# Two (0x8E, 0x9E) are undefined in Windows-1254 but defined in Windows-1252;
150# these are excluded here because undefined bytes are not useful for
151# identifying Turkish text. The remaining six positions map to
152# Turkish-specific letters and are the primary distinguishing signal.
153_WINDOWS_1254_DISTINGUISHING: frozenset[int] = frozenset(
154 {0xD0, 0xDD, 0xDE, 0xF0, 0xFD, 0xFE}
155)
157# Bytes where HP-Roman8 maps to lowercase accented letters but ISO-8859-1
158# maps to uppercase letters. Real HP-Roman8 text (from HP-UX terminals)
159# contains these bytes; data misdetected as HP-Roman8 typically does not.
160# {b for b in range(0x80, 0x100)
161# if (unicodedata.category(bytes([b]).decode('hp-roman8')) == 'Ll'
162# and unicodedata.category(bytes([b]).decode('iso-8859-1')) == 'Lu')}
163_HP_ROMAN8_DISTINGUISHING: frozenset[int] = frozenset(
164 {
165 0xC0,
166 0xC1,
167 0xC2,
168 0xC3,
169 0xC4,
170 0xC5,
171 0xC6,
172 0xC7,
173 0xC8,
174 0xC9,
175 0xCA,
176 0xCB,
177 0xCC,
178 0xCD,
179 0xCE,
180 0xCF,
181 0xD1,
182 0xD4,
183 0xD5,
184 0xD6,
185 0xD9,
186 0xDD,
187 0xDE,
188 }
189)
191# Encodings that are often false positives when their distinguishing bytes
192# are absent. Keyed by encoding name -> frozenset of byte values where
193# that encoding differs from iso-8859-1 (or windows-1252 in the case of
194# windows-1254).
195_DEMOTION_CANDIDATES: dict[str, frozenset[int]] = {
196 "iso8859-10": _ISO_8859_10_DISTINGUISHING,
197 "iso8859-14": _ISO_8859_14_DISTINGUISHING,
198 "cp1254": _WINDOWS_1254_DISTINGUISHING,
199 "hp-roman8": _HP_ROMAN8_DISTINGUISHING,
200}
202# Bytes where KOI8-T maps to Tajik-specific Cyrillic letters but KOI8-R
203# maps to box-drawing characters. Presence of any of these bytes is strong
204# evidence for KOI8-T over KOI8-R.
205_KOI8_T_DISTINGUISHING: frozenset[int] = frozenset(
206 {0x80, 0x81, 0x83, 0x8A, 0x8C, 0x8D, 0x8E, 0x90, 0xA1, 0xA2, 0xA5, 0xB5}
207)
210# Deletion tables for bytes.translate: length changes iff a distinguishing
211# byte occurs in the data. A translate scan runs at C speed where the
212# equivalent generator expression iterates the whole (up to max_bytes) input
213# with boxed set-membership tests. All distinguishing bytes are > 0x7F, so
214# the old per-byte high-bit filter is subsumed by set membership.
215_DEMOTION_DELETE: dict[str, bytes] = {
216 enc: bytes(byte_set) for enc, byte_set in _DEMOTION_CANDIDATES.items()
217}
218_KOI8_T_DELETE: bytes = bytes(_KOI8_T_DISTINGUISHING)
221def _should_demote(data: bytes, top: DetectionResult, target: DetectionResult) -> bool:
222 """Return True if *top*, a demotion candidate, has no byte evidence over *target*.
224 Callers guarantee ``top.encoding`` is in :data:`_DEMOTION_CANDIDATES`.
225 Two questions, cheapest first. Does *data* contain any byte the
226 candidate decodes differently from ISO-8859-1? If not, the data is
227 equally valid under both encodings, nothing at the byte level favors
228 the candidate, and it is demoted.
230 If such bytes are present, do they favor the candidate? Presence
231 alone is symmetric evidence: a Windows-1252 file whose only non-ASCII
232 letter is an ``Ö`` carries 0xD6, which HP-Roman8 reads as ``ø``, so
233 both candidates "contain" the byte and the question is which reading
234 holds up. That is a confusion-style arbitration between the candidate
235 and its swap target on the distinguishing bytes alone (see
236 :func:`~chardet.pipeline.confusion.arbitrate_distinguishing_bytes`),
237 each side scored under the variant that actually won its slot. The
238 models decide when they can: one Welsh ``ŵ`` keeps ISO-8859-14 because
239 the Welsh model knows that bigram and the Windows-1252 reading is a
240 ``ð`` no Welsh model has seen. When the models are silent, word shape
241 decides: a Kven ``đ`` keeps a Finnish file on ISO-8859-10 because a
242 letter between letters beats the superscript ``¹`` Windows-1252 reads
243 there. A lone ``Ö`` mid-word decides nothing either way, and the more
244 prevalent encoding takes the evidence-free tie.
246 The arbitration is only asked when the candidate's lead over the swap
247 target is within :data:`~chardet.pipeline.confusion.CONFUSION_BAND`.
248 A win by more than the band was decided on the full statistics, and
249 re-litigating it on a handful of bytes is neither sound nor free: the
250 scan is Python level, and mainstream Turkish text tops as windows-1254
251 with hundreds of distinguishing bytes and a lead of 0.2.
252 """
253 encoding = top.encoding or ""
254 if len(data.translate(None, _DEMOTION_DELETE[encoding])) == len(data):
255 return True
256 if top.confidence - target.confidence > CONFUSION_BAND:
257 return False
258 winner = arbitrate_distinguishing_bytes(
259 data,
260 encoding,
261 target.encoding or "",
262 _DEMOTION_CANDIDATES[encoding],
263 languages_a=None if top.language is None else frozenset((top.language,)),
264 languages_b=None if target.language is None else frozenset((target.language,)),
265 )
266 return winner != encoding
269def _swap_target(candidates: list[DetectionResult]) -> DetectionResult:
270 """Pick the common Latin candidate that replaces a demoted top.
272 Among the candidates within :data:`_DEAD_HEAT_EPSILON` of the
273 highest-scoring one, era prevalence chooses (windows-1252 over
274 iso-8859-1): inside that band the confidence order is noise, the very
275 premise of the demotion. A candidate trailing the best common Latin
276 by more than the epsilon lost to it on real evidence and stays put.
277 Equal era ranks (iso-8859-1 against iso-8859-15, both legacy ISO) keep
278 confidence order, since ``min`` returns the first of equals and the
279 candidates arrive ranked.
280 """
281 lead_conf = max(r.confidence for r in candidates)
282 in_band = [r for r in candidates if lead_conf - r.confidence <= _DEAD_HEAT_EPSILON]
283 return min(in_band, key=lambda r: _era_rank(r.encoding or ""))
286def _demote_niche_latin(
287 data: bytes,
288 results: list[DetectionResult],
289) -> list[DetectionResult]:
290 """Demote a niche Latin top that its distinguishing bytes do not support.
292 Some bigram models (iso-8859-10, iso-8859-14, windows-1254, hp-roman8)
293 can win on data that contains only bytes shared with the common Western
294 Latin encodings, or on a lone shared byte the models cannot arbitrate.
295 When :func:`_should_demote` finds no byte-level evidence for the
296 winning encoding, promote the swap target :func:`_swap_target` picks
297 among the common Latin candidates and push the demoted encoding to
298 last.
300 The demoted entries take the confidence of the candidate they now sit
301 behind. Rank position alone does not survive the trip out to callers:
302 ``detect_all`` re-sorts by confidence, and a stable sort hands an entry
303 that kept the top score its old place back.
305 :param data: The raw byte data the results were produced from.
306 :param results: A list of :class:`DetectionResult` ranked by confidence.
307 :returns: A new list (or the same list) with the demotion applied.
308 """
309 if len(results) < 2 or results[0].encoding not in _DEMOTION_CANDIDATES:
310 return results
311 candidates = [r for r in results[1:] if r.encoding in _COMMON_LATIN_ENCODINGS]
312 if not candidates:
313 return results
314 target = _swap_target(candidates)
315 if not _should_demote(data, results[0], target):
316 return results
317 demoted_encoding = results[0].encoding
318 top_conf = results[0].confidence
319 promoted = DetectionResult(
320 target.encoding, top_conf, target.language, target.mime_type
321 )
322 others = [x for x in results if x.encoding != demoted_encoding and x is not target]
323 tail_conf = others[-1].confidence if others else top_conf
324 demoted_entries = [
325 DetectionResult(
326 x.encoding,
327 min(x.confidence, tail_conf),
328 x.language,
329 x.mime_type,
330 )
331 for x in results
332 if x.encoding == demoted_encoding
333 ]
334 return [promoted, *others, *demoted_entries]
337def _promote_koi8t(
338 data: bytes,
339 results: list[DetectionResult],
340) -> list[DetectionResult]:
341 """Promote KOI8-T over KOI8-R when Tajik-specific bytes are present.
343 KOI8-T and KOI8-R share the entire 0xC0-0xFF Cyrillic letter block,
344 making statistical discrimination difficult. However, KOI8-T maps 12
345 bytes in 0x80-0xBF to Tajik-specific Cyrillic letters where KOI8-R has
346 box-drawing characters. If any of these bytes appear, KOI8-T is the
347 better match.
348 """
349 if not results or results[0].encoding != "koi8-r":
350 return results
351 # Check if KOI8-T is anywhere in the results
352 koi8t_idx = next((i for i, r in enumerate(results) if r.encoding == "koi8-t"), None)
353 if koi8t_idx is None:
354 return results
355 # Check for Tajik-specific bytes
356 if len(data.translate(None, _KOI8_T_DELETE)) != len(data):
357 return _promote_to_top(results, koi8t_idx)
358 return results
361# Confidence gap below which two candidates are a statistical dead heat:
362# their scores differ only through model-norm noise on bigrams that carry no
363# real evidence (observed dead heats sit within ~1e-5; genuinely decided
364# rankings lead by >= 2e-3).
365_DEAD_HEAT_EPSILON = 1e-4
367# On a dead heat between an encoding and its Windows superset, prefer the
368# superset: it decodes everything the base encoding does, so it is never a
369# worse answer when the statistics cannot separate them. Mirrors
370# ``markup._MARKUP_SUPERSET_PROMOTIONS``.
371_DEAD_HEAT_SUPERSETS: dict[str, str] = {
372 "shift_jis": "cp932",
373 "shift_jis_2004": "cp932",
374 "euc_kr": "cp949",
375}
377# Confidence band for the classic-Mac line-ending promotion. Wider than the
378# dead-heat epsilon because bare-\r line endings are decisive platform
379# evidence, not just a prior. Structurally the confusion band: retuning
380# ``CONFUSION_BAND`` carries this promotion's reach with it, keeping the
381# band inside ``_CORRECTION_REACH`` so pruning always scores what it scans.
382_CR_MAC_BAND = CONFUSION_BAND
384# Minimum number of \r line endings before the classic-Mac promotion fires.
385_CR_MAC_MIN_LINES = 3
387# EncodingEra.LEGACY_MAC — value inlined to avoid importing the enum into
388# this mypyc-compiled hot path for a single constant.
389_LEGACY_MAC_ERA = 4
392# Cap on the data scanned for high-byte bigram evidence — matches the
393# window statistical scoring uses (``orchestrator._STAT_SCORE_MAX_BYTES``),
394# so the evidence check sees the same bytes the scores were computed from.
395_EVIDENCE_SCAN_MAX_BYTES = 16384
398def _era_rank(encoding: str) -> int:
399 """Return the lowest era bit for *encoding* (lower = more prevalent today)."""
400 info = REGISTRY.get(encoding)
401 if info is None:
402 return 1 << 30
403 era = int(info.era)
404 return era & -era
407def _has_high_byte_evidence(data: bytes, encoding: str, language: "str | None") -> bool:
408 """Return True if *encoding*'s winning model weights a high-byte bigram present in *data*.
410 A candidate whose model assigns zero weight to every non-ASCII bigram in
411 the data earned its statistical score purely from ASCII bigrams — noise
412 that cannot distinguish encodings. Only the variant that actually won
413 (*language*) counts: another language's variant having weight for those
414 bytes says nothing about why *this* result is on top. Only called on
415 dead heats, so the Python-level scan of the (capped) data is off the
416 hot path.
417 """
418 variants = get_enc_index().get(encoding)
419 if not variants:
420 return False
421 window = data[:_EVIDENCE_SCAN_MAX_BYTES]
422 seen: set[int] = set()
423 prev = window[0]
424 for i in range(1, len(window)):
425 b = window[i]
426 if prev >= 0x80 or b >= 0x80:
427 seen.add((prev << 8) | b)
428 prev = b
429 if not seen:
430 return False
431 for lang, table, _key in variants:
432 if language is not None and lang != language:
433 continue
434 for idx in seen:
435 if table[idx]:
436 return True
437 return False
440def _prefer_prevalent_on_dead_heat(
441 data: bytes,
442 results: list[DetectionResult],
443) -> list[DetectionResult]:
444 """Break statistical dead heats in favor of the more prevalent era.
446 When several encodings score within :data:`_DEAD_HEAT_EPSILON` of the
447 top result, the ranking among them is mostly an artifact of
448 ASCII-bigram noise. Promote the candidate from the most prevalent era
449 (modern web > legacy ISO > Mac > regional > DOS > mainframe) so
450 evidence-free dead heats resolve to the likeliest real-world answer.
452 A top result whose models carry no weight for any high-byte bigram in
453 the data has no evidence at all and yields outright. One whose models
454 do weight an observed bigram is not thereby safe: an English file with
455 one capital ``É`` ranks MacRoman first because the MacRoman model
456 reads 0xC9 as the ellipsis English text is full of, a lead worth
457 2e-5. Such a top is arbitrated against the prevalent candidate on the
458 bytes the two read differently, under the languages the two can be
459 compared in (see :func:`~chardet.pipeline.confusion.arbitrate_distinguishing_bytes`
460 and confusion's ``_comparable_languages``): the Windows-1252 French
461 model knows ``École`` even when the English one does not, while its
462 Icelandic model may not read a Welsh ``dŵr`` as ``dðr`` against an
463 encoding that models no Icelandic. The prevalent candidate is
464 promoted only when it wins outright; a tie keeps the top, so an
465 ISO-8859-1 result tied with Windows-1252 on data without C1 bytes
466 stays where the statistics put it. Genuine MacRoman text never
467 reaches the arbitration, since its hundreds of distinguishing bytes
468 put Windows-1252 far outside the band.
469 """
470 top = results[0] if results else None
471 if top is None or top.encoding is None or len(results) < 2:
472 return results
473 best_idx = 0
474 best_rank = _era_rank(top.encoding)
475 for i in range(1, len(results)):
476 r = results[i]
477 if r.encoding is None:
478 continue
479 if top.confidence - r.confidence > _DEAD_HEAT_EPSILON:
480 break
481 rank = _era_rank(r.encoding)
482 if rank < best_rank:
483 best_rank = rank
484 best_idx = i
485 if best_idx == 0:
486 return results
487 if not _has_high_byte_evidence(data, top.encoding, top.language):
488 return _promote_to_top(results, best_idx)
489 rival = results[best_idx].encoding or ""
490 comparable = _comparable_languages(
491 top.encoding,
492 rival,
493 frozenset(
494 lang
495 for lang in (top.language, results[best_idx].language)
496 if lang is not None
497 ),
498 )
499 winner = arbitrate_distinguishing_bytes(
500 data,
501 top.encoding,
502 rival,
503 differing_high_bytes(top.encoding, rival),
504 languages_a=comparable,
505 languages_b=comparable,
506 )
507 if winner == rival:
508 return _promote_to_top(results, best_idx)
509 return results
512def _promote_to_top(results: list[DetectionResult], i: int) -> list[DetectionResult]:
513 """Move ``results[i]`` to the top, carrying the current top confidence."""
514 r = results[i]
515 promoted = DetectionResult(
516 r.encoding, results[0].confidence, r.language, r.mime_type
517 )
518 rest = [x for j, x in enumerate(results) if j != i]
519 return [promoted, *rest]
522def _promote_superset_on_dead_heat(
523 data: bytes,
524 results: list[DetectionResult],
525) -> list[DetectionResult]:
526 """Promote a Windows superset over its base encoding on a dead heat."""
527 top = results[0] if results else None
528 if top is None or top.encoding is None or len(results) < 2:
529 return results
530 superset = _DEAD_HEAT_SUPERSETS.get(top.encoding)
531 if superset is None:
532 return results
533 for i in range(1, len(results)):
534 r = results[i]
535 if top.confidence - r.confidence > _DEAD_HEAT_EPSILON:
536 break
537 if r.encoding == superset and decodes_without_error(data, superset):
538 return _promote_to_top(results, i)
539 return results
542# Languages whose (language, encoding) variants never accumulated a
543# measurable legacy document population. Not a judgment about the
544# languages — a graded prior about legacy-era *bytes*: iso8859-14 (Latin-8,
545# the Celtic code page) was standardized in 1998 but Celtic text lived in
546# latin-1 and moved to UTF-8; this project's wild-page mining has found no
547# native specimen, and web surveys place the encoding at noise level. The
548# one documented genuine niche — Irish gettext .po catalogues that declare
549# ISO-8859-14 (Scannell's vim/gettext translations, in the test suite) —
550# measures safely outside the arbitration gate: genuine Celtic text has no
551# prevalent-language rival anywhere near it. Revision protocol per
552# ADR-0005: any new genuine specimen goes into test-data and forces a
553# re-audit of the set. The set itself is :data:`chardet.models.RARE_LANGUAGES`
554# — one definition, used by this gate and by the language fill's thin-margin
555# band, so the two can never drift apart. Membership changes happen there.
557# Maximum lead over the best prevalent-language candidate for a
558# rare-language winner to count as a coin flip rather than evidence.
559_RARE_ARBITRATION_MARGIN = 0.02
561# Maximum absolute confidence for arbitration to apply: genuine
562# rare-language text scores well above this even when short, so the gate
563# only opens in the evidence-free zone.
564_RARE_ARBITRATION_MAX_CONFIDENCE = 0.15
567def _arbitrate_rare_language(
568 results: list[DetectionResult],
569) -> list[DetectionResult]:
570 """Demote a rare-language winner that leads a prevalent rival by a coin flip.
572 Fires only when the winner's language is in
573 :data:`~chardet.models.RARE_LANGUAGES`, its absolute confidence is
574 inside the evidence-free zone, and a prevalent-language candidate sits
575 within :data:`_RARE_ARBITRATION_MARGIN`. Genuine rare-language text fails
576 both gates: even short files score confidently, and their entire
577 neighborhood is same-language variants.
578 """
579 top = results[0] if results else None
580 if (
581 top is None
582 or top.encoding is None
583 or top.language not in RARE_LANGUAGES
584 or top.confidence >= _RARE_ARBITRATION_MAX_CONFIDENCE
585 or len(results) < 2
586 ):
587 return results
588 for i in range(1, len(results)):
589 r = results[i]
590 if top.confidence - r.confidence > _RARE_ARBITRATION_MARGIN:
591 break
592 if r.encoding is None or r.language is None:
593 continue
594 if r.language not in RARE_LANGUAGES:
595 return _promote_to_top(results, i)
596 return results
599def _promote_mac_on_cr_line_endings(
600 data: bytes,
601 results: list[DetectionResult],
602) -> list[DetectionResult]:
603 r"""Promote a classic-Mac candidate when line endings are bare ``\r``.
605 Classic Mac OS is the only platform that terminated lines with a lone
606 carriage return, so data with several ``\r`` bytes and no ``\n`` is
607 near-certainly Mac-era text. When a LEGACY_MAC candidate scores within
608 :data:`_CR_MAC_BAND` of a non-Mac top result, promote it — unless the
609 pair has a distinguishing-byte map and the byte-level evidence says the
610 current top wins: a platform prior must not overturn direct evidence
611 that confusion resolution may have just used to establish the top.
612 """
613 top = results[0] if results else None
614 if top is None or top.encoding is None or len(results) < 2:
615 return results
616 if _era_rank(top.encoding) == _LEGACY_MAC_ERA:
617 return results
618 # An art-model win is not up for prose-based review: the pairwise
619 # veto below reasons about word shapes and prose bigrams, which
620 # box-drawing data is not, and old ANSI art legitimately carries
621 # bare-CR line endings.
622 if top.language == ART_LANGUAGE:
623 return results
624 if data.find(b"\n") >= 0 or data.count(b"\r") < _CR_MAC_MIN_LINES:
625 return results
626 for i in range(1, len(results)):
627 r = results[i]
628 if top.confidence - r.confidence > _CR_MAC_BAND:
629 break
630 if r.encoding is not None and _era_rank(r.encoding) == _LEGACY_MAC_ERA:
631 # Pass both languages so the veto arbitrates this pair under
632 # the same rule confusion resolution just applied to it.
633 langs = frozenset(
634 lang for lang in (top.language, r.language) if lang is not None
635 )
636 if (
637 confusion_pair_winner(data, top.encoding, r.encoding, langs)
638 == top.encoding
639 ):
640 # Byte-level evidence says the top beats the best-ranked
641 # Mac candidate: stop entirely rather than letting a
642 # lower-ranked sibling take the promotion just because it
643 # has no distinguishing-byte map to be checked against.
644 break
645 return _promote_to_top(results, i)
646 return results
649def _decodes_under_public_names(data: bytes, encoding: str) -> bool:
650 """Check that *data* decodes completely under *encoding* and its output name.
652 The flip's promise is that the caller's ``data.decode(result)`` works,
653 and the caller sees the *public* name: ``compat_names=True`` (the
654 default) can remap to a strictly narrower codec (``euc_jis_2004`` is
655 reported as ``EUC-JP``), so a rival must decode under both names to be
656 promoted. ``prefer_superset=True`` can also narrow (cp125x leaves
657 codepoints undefined that iso-8859-x maps), but that output transform
658 checks the data itself before remapping.
659 """
660 if not decodes_completely(data, encoding):
661 return False
662 display = _COMPAT_NAMES.get(encoding)
663 return display is None or decodes_completely(data, display)
666def _prefer_decodable_on_tie(
667 data: bytes,
668 results: list[DetectionResult],
669 *,
670 input_truncated: bool,
671) -> list[DetectionResult]:
672 """Promote a strictly decoding rival over a winner with no real evidence.
674 Byte-validity filtering runs incremental decoders with ``final=False``,
675 tolerating an incomplete multi-byte sequence at the end because detection
676 input is often a prefix of a larger whole. When chardet examined the
677 caller's *entire* input, that tolerance can hand back an encoding the
678 caller's very next ``data.decode()`` will reject --- a four-byte
679 ``iso-8859-1`` word ending in ``0xE1`` detected as utf-8 (issue #380).
681 Fires only when *data* is the whole of what the caller handed over ---
682 *input_truncated* is False --- the tail can actually hold a
683 dangling sequence (a high byte in the final four, multi-byte winner),
684 and the winner's tolerant decode is **non-empty pure ASCII** --- its
685 only multi-byte evidence is the dangling tail itself. An empty
686 tolerant decode (the whole input is one clipped sequence) is zero
687 evidence, not ASCII evidence, and disqualifies the flip. The
688 best-ranked rival that decodes the input completely under both its
689 internal and public names then takes the top slot, regardless of the
690 confidence gap: an all-ASCII-evidence winner detected nothing the
691 rival did not also detect, and any statistical lead it holds comes
692 from ASCII bigrams the rival matched equally well. The scan sees the
693 ranking as given, which under ``full_ranking=False`` is pruned; if no
694 listed rival decodes, the winner stands (measured across a 648-case
695 accent-final sweep, the pruned ranking always carried a decodable
696 rival).
698 The pure-ASCII condition is what makes the unconditional flip safe. A
699 short mid-character CJK cut has a correct answer that cannot decode the
700 input --- flipping it to whichever single-byte codec happens to decode
701 the bytes trades a right answer for a wrong one, and a 5-40 byte sweep
702 measured exactly that under a gap-based rule (34 correct CJK answers
703 lost, Big5 becoming cp1125). Such a winner has decoded real multi-byte
704 characters and keeps its ranking; with the pure-ASCII condition in
705 place, the same sweep measures zero lost answers at any gap.
706 """
707 top = results[0] if results else None
708 if (
709 input_truncated
710 or top is None
711 or top.encoding is None
712 or len(results) < 2
713 # A dangling multi-byte tail needs a high byte among the final
714 # bytes (empty data trivially has none). No cheaper winner gate
715 # exists: the registry's is_multibyte means CJK-style structural
716 # multibyte and is False for utf-8, the main deferring codec. The
717 # helper below is a single tolerant decode; validity already ran
718 # the same decode once per candidate, so this adds at most one
719 # more, and only for high-byte-tailed winners.
720 or not any(b >= 0x80 for b in data[-4:])
721 or not dangling_tail_with_ascii_prefix(data, top.encoding)
722 ):
723 return results
724 for i in range(1, len(results)):
725 r = results[i]
726 if r.encoding is None or not _decodes_under_public_names(data, r.encoding):
727 continue
728 return _promote_to_top(results, i)
729 return results
732# ---------------------------------------------------------------------------
733# The pruning contract: what statistical pruning must score exactly
734# ---------------------------------------------------------------------------
736#: How far below the running second-best score a candidate can sit and still
737#: be examined by a rank correction: rare-language arbitration reads margins
738#: up to ``_RARE_ARBITRATION_MARGIN`` from the top, and confusion resolution
739#: examines the band, kept with a 2x cushion for float noise.
740_CORRECTION_REACH = _RARE_ARBITRATION_MARGIN + 2 * CONFUSION_BAND
743def scoring_floor(top1: float, top2: float) -> float:
744 """Return the score below which the rank corrections cannot examine a candidate.
746 One half of the pruning contract statistical scoring consumes: given
747 the running top two encoding scores, every candidate at or above this
748 floor must carry its exact full-ranking score, or a correction could
749 fire on an understated (or missing) entry and ``detect()`` would
750 diverge from the unpruned full ranking. The floor trails the
751 second-best score by the corrections' reach; while the top is low
752 enough for confusion resolution's strict tier to open, it extends down
753 to that tier's floor, because a strict-tier promotion may raise any
754 candidate above the tier floor into position 0 before the other
755 corrections evaluate their triggers.
757 One correction sits deliberately outside this floor: the decode-safety
758 flip (:func:`_prefer_decodable_on_tie`) scans the ranking *as given*,
759 including pruned-path tails whose entries may be understated or absent
760 — a measured trade-off documented in its own docstring, not a floor
761 violation to fix by widening the reach.
762 """
763 floor = top2 - _CORRECTION_REACH
764 if top1 < STRICT_TIER_MAX_CONF:
765 floor = min(floor, top1 * CONFUSION_FLOOR_RATIO)
766 return floor
769def forced_encodings(near_top: list[str]) -> list[str]:
770 """Return the encodings the corrections look up by name, given the near-top set.
772 The other half of the pruning contract: :func:`postprocess_results`
773 inspects some encodings wherever they rank (the common Western Latin
774 trio for niche Latin demotion, KOI8-T for the KOI8-R promotion), so
775 pruning must score every variant of these whenever a trigger encoding
776 sits at or above the :func:`scoring_floor` — confusion resolution may
777 promote any such candidate to the top before those triggers are
778 evaluated.
779 """
780 forced: list[str] = []
781 if any(e in _DEMOTION_CANDIDATES for e in near_top):
782 forced.extend(_COMMON_LATIN_ENCODINGS)
783 if "koi8-r" in near_top:
784 forced.append("koi8-t")
785 return forced
788def postprocess_results(
789 data: bytes,
790 results: list[DetectionResult],
791 *,
792 input_truncated: bool = False,
793) -> list[DetectionResult]:
794 """Apply rank corrections to the statistically scored results.
796 Steps run in sequence, weakest evidence first: dead-heat priors
797 (superset preference, era prevalence), then confusion-group resolution,
798 niche Latin demotion, and KOI8-T promotion (byte-level evidence), and
799 finally the classic-Mac line-ending promotion (platform evidence that
800 should override the priors). The decode-safety tiebreak runs last of
801 all: whatever the ranking settled on, a winner that cannot decode the
802 caller's complete input, and whose own evidence is nothing but the
803 undecodable tail, yields to the best-ranked rival that can decode it.
805 :param data: The raw byte data the results were produced from.
806 :param results: A list of :class:`DetectionResult` ranked by confidence.
807 :param input_truncated: True when *data* is a chardet-made slice rather
808 than the caller's whole input --- the ``max_bytes`` slice, the
809 evidence-cap slice, or ``UniversalDetector``'s buffer cap. Any of
810 them means the bytes here are not the whole story, so the
811 decode-safety tiebreak, which reasons about what the caller will
812 decode, stands down.
813 :returns: A new list (or the same list) with rank corrections applied.
814 """
815 results = _promote_superset_on_dead_heat(data, results)
816 results = _prefer_prevalent_on_dead_heat(data, results)
817 results = _arbitrate_rare_language(results)
818 results = resolve_confusion_groups(data, results)
819 results = _demote_niche_latin(data, results)
820 results = _promote_koi8t(data, results)
821 results = _promote_mac_on_cr_line_endings(data, results)
822 return _prefer_decodable_on_tie(data, results, input_truncated=input_truncated)