Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Three-tier language detection for filling DetectionResult languages.
3Tier 1: hardcoded mapping for single-language encodings (e.g. Big5 -> Chinese).
4Tier 2: statistical bigram scoring against the encoding's language-model variants.
5Tier 3: decode to UTF-8 and score against the UTF-8 byte-level language models.
7Note: ``from __future__ import annotations`` is intentionally omitted because
8this module is compiled with mypyc, which does not support PEP 563 string
9annotations.
10"""
12from chardet.models import (
13 _THIN_RARE_MAX_BYTES,
14 RARE_LANGUAGES,
15 BigramProfile,
16 has_model_variants,
17 infer_language,
18 score_best_language,
19)
20from chardet.pipeline import DetectionResult
22# Maximum bytes of data used for language scoring.
23# Language bigrams converge quickly — 2 KB is sufficient for discrimination
24# across all language models while keeping Tier 3 (language-model scoring) fast.
25_LANG_SCORE_MAX_BYTES = 2048
28def _to_utf8(data: bytes, encoding: str) -> bytes | None:
29 """Decode data from encoding and re-encode as UTF-8 for language scoring.
31 Returns None if the encoding is unknown. For UTF-8, returns data as-is.
32 Uses ``errors="ignore"`` because the data already passed byte-validity
33 filtering for the detected encoding; any residual invalid bytes are
34 irrelevant for language scoring.
35 """
36 if encoding == "utf-8":
37 return data
38 try:
39 return data.decode(encoding, errors="ignore").encode(
40 "utf-8", errors="surrogatepass"
41 )
42 except (LookupError, TypeError, ValueError):
43 return None
46def fill_languages(
47 data: bytes, results: list[DetectionResult]
48) -> list[DetectionResult]:
49 """Fill missing ``language`` fields on text results via the three-tier algorithm.
51 Tier 1: single-language encodings via hardcoded map (instant).
52 Tier 2: multi-language encodings via statistical bigram scoring (lazy).
53 Tier 3: decode to UTF-8, score against UTF-8 language models (universal fallback).
55 Binary results (``encoding is None``) are passed through unchanged, as
56 are results that already have a non-``None`` language — except a
57 :data:`~chardet.models.RARE_LANGUAGES` label on a thin input, which is
58 re-derived through the same scoring so the thin-rare demotion band
59 applies to statistically-attached labels too, not only to labels this
60 function computes. A re-derivation can only *demote* to a prevalent
61 language; it never swaps one rare label for another.
63 :param data: The raw byte data the results were produced from. Truncated
64 to the first 2 KB internally — bigram language models converge quickly.
65 :param results: A list of :class:`DetectionResult` from the pipeline.
66 :returns: A list of results with ``language`` filled in where possible.
67 """
68 data = data[:_LANG_SCORE_MAX_BYTES]
69 # Thinness is judged once, on the bytes the caller actually has. Tier 3
70 # transcodes to UTF-8 before scoring, which can inflate curly punctuation
71 # 3x — judging length after that would exempt exactly the inputs the
72 # band exists for.
73 thin = 0 < len(data) < _THIN_RARE_MAX_BYTES
74 filled: list[DetectionResult] = []
75 profile: BigramProfile | None = None
76 utf8_profile: BigramProfile | None = None
77 utf8_profile_src: bytes | None = None
78 for result in results:
79 recheck = (
80 thin
81 and result.language is not None
82 and result.language in RARE_LANGUAGES
83 and result.encoding is not None
84 )
85 if result.encoding is None or (result.language is not None and not recheck):
86 filled.append(result)
87 continue
88 encoding = result.encoding
89 # Tier 1: single-language encoding (skipped on re-check: the label
90 # exists; only the scored tiers can justify a demotion)
91 lang = None if recheck else infer_language(encoding)
92 # Tier 2: statistical scoring for multi-language encodings
93 if lang is None and data and has_model_variants(encoding):
94 if profile is None:
95 profile = BigramProfile(data)
96 _, lang = score_best_language(
97 data, encoding, profile=profile, demote_thin_rare=thin
98 )
99 # Tier 3: decode to UTF-8, score against UTF-8 language models.
100 # Also entered by a thin rare Tier-2 label: an encoding whose
101 # variant set is all-Celtic (iso8859-14) can never offer the band a
102 # prevalent rival, so the utf-8 models — which always have one —
103 # get the deciding vote. Their verdict is only accepted as a
104 # demotion; a rare verdict leaves the Tier-2 label in place.
105 escalate = thin and lang is not None and lang in RARE_LANGUAGES
106 if (lang is None or escalate) and data and has_model_variants("utf-8"):
107 utf8_data = _to_utf8(data, encoding)
108 if utf8_data:
109 if utf8_data != utf8_profile_src:
110 utf8_profile = BigramProfile(utf8_data)
111 utf8_profile_src = utf8_data
112 _, utf8_lang = score_best_language(
113 utf8_data, "utf-8", profile=utf8_profile, demote_thin_rare=thin
114 )
115 if lang is None or (
116 utf8_lang is not None and utf8_lang not in RARE_LANGUAGES
117 ):
118 lang = utf8_lang
119 if recheck and (lang is None or lang in RARE_LANGUAGES):
120 # The band did not fire (or scoring was unavailable): the
121 # original label stands. Never replace one rare label with
122 # another — the re-check's only authority is the demotion.
123 filled.append(result)
124 elif lang is None:
125 filled.append(result)
126 else:
127 filled.append(
128 DetectionResult(encoding, result.confidence, lang, result.mime_type)
129 )
130 return filled