Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Three-tier language detection for filling DetectionResult languages.
3Tier 1: hardcoded mapping for single-language encodings (e.g. Big5 -> Chinese).
4Tier 2: statistical bigram scoring against the encoding's language-model variants.
5Tier 3: decode to UTF-8 and score against the UTF-8 byte-level language models.
7Note: ``from __future__ import annotations`` is intentionally omitted because
8this module is compiled with mypyc, which does not support PEP 563 string
9annotations.
10"""
12from chardet.models import (
13 _THIN_RARE_MAX_BYTES,
14 RARE_LANGUAGES,
15 BigramProfile,
16 has_model_variants,
17 infer_language,
18 score_best_language,
19)
20from chardet.pipeline import DetectionResult
22# Maximum bytes of data used for language scoring.
23# Language bigrams converge quickly — 2 KB is sufficient for discrimination
24# across all language models while keeping Tier 3 (language-model scoring) fast.
25_LANG_SCORE_MAX_BYTES = 2048
28def _to_utf8(data: bytes, encoding: str) -> bytes | None:
29 """Decode data from encoding and re-encode as UTF-8 for language scoring.
31 Returns None if the encoding is unknown. For UTF-8, returns data as-is.
32 Uses ``errors="ignore"`` because the data already passed byte-validity
33 filtering for the detected encoding; any residual invalid bytes are
34 irrelevant for language scoring.
35 """
36 if encoding == "utf-8":
37 return data
38 try:
39 return data.decode(encoding, errors="ignore").encode(
40 "utf-8", errors="surrogatepass"
41 )
42 except (LookupError, TypeError, ValueError):
43 return None
46def fill_languages(
47 data: bytes, results: list[DetectionResult]
48) -> list[DetectionResult]:
49 """Fill missing ``language`` fields on text results via the three-tier algorithm.
51 Tier 1: single-language encodings via hardcoded map (instant).
52 Tier 2: multi-language encodings via statistical bigram scoring (lazy).
53 Tier 3: decode to UTF-8, score against UTF-8 language models (universal fallback).
55 Tiers 2 and 3 score in the IDF-weighted space of
56 :meth:`~chardet.models.BigramProfile.for_language`, not the plain
57 cosine the encoding ranking uses: with 49 languages in play, the plain
58 cosine let the UTF-8 bigram for ``é`` alone hand French text to
59 Hungarian.
61 Binary results (``encoding is None``) are passed through unchanged, as
62 are results that already have a non-``None`` language — except a
63 :data:`~chardet.models.RARE_LANGUAGES` label on a thin input, which is
64 re-derived through the same scoring so the thin-rare demotion band
65 applies to statistically-attached labels too, not only to labels this
66 function computes. A re-derivation can only *demote* to a prevalent
67 language; it never swaps one rare label for another.
69 :param data: The raw byte data the results were produced from. Truncated
70 to the first 2 KB internally — bigram language models converge quickly.
71 :param results: A list of :class:`DetectionResult` from the pipeline.
72 :returns: A list of results with ``language`` filled in where possible.
73 """
74 data = data[:_LANG_SCORE_MAX_BYTES]
75 # Thinness is judged once, on the bytes the caller actually has. Tier 3
76 # transcodes to UTF-8 before scoring, which can inflate curly punctuation
77 # 3x — judging length after that would exempt exactly the inputs the
78 # band exists for.
79 thin = 0 < len(data) < _THIN_RARE_MAX_BYTES
80 filled: list[DetectionResult] = []
81 profile: BigramProfile | None = None
82 utf8_profile: BigramProfile | None = None
83 utf8_profile_src: bytes | None = None
84 for result in results:
85 recheck = (
86 thin
87 and result.language is not None
88 and result.language in RARE_LANGUAGES
89 and result.encoding is not None
90 )
91 if result.encoding is None or (result.language is not None and not recheck):
92 filled.append(result)
93 continue
94 encoding = result.encoding
95 # Tier 1: single-language encoding (skipped on re-check: the label
96 # exists; only the scored tiers can justify a demotion)
97 lang = None if recheck else infer_language(encoding)
98 # Tier 2: statistical scoring for multi-language encodings
99 if lang is None and data and has_model_variants(encoding):
100 if profile is None:
101 profile = BigramProfile.for_language(data)
102 _, lang = score_best_language(
103 data,
104 encoding,
105 profile=profile,
106 demote_thin_rare=thin,
107 language_weighting=True,
108 )
109 # Tier 3: decode to UTF-8, score against UTF-8 language models.
110 # Also entered by a thin rare Tier-2 label: an encoding whose
111 # variant set is all-Celtic (iso8859-14) can never offer the band a
112 # prevalent rival, so the utf-8 models — which always have one —
113 # get the deciding vote. Their verdict is only accepted as a
114 # demotion; a rare verdict leaves the Tier-2 label in place.
115 escalate = thin and lang is not None and lang in RARE_LANGUAGES
116 if (lang is None or escalate) and data and has_model_variants("utf-8"):
117 utf8_data = _to_utf8(data, encoding)
118 if utf8_data:
119 if utf8_data != utf8_profile_src:
120 utf8_profile = BigramProfile.for_language(utf8_data)
121 utf8_profile_src = utf8_data
122 _, utf8_lang = score_best_language(
123 utf8_data,
124 "utf-8",
125 profile=utf8_profile,
126 demote_thin_rare=thin,
127 language_weighting=True,
128 )
129 if lang is None or (
130 utf8_lang is not None and utf8_lang not in RARE_LANGUAGES
131 ):
132 lang = utf8_lang
133 if recheck and (lang is None or lang in RARE_LANGUAGES):
134 # The band did not fire (or scoring was unavailable): the
135 # original label stands. Never replace one rare label with
136 # another — the re-check's only authority is the demotion.
137 filled.append(result)
138 elif lang is None:
139 filled.append(result)
140 else:
141 filled.append(
142 DetectionResult(encoding, result.confidence, lang, result.mime_type)
143 )
144 return filled