Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Model loading and bigram scoring utilities.
3Note: ``from __future__ import annotations`` is intentionally omitted because
4this module is compiled with mypyc, which does not support PEP 563 string
5annotations.
6"""
8import array
9import functools
10import hashlib
11import importlib.resources
12import math
13import operator
14import sys
15import warnings
17from chardet import _kernel
18from chardet._kernel import dot_packed, pack_profile
19from chardet.models._format import (
20 langnorm_from_table,
21 parse_langnorm_bin,
22 parse_models_bin,
23 parse_rowmax_bin,
24 rowmax_from_table,
25)
26from chardet.registry import REGISTRY, lookup_encoding
28#: Shared empty packed buffers for profiles scored through another path.
29#: Safe to share because nothing ever writes to a profile's packed
30#: buffers --- dot_packed only reads them, and pack_profile always
31#: returns fresh arrays. Rebuilding these per profile allocated four
32#: throwaway arrays on the focused-profile path, which the surrounding
33#: code exists to keep allocation-free.
34_EMPTY_PACKED = (array.array("i"), array.array("i"))
36#: Whether _kernel was compiled into an extension for this install.
37#: The packed buffers only pay off when it was: read from C they are two
38#: contiguous int32 arrays, but read from the interpreter every element
39#: access boxes an int, which is slower than the dense list it replaced.
40#: A build with mypyc but no Cython would otherwise get the packed layout
41#: with an interpreted loop -- measured 3.5x slower than either path
42#: alone -- so profiles keep the dense table instead when it is absent.
43#:
44#: This flag makes two scoring paths and two storage shapes live in one
45#: file, and **no single test run exercises both**: a compiled run never
46#: takes the dense branch, an interpreted run never takes the packed one.
47#: Running the suite against both builds is therefore load-bearing rather
48#: than thorough --- it is the only thing standing between these branches
49#: and silently diverging. Change either one and check the other.
50_KERNEL_COMPILED = _kernel.__file__.endswith((".so", ".pyd"))
52#: Whether interpreted dense profiles score through ``operator.itemgetter``.
53#: On CPython that moves the per-bigram gather and multiply-add into C,
54#: about 1.7x faster per model than the Python loop. PyPy's JIT runs the
55#: plain loop about 4x faster than the itemgetter form, so it keeps the loop.
56_GATHER_DOT = sys.implementation.name == "cpython"
58# 256-entry membership table for whitespace bytes whose runs training
59# collapsed — native byte indexing under mypyc, used in the bigram-profile
60# hot loop. Covers the ASCII whitespace bytes (space, tab, LF, VT, FF, CR)
61# plus NBSP (0xA0): training's run collapse operates on decoded text where
62# regex \s matches all of these, so Latin models carry no run weight for
63# them and an uncollapsed input run would match only unrelated models
64# where the byte happens to be a letter. 0x85 (NEL in the ISO family) is
65# deliberately absent: it decodes to the ellipsis in windows-1252, whose
66# models legitimately carry ellipsis-run weight.
67_ASCII_WHITESPACE_TABLE = bytes(
68 1 if b in (0x20, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xA0) else 0 for b in range(256)
69)
71# Encodings that map to exactly one language, derived from the registry.
72# Keyed by canonical name only — callers always use canonical names.
73_SINGLE_LANG_MAP: dict[str, str] = {}
74for _enc in REGISTRY.values():
75 if len(_enc.languages) == 1:
76 _SINGLE_LANG_MAP[_enc.name] = _enc.languages[0]
79@functools.cache
80def _load_models_data() -> tuple[dict[str, bytes], dict[str, float]]:
81 """Load and parse models.bin, returning (models, norms).
83 Cached: only reads from disk on first call.
84 """
85 ref = importlib.resources.files("chardet.models").joinpath("models.bin")
86 data = ref.read_bytes()
88 if not data:
89 warnings.warn(
90 "chardet models.bin is empty — statistical detection disabled; "
91 "reinstall chardet to fix",
92 RuntimeWarning,
93 stacklevel=2,
94 )
95 return {}, {}
97 return parse_models_bin(data)
100def load_models() -> dict[str, bytes]:
101 """Load all bigram models from the bundled models.bin file.
103 Each model is a bytes object of length 65536 (256*256).
104 Index: (b1 << 8) | b2 -> weight (0-255).
106 :returns: A dict mapping model key strings to 65536-byte lookup tables.
107 """
108 return _load_models_data()[0]
111def _build_enc_index(
112 models: dict[str, bytes],
113) -> dict[str, list[tuple[str | None, bytes, str]]]:
114 """Build a grouped index from a models dict.
116 :param models: Mapping of ``"lang/encoding"`` keys to 65536-byte tables.
117 :returns: Mapping of encoding name to ``[(lang, model, model_key), ...]``.
118 """
119 index: dict[str, list[tuple[str | None, bytes, str]]] = {}
120 for key, model in models.items():
121 lang, enc = key.split("/", 1)
122 index.setdefault(enc, []).append((lang, model, key))
124 # Resolve aliases: if a model key uses a non-canonical name,
125 # copy the entry under the canonical name.
126 for enc_name in list(index):
127 canonical = lookup_encoding(enc_name)
128 if canonical is not None and canonical not in index:
129 index[canonical] = index[enc_name]
131 return index
134@functools.cache
135def get_enc_index() -> dict[str, list[tuple[str | None, bytes, str]]]:
136 """Return a pre-grouped index mapping encoding name -> [(lang, model, model_key), ...]."""
137 return _build_enc_index(load_models())
140def infer_language(encoding: str) -> str | None:
141 """Return the language for a single-language encoding, or None.
143 :param encoding: The canonical encoding name.
144 :returns: An ISO 639-1 language code, or ``None`` if the encoding is
145 multi-language.
146 """
147 return _SINGLE_LANG_MAP.get(encoding)
150def has_model_variants(encoding: str) -> bool:
151 """Return True if the encoding has language variants in the model index.
153 :param encoding: The canonical encoding name.
154 :returns: ``True`` if bigram models exist for this encoding.
155 """
156 return encoding in get_enc_index()
159def _get_model_norms() -> dict[str, float]:
160 """Return cached L2 norms for all models, keyed by model key string."""
161 return _load_models_data()[1]
164@functools.cache
165def get_rowmax() -> dict[str, bytes]:
166 """Return per-model row-maximum tables for upper-bound prescreening.
168 For each model, entry ``b1`` of its 256-byte table holds the maximum
169 weight in the model's row for lead byte ``b1``. Because every bigram
170 weight is bounded by its row maximum, a dot product against the row
171 maxima (256 terms) upper-bounds the dot product against the full table
172 (65536 terms) — statistical scoring uses this to rule out candidate
173 models without scoring them fully.
175 Loads the precomputed ``rowmax.bin`` (written by
176 ``chardet.models._format.write_model_artifacts`` in the same model
177 order as ``models.bin``). The file starts with a ``CRM1`` magic and
178 the SHA-256 of the ``models.bin`` it was derived from: a stale or
179 mismatched file would silently under-estimate row maxima and break
180 the upper bound that pruning depends on, so anything that does not match
181 the *current* ``models.bin`` byte-for-byte is rejected and the tables
182 are derived from the models directly (slower, but always correct).
183 """
184 models = load_models()
185 files = importlib.resources.files("chardet.models")
186 try:
187 data = files.joinpath("rowmax.bin").read_bytes()
188 models_digest = hashlib.sha256(
189 files.joinpath("models.bin").read_bytes()
190 ).digest()
191 except (FileNotFoundError, OSError):
192 data = b""
193 models_digest = b""
194 # models preserves models.bin header order, matching rowmax.bin.
195 tables = parse_rowmax_bin(data, models_digest, list(models))
196 if tables is not None:
197 return tables
198 if models:
199 warnings.warn(
200 "chardet rowmax.bin is missing or does not match models.bin; "
201 "deriving row maxima from the models (slower startup)",
202 RuntimeWarning,
203 stacklevel=2,
204 )
205 return {key: rowmax_from_table(table) for key, table in models.items()}
208@functools.cache
209def get_language_norms() -> dict[str, float]:
210 """Return per-model norms for language scoring, keyed by model key.
212 The language tiers score in an IDF-weighted space (see
213 :meth:`BigramProfile.for_language`), where a model's length is its
214 norm after each weight is scaled by the square root of its bigram's
215 IDF. Loads the precomputed ``langnorm.bin`` (written by
216 ``chardet.models._format.write_model_artifacts`` in the same model
217 order as ``models.bin``) and, like :func:`get_rowmax`, rejects a file
218 whose embedded digest does not match the current ``models.bin``:
219 a stale norm table would tilt every language comparison. The
220 fallback derives the norms from the tables directly, which is correct
221 but costs a pass over every model.
222 """
223 models = load_models()
224 files = importlib.resources.files("chardet.models")
225 try:
226 data = files.joinpath("langnorm.bin").read_bytes()
227 models_digest = hashlib.sha256(
228 files.joinpath("models.bin").read_bytes()
229 ).digest()
230 except (FileNotFoundError, OSError):
231 data = b""
232 models_digest = b""
233 norms = parse_langnorm_bin(data, models_digest, list(models))
234 if norms is not None:
235 return norms
236 if models:
237 warnings.warn(
238 "chardet langnorm.bin is missing or does not match models.bin; "
239 "deriving language norms from the models (slower startup)",
240 RuntimeWarning,
241 stacklevel=2,
242 )
243 idf = get_idf_weights()
244 return {key: langnorm_from_table(table, idf) for key, table in models.items()}
247#: The root-IDF scale table carries this factor so its entries fill a byte:
248#: ``round(16 * sqrt(idf))`` runs from 16 to 255. The language profile's
249#: norm carries the same factor, so it cancels out of every score.
250_LANGUAGE_SCALE_FACTOR = 16
253@functools.cache
254def get_language_scale() -> bytes:
255 """Return the 65536-byte root-IDF table the language profile multiplies by.
257 Entry ``i`` is ``round(16 * sqrt(idf[i]))``. The language tiers
258 compare an IDF-weighted profile against models weighted by the square
259 root of IDF; folding the model-side factor into the profile, as the
260 product of this table and the IDF table per occurrence, leaves the
261 model tables untouched, so the same scoring loop serves both metrics
262 and only the norm differs (see :func:`get_language_norms`). A byte
263 table keeps the profile walk on native indexing: a list of the
264 products costs about as much again as the walk itself.
265 """
266 idf = get_idf_weights()
267 roots = bytes(round(_LANGUAGE_SCALE_FACTOR * math.sqrt(v)) for v in range(256))
268 return bytes(roots[v] for v in idf)
271@functools.cache
272def get_idf_weights() -> bytes:
273 """Return a 65536-byte IDF weight table for bigram profile construction.
275 Loads a precomputed table from ``idf.bin`` (generated at training time).
276 For each bigram index, the weight reflects how discriminative that bigram
277 is across all models:
279 - Bigrams in every model (common ASCII) → weight 1 (minimal signal)
280 - Bigrams in one model → weight 255 (maximum signal)
281 - Bigrams not in any model → weight 1 (unknown, treat as neutral)
282 """
283 ref = importlib.resources.files("chardet.models").joinpath("idf.bin")
284 data = ref.read_bytes()
285 if len(data) != 65536:
286 warnings.warn(
287 f"chardet idf.bin has wrong size ({len(data)}), "
288 "falling back to uniform weights",
289 RuntimeWarning,
290 stacklevel=2,
291 )
292 return b"\x01" * 65536
293 return data
296class BigramProfile:
297 """Pre-computed bigram frequency distribution for a data sample.
299 Computing this once and reusing it across all models reduces per-model
300 scoring from O(n) to O(distinct_bigrams).
302 Each bigram is weighted by its IDF (inverse document frequency) across all
303 models — bigrams unique to few models get high weight, bigrams common to
304 all models get weight 1. ``nonzero`` lists the indices carrying weight,
305 in first-encounter order. :meth:`for_language` builds the same profile
306 under the language tiers' heavier weighting instead.
308 The weights are reachable two ways, and which one a profile fills
309 depends on how it was built:
311 * ``idx_arr``/``val_arr`` — parallel ``array('i')`` buffers, what
312 :func:`score_with_profile` reads for a streaming profile. Filled by
313 the streaming constructor only.
314 * ``values`` — a plain list parallel to ``nonzero``, filled by
315 :meth:`from_weighted_freq` for the small focused profiles confusion
316 resolution builds, which are scored inline instead.
317 * ``gather``/``dense_vals`` — an :func:`operator.itemgetter` over
318 ``nonzero`` and the weights it picks out of the dense table, filled
319 for a streaming profile only when the kernel is not compiled and
320 the interpreter is CPython.
321 ``row_freq`` aggregates the weights by lead byte (256 entries) and
322 ``nonzero_rows`` lists the lead bytes with non-zero total; together with
323 per-model row maxima (:func:`get_rowmax`) they let statistical scoring
324 compute a cheap upper bound on a model's score.
326 **Input limit.** Weights are packed as int32, which holds any value an
327 input of at most 16 MB can produce (``255`` per occurrence against a
328 2.1-billion ceiling). Detection truncates to ``max_bytes`` long before
329 that; construct a profile directly from a larger buffer and packing
330 raises :exc:`OverflowError`.
331 """
333 __slots__ = (
334 "dense_vals",
335 "freq",
336 "gather",
337 "idx_arr",
338 "input_norm",
339 "nonzero",
340 "nonzero_rows",
341 "row_freq",
342 "val_arr",
343 "values",
344 "weight_sum",
345 )
347 def __init__(self, data: bytes) -> None:
348 """Compute the bigram frequency distribution for *data*.
350 Each bigram is weighted by its IDF (inverse document frequency) across
351 all loaded models. Bigrams unique to few models get high weight;
352 bigrams common to all models get weight 1.
354 :param data: The raw byte data to profile.
355 """
356 total_bigrams = len(data) - 1
357 if total_bigrams <= 0:
358 # Empty lists, not [0]*65536: a no-op profile allocates nothing.
359 self.freq: list[int] = []
360 self.nonzero: list[int] = []
361 self.values: list[int] = []
362 self.idx_arr, self.val_arr = _EMPTY_PACKED
363 self.row_freq: list[int] = []
364 self.nonzero_rows: list[int] = []
365 self.weight_sum: int = 0
366 self.input_norm: float = 0.0
367 return
369 idf = get_idf_weights()
370 freq: list[int] = [0] * 65536
371 nonzero: list[int] = []
372 w_sum = 0
373 for i in range(total_bigrams):
374 b1 = data[i]
375 b2 = data[i + 1]
376 # Skip repeated-whitespace bigrams (equivalent to collapsing
377 # whitespace runs, which training does before counting): padding
378 # and indentation carry no encoding signal, and models trained
379 # on lossy transcodes can carry spurious weight for them.
380 if b1 == b2 and _ASCII_WHITESPACE_TABLE[b1]:
381 continue
382 idx = (b1 << 8) | b2
383 w = idf[idx]
384 if freq[idx] == 0:
385 nonzero.append(idx)
386 freq[idx] += w
387 w_sum += w
388 # No ``values``: this path already has the dense table, and
389 # materializing a parallel list over every nonzero bigram costs
390 # more than the sparse constructor's allocation saves.
391 self._finish(freq, nonzero, [], w_sum)
393 @classmethod
394 def for_language(cls, data: bytes) -> "BigramProfile":
395 """Build the profile the language tiers score with.
397 Same bigram walk as the streaming constructor, but each occurrence
398 carries IDF times :func:`get_language_scale` (a byte-quantized
399 root of IDF) rather than IDF alone. Scored against
400 :func:`get_language_norms`, this is the cosine between the
401 IDF-weighted input and models weighted by the square root of IDF.
402 The plain cosine lets one high-IDF bigram that
403 two languages share, such as the UTF-8 pair for ``é``, outweigh
404 every bigram that tells them apart: it rates the model by the share
405 that bigram takes of its *unweighted* length, so an accent-heavy
406 language wins French text on the strength of ``é`` alone. Weighting
407 the model side too charges each model for its rare bigrams, and the
408 common bigrams decide again.
410 :param data: The raw byte data to profile.
411 :returns: A profile for use with ``language_weighting=True``.
412 """
413 profile = cls(b"")
414 total_bigrams = len(data) - 1
415 if total_bigrams <= 0:
416 return profile
417 idf = get_idf_weights()
418 freq: list[int] = [0] * 65536
419 nonzero: list[int] = []
420 for i in range(total_bigrams):
421 b1 = data[i]
422 b2 = data[i + 1]
423 if b1 == b2 and _ASCII_WHITESPACE_TABLE[b1]:
424 continue
425 idx = (b1 << 8) | b2
426 if freq[idx] == 0:
427 nonzero.append(idx)
428 freq[idx] += idf[idx]
429 # The walk above is the streaming constructor's, under IDF alone.
430 # The model-side factor (root IDF, scaled to a byte) is applied once
431 # per distinct bigram rather than once per occurrence, and the
432 # profile's own length is taken under IDF with the same scale
433 # factor so it cancels: the quotient is then the cosine between the
434 # IDF-weighted input and the root-IDF-weighted model.
435 scale = get_language_scale()
436 w_sum = 0
437 norm_sq = 0
438 for idx in nonzero:
439 v = freq[idx]
440 norm_sq += v * v
441 w = v * scale[idx]
442 freq[idx] = w
443 w_sum += w
444 profile._finish(freq, nonzero, [], w_sum)
445 profile.input_norm = _LANGUAGE_SCALE_FACTOR * math.sqrt(norm_sq)
446 return profile
448 def _finish(
449 self,
450 freq: list[int],
451 nonzero: list[int],
452 values: list[int],
453 weight_sum: int,
454 ) -> None:
455 """Store the frequency data and derive the norm and row aggregates.
457 Single finalization path shared by both constructors so pruning
458 fields cannot silently diverge between them.
460 Exactly one of *freq* and *values* carries the weights, never both.
461 The streaming constructor passes the dense *freq* table it had to
462 build and leaves *values* empty; the sparse constructor fills
463 *values* parallel to *nonzero* and passes no table, which is what
464 lets it skip a 65536-entry allocation per call.
466 Neither is retained. *freq* is read here to derive the norm, the row
467 aggregates and the packed buffers, and is then dropped -- scoring
468 reads ``idx_arr``/``val_arr`` or ``values``, never the dense table,
469 so keeping it alive would pin 512 KB per profile for nothing.
470 """
471 self.nonzero = nonzero
472 self.values = values
473 self.weight_sum = weight_sum
474 norm_sq = 0
475 row_freq: list[int] = [0] * 256
476 if values:
477 for i in range(len(nonzero)):
478 v = values[i]
479 norm_sq += v * v
480 row_freq[nonzero[i] >> 8] += v
481 else:
482 for idx in nonzero:
483 v = freq[idx]
484 norm_sq += v * v
485 row_freq[idx >> 8] += v
486 self.input_norm = math.sqrt(norm_sq)
487 self.row_freq = row_freq
488 self.nonzero_rows = [b1 for b1 in range(256) if row_freq[b1]]
489 # Dense profiles are scored through the packed kernel when it is
490 # compiled; sparse ones (a median of eight bigrams) are scored inline
491 # either way and skip the packing.
492 if values or not _KERNEL_COMPILED:
493 self.idx_arr, self.val_arr = _EMPTY_PACKED
494 else:
495 self.idx_arr, self.val_arr = pack_profile(nonzero, freq)
496 # Retained only when scoring will read it -- see _KERNEL_COMPILED.
497 self.freq = [] if (values or _KERNEL_COMPILED) else freq
498 # See _GATHER_DOT. Skipped below two bigrams, where itemgetter
499 # returns a bare value rather than a tuple.
500 self.gather: operator.itemgetter[tuple[int, ...]] | None = None
501 self.dense_vals: tuple[int, ...] = ()
502 if self.freq and len(nonzero) > 1 and _GATHER_DOT:
503 gather = operator.itemgetter(*nonzero)
504 self.gather = gather
505 self.dense_vals = gather(freq)
507 @classmethod
508 def from_weighted_freq(cls, weighted_freq: dict[int, int]) -> "BigramProfile":
509 """Create a BigramProfile from pre-computed weighted frequencies.
511 Computes ``weight_sum`` and ``input_norm`` from *weighted_freq* to
512 ensure consistency between the stored fields.
514 Deliberately does not build the dense 65536-entry ``freq`` table
515 the streaming constructor uses. Callers here pass a handful of
516 bigrams — confusion resolution's focused profiles hold a median of
517 eight — and allocating a 65536-element list per call cost about
518 24us, which measured as roughly 40% of the whole bigram-rescore
519 stage. Scoring reads ``nonzero``/``values``, so the table is
520 never needed.
522 :param weighted_freq: Mapping of bigram index to weighted count.
523 :returns: A new :class:`BigramProfile` instance.
524 """
525 profile = cls(b"")
526 nonzero: list[int] = []
527 values: list[int] = []
528 w_sum = 0
529 for idx, count in weighted_freq.items():
530 if count:
531 nonzero.append(idx)
532 values.append(count)
533 w_sum += count
534 profile._finish([], nonzero, values, w_sum)
535 return profile
538def score_with_profile(
539 profile: BigramProfile,
540 model: "bytes | bytearray | memoryview",
541 model_key: str = "",
542 *,
543 language_weighting: bool = False,
544) -> float:
545 """Score a pre-computed bigram profile against a single model using cosine similarity.
547 ``bytearray``/``memoryview`` tables are accepted for compatibility but
548 copied to ``bytes`` first: the narrow type lets mypyc compile the
549 dot-product loop with native byte indexing (and keeps compiled and
550 pure-Python installs accepting the same argument types).
552 With *language_weighting* the profile must come from
553 :meth:`BigramProfile.for_language` and the model is normalized by its
554 :func:`get_language_norms` entry, which together give the cosine in
555 the language tiers' IDF-weighted space.
556 """
557 if not isinstance(model, bytes):
558 model = bytes(model)
559 if profile.input_norm == 0.0:
560 return 0.0
561 norms = get_language_norms() if language_weighting else _get_model_norms()
562 model_norm = norms.get(model_key) if model_key else None
563 if model_norm is None:
564 if language_weighting:
565 model_norm = langnorm_from_table(model, get_idf_weights())
566 else:
567 sq_sum = 0
568 for i in range(65536):
569 v = model[i]
570 if v:
571 sq_sum += v * v
572 model_norm = math.sqrt(sq_sum)
573 if model_norm == 0.0:
574 return 0.0
575 nonzero = profile.nonzero
576 values = profile.values
577 if values:
578 # Focused confusion profiles hold a median of eight bigrams, so this
579 # branch stays inline: a call into _kernel would cost about what the
580 # loop does. It measured 1.3% of compiled runtime.
581 dot = 0
582 for i in range(len(nonzero)):
583 dot += model[nonzero[i]] * values[i]
584 elif _KERNEL_COMPILED:
585 dot = dot_packed(profile.idx_arr, profile.val_arr, model)
586 else:
587 # No compiled kernel: the dense table beats packed buffers here,
588 # because a list returns a cached int where array('i') boxes a new one.
589 gather = profile.gather
590 if gather is not None:
591 dot = sum(map(operator.mul, gather(model), profile.dense_vals))
592 else:
593 dot = 0
594 freq = profile.freq
595 for idx in nonzero:
596 dot += model[idx] * freq[idx]
597 return dot / (model_norm * profile.input_norm)
600#: ADR-0005's hand-audited rare-language set, shared with the encoding-side
601#: arbitration gate in ``pipeline.postprocess``. One set, two gates: the
602#: deployment evidence that justifies membership is recorded in the ADR and
603#: any new genuine specimen forces a re-audit of both.
604RARE_LANGUAGES: frozenset[str] = frozenset({"gd", "cy", "ga", "br"})
606#: The ANSI/ASCII-art pseudo-language. Never a valid demotion target for
607#: the thin-rare band: swapping a rare label for "zxx" would tell the
608#: orchestrator the text has no linguistic content at all.
609ART_LANGUAGE = "zxx"
611#: The thin-margin band for ``demote_thin_rare``: inputs shorter than this
612#: are where bigram cosines stop discriminating (apostrophe-rich English
613#: snippets score as Gaelic). The smallest genuine rare-language files in
614#: the test corpus are 253 bytes (legacy path) and 560 bytes (fill path),
615#: 2x and 4.4x above this gate; the callers in ``pipeline.language`` own
616#: the length judgment because only they know the pre-transcoding size.
617_THIN_RARE_MAX_BYTES = 128
619#: Maximum lead over the best prevalent-language variant for a rare win on
620#: a short input to count as noise, measured in the language tiers' weighted
621#: cosine. The calibrated English mislabel snippet wins as Gaelic by 0.039
622#: and the pinned Irish casualty by 0.053. This is NOT a safety floor:
623#: genuine gd/cy/ga snippets measure 0.14+ at 60 characters, but short
624#: Breton and Irish can sit inside the band and are the accepted casualty
625#: class per ADR-0005's addendum — widening the margin widens that class.
626_THIN_RARE_MARGIN = 0.06
629def score_best_language(
630 data: bytes,
631 encoding: str,
632 profile: BigramProfile | None = None,
633 *,
634 demote_thin_rare: bool = False,
635 language_weighting: bool = False,
636) -> tuple[float, str | None]:
637 """Score data against all language variants of an encoding.
639 Returns (best_score, best_language). Uses a pre-grouped index for O(L)
640 lookup where L is the number of language variants for the encoding.
642 If *profile* is provided, it is reused instead of recomputing the bigram
643 frequency distribution from *data*.
645 :param data: The raw byte data to score.
646 :param encoding: The canonical encoding name to match against.
647 :param profile: Optional pre-computed :class:`BigramProfile` to reuse.
648 :param demote_thin_rare: Pass true only when the caller has judged the
649 *original* input thin (under :data:`_THIN_RARE_MAX_BYTES` before
650 any transcoding). A :data:`RARE_LANGUAGES` winner that leads the
651 best prevalent-language variant by less than the measured noise
652 band is then reported under the prevalent language instead. The
653 length judgment deliberately lives with the caller: this function
654 may receive transcoded bytes or a profile without its source data,
655 so ``len(data)`` here is not a reliable proxy for input size.
656 Language-fill callers pass this; encoding-ranking callers must not,
657 so that candidate ordering stays byte-identical.
658 :param language_weighting: Score in the language tiers' IDF-weighted
659 space (see :meth:`BigramProfile.for_language`). A *profile* passed
660 alongside must have been built by that constructor. The
661 encoding-ranking caller leaves this off for the same reason as
662 *demote_thin_rare*: its scores feed the confidence and ordering of
663 encodings, and the weighted cosine lives on a different scale.
664 :returns: A ``(score, language)`` tuple. The score is always the best
665 cosine similarity across variants; the language matches it except
666 when ``demote_thin_rare`` fires, in which case the label is the
667 best prevalent-language variant's while the score remains the
668 rare winner's.
669 """
670 if not data and profile is None:
671 return 0.0, None
673 index = get_enc_index()
674 variants = index.get(encoding)
675 if variants is None:
676 return 0.0, None
678 if profile is None:
679 profile = (
680 BigramProfile.for_language(data)
681 if language_weighting
682 else BigramProfile(data)
683 )
685 best_score = 0.0
686 best_lang: str | None = None
687 best_prevalent = 0.0
688 best_prevalent_lang: str | None = None
689 for lang, model, model_key in variants:
690 s = score_with_profile(
691 profile, model, model_key, language_weighting=language_weighting
692 )
693 if s > best_score:
694 best_score = s
695 best_lang = lang
696 if (
697 demote_thin_rare
698 and lang not in RARE_LANGUAGES
699 and lang != ART_LANGUAGE
700 and s > best_prevalent
701 ):
702 best_prevalent = s
703 best_prevalent_lang = lang
705 if (
706 demote_thin_rare
707 and best_lang is not None
708 and best_lang in RARE_LANGUAGES
709 and best_prevalent_lang is not None
710 and best_score - best_prevalent < _THIN_RARE_MARGIN
711 ):
712 best_lang = best_prevalent_lang
714 return best_score, best_lang