Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/turbohtml/detect.py: 80%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2turbohtml.detect: standalone character-encoding detection over bytes.
4:func:`detect` answers the question ``chardet``, ``charset-normalizer``, and ``cchardet`` exist for -- "what encoding
5are these bytes?" -- without an HTML parser in the call path. It runs the same C pipeline :func:`turbohtml.parse`
6uses for ``bytes`` input: the WHATWG sniff first (a byte-order mark, then a ``<meta>`` prescan of the first 1024
7bytes), then a content detector that validates UTF-8 structurally and otherwise scores the CJK and single-byte
8candidates on character-pair frequencies, then the spec's windows-1252 fallback. A non-mark input therefore yields the
9same encoding whether you detect it standalone or parse it with ``detect_encoding=True``, with two divergences. A UTF-8
10byte-order mark is reported here as ``UTF-8-SIG`` so a caller can strip it, where the spec-locked parse path keeps the
11plain ``UTF-8`` name; the UTF-16 marks report the same label both ways. And a ``<meta>`` charset past the prescan's
121024-byte window is invisible here, because these functions read bytes and have no tree to consult, where
13:func:`turbohtml.parse` redoes the parse against what it declares.
15A result is an :class:`EncodingMatch` with the WHATWG canonical name, a confidence, and the language the frequency
16model matched, mirroring the ``chardet.detect`` dict shape as a typed record. :func:`detect_all` ranks every
17surviving candidate, :class:`EncodingDetector` accumulates a stream chunk by chunk like chardet's
18``UniversalDetector``, and a frozen :class:`Detection` config carries the knobs (a confidence floor and
19encoding/language constraints).
21:func:`detect_language` answers the separate question ``whatlang``, ``resiliparse``, and ``trafilatura`` exist for --
22"what natural language is this text?" -- from the visible text rather than an ``<html lang>`` attribute. It runs a
23character-trigram model: find the dominant Unicode script, then rank the languages sharing it by how closely the
24text's most frequent character trigrams match each language's embedded profile. It returns a :class:`LanguageMatch`
25(ISO 639-3 code, confidence, script, English name); a frozen :class:`LanguageDetection` config carries a confidence
26floor and language constraints.
28:func:`normalize` runs Unicode normalization (UAX #15) in all four forms -- NFC, NFD, NFKC, NFKD -- the transform
29:func:`unicodedata.normalize` does whole strings, reimplemented in C over pinned tables generated from the same
30``unicodedata`` so the two agree exactly. A quick check returns already-normalized text untouched, and
31:func:`is_normalized` answers the membership question without building the normalized copy.
32"""
34from __future__ import annotations
36import codecs
37import string
38from dataclasses import dataclass
39from itertools import starmap
40from typing import Final, Literal
42from ._html import (
43 _codec_label,
44 _decode,
45 _detect,
46 _detect_language,
47 _detect_rank,
48 _DetectStream,
49 _is_normalized,
50 _normalize,
51)
53__all__ = [
54 "Detection",
55 "EncodingDetector",
56 "EncodingMatch",
57 "LanguageDetection",
58 "LanguageMatch",
59 "NormalizationForm",
60 "detect",
61 "detect_all",
62 "detect_language",
63 "is_normalized",
64 "normalize",
65]
67NormalizationForm = Literal["NFC", "NFD", "NFKC", "NFKD"]
69_FORMS: Final[dict[str, int]] = {"NFC": 0, "NFD": 1, "NFKC": 2, "NFKD": 3}
72def _refuse_encode(text: str, errors: str = "strict", /) -> tuple[bytes, int]: # ruff:ignore[unused-function-argument]
73 """Refuse to encode: the generated tables are decode-side only, and the spec's encoders are a separate algorithm."""
74 msg = "a whatwg-* codec decodes only; encode with the CPython codec of your choice"
75 raise UnicodeError(msg)
78def _search(name: str) -> codecs.CodecInfo | None:
79 """
80 Resolve a ``whatwg-*`` codec name.
82 These codecs exist because no safe name already does: ``bytes.decode("big5")`` reaches CPython's Big5, a strict
83 subset of the spec's, ``koi8-u`` reaches KOI8-U where the spec means KOI8-RU, and ``x-mac-cyrillic`` reaches no
84 codec at all. Which label a name resolves to is decided in C, which knows every spec alias and the byte-order-mark
85 names that delegate to CPython's own decoders.
86 """
87 if (found := _codec_label(name)) is None:
88 return None
89 delegate, label = found
91 def decode(data: bytes, errors: str = "strict", /) -> tuple[str, int]: # ruff:ignore[unused-function-argument]
92 if delegate:
93 # "replace" turns malformed bytes into U+FFFD as turbohtml.parse does, where the strict codec would raise
94 return codecs.lookup(label).decode(data, "replace")
95 return _decode(data, label), len(data)
97 # CodecInfo's decoder is typed against _typeshed.ReadableBuffer, which Sphinx cannot import when it walks the
98 # annotations at doc-build time; bytes is what the codecs machinery ever passes a decode function.
99 return codecs.CodecInfo(_refuse_encode, decode, name=name) # ty: ignore[invalid-argument-type]
102codecs.register(_search)
105_LANGUAGES: Final[dict[str, str]] = {
106 "big5": "Chinese",
107 "euc-jp": "Japanese",
108 "euc-kr": "Korean",
109 "gbk": "Chinese",
110 "ibm866": "Russian",
111 "iso-2022-jp": "Japanese",
112 "iso-8859-5": "Russian",
113 "iso-8859-6": "Arabic",
114 "iso-8859-7": "Greek",
115 "iso-8859-8": "Hebrew",
116 "koi8-u": "Russian",
117 "shift_jis": "Japanese",
118 "windows-1251": "Russian",
119 "windows-1253": "Greek",
120 "windows-1254": "Turkish",
121 "windows-1255": "Hebrew",
122 "windows-1256": "Arabic",
123 "windows-1258": "Vietnamese",
124 "windows-874": "Thai",
125}
126"""The language each frequency-scored candidate models; multi-language Latin encodings stay unmapped."""
129@dataclass(frozen=True, slots=True)
130class EncodingMatch:
131 """
132 One detection result: ``chardet.detect``'s ``{"encoding", "confidence", "language"}`` dict as a typed record.
134 ``encoding`` is the WHATWG canonical name (the same string :attr:`turbohtml.Document.encoding` reports), or
135 ``None`` when the input is empty or every candidate was ruled out. A leading byte-order mark reports the mark's own
136 label instead: ``"UTF-8-SIG"`` for a UTF-8 mark (so a caller can decode with the ``utf-8-sig`` codec to strip it),
137 and ``"UTF-16LE"`` / ``"UTF-16BE"`` for the UTF-16 marks, which a mark identifies unambiguously with no heuristic.
138 The WHATWG BOM sniff has no UTF-32, so ``FF FE 00 00`` is ``"UTF-16LE"`` here, as in the parser.
140 ``codec`` is the name to hand :meth:`bytes.decode`; ``encoding`` is not. A WHATWG name and the CPython codec that
141 answers to it are different encodings: ``bytes.decode("big5")`` reaches a strict subset of the spec's Big5,
142 ``koi8-u`` reaches KOI8-U where the spec means KOI8-RU, and ``x-mac-cyrillic`` reaches no codec at all. So
143 ``codec`` names a ``whatwg-*`` codec this module registers, whose decoder is the one :func:`turbohtml.parse`
144 uses, and ``data.decode(match.codec)`` reproduces the parser's text. It is ``None`` with a ``None`` encoding.
145 Those codecs decode only: encoding to a legacy charset is a separate spec algorithm turbohtml omits.
147 ``confidence`` is 1.0 for a certain result (a byte-order mark, a ``<meta>`` declaration, structurally valid UTF-8,
148 escape-driven ISO-2022-JP, or pure ASCII), the candidate's share of the positive frequency scores for a
149 content-scored result, and 0.0 for the windows-1252 fallback chosen with no positive evidence. ``language`` names
150 the language the winning frequency model targets (``"Russian"`` for windows-1251, ``"Japanese"`` for Shift_JIS,
151 ...); it is ``None`` for UTF-8 and the Latin encodings whose model spans several languages. ``bom`` is true only
152 when a byte-order mark decided the result, telling a caller the decoded text still carries the mark unless the
153 codec strips it.
154 """
156 encoding: str | None
157 confidence: float
158 language: str | None
159 bom: bool = False
160 codec: str | None = None
163# Every character a DNS label may hold once it is lower-cased and Punycode-encoded (RFC 1123, plus RFC 3492's "xn--").
164_TLD_ALPHABET: Final[frozenset[str]] = frozenset(string.ascii_lowercase + string.digits + "-")
167@dataclass(frozen=True, slots=True)
168class Detection:
169 """
170 Options for :func:`detect`, :func:`detect_all`, and :class:`EncodingDetector`.
172 :param threshold: the confidence floor; a candidate below it is dropped, and when every candidate falls below it
173 the result's ``encoding`` is ``None``. The default 0.0 always answers, like ``chardet.detect``.
174 :param language: prefer this language when the evidence is ambiguous: candidates whose frequency model targets it
175 (the value :class:`EncodingMatch` reports as its language) rank ahead of the rest, provided they scored
176 positively.
177 :param allowed: when set, only these encodings (WHATWG names, any case) may be returned.
178 :param excluded: these encodings are never returned; mutually exclusive with ``allowed``.
179 :param tld: the rightmost DNS label of the host that served the bytes (``"jp"``, ``"ru"``, ``"xn--p1ai"``), which
180 narrows the candidates the way a browser does. Give it in lower-case ASCII, without the leading dot, and in
181 Punycode for an internationalized domain. A two-letter label the classifier does not carry reads as Western
182 European; a generic label such as ``"com"`` carries no hint, and neither does the default ``None``.
183 """
185 threshold: float = 0.0
186 language: str | None = None
187 allowed: frozenset[str] | None = None
188 excluded: frozenset[str] = frozenset()
189 tld: str | None = None
191 def __post_init__(self) -> None:
192 """Reject an out-of-range threshold, the contradictory allowed+excluded combination, and a malformed TLD."""
193 if not 0.0 <= self.threshold <= 1.0:
194 msg = f"threshold must be within [0.0, 1.0], got {self.threshold}"
195 raise ValueError(msg)
196 if self.allowed is not None and self.excluded:
197 msg = "allowed and excluded are mutually exclusive"
198 raise ValueError(msg)
199 # a whole hostname, a leading dot, or an upper-case letter would otherwise read as no hint at all rather
200 # than as the mistake it is; chardetng panics on the same labels rather than classify them
201 if self.tld is not None and not (self.tld and set(self.tld) <= _TLD_ALPHABET):
202 msg = f"tld must be the rightmost DNS label in lower-case ASCII, got {self.tld!r}"
203 raise ValueError(msg)
205 @classmethod
206 def chardet(cls) -> Detection:
207 """Chardet's ``UniversalDetector`` mode: report ``None`` under its 0.2 minimum confidence."""
208 return cls(threshold=0.2)
211_DEFAULT: Final = Detection()
214def detect(data: bytes, options: Detection | None = None, /) -> EncodingMatch:
215 """
216 Detect the character encoding of a byte string, the ``chardet.detect`` / ``cchardet.detect`` successor.
218 :param data: the bytes to sniff; HTML input also honors a ``<meta>`` charset declaration.
219 :param options: the detection options; defaults to :class:`Detection` (always answer, no constraints).
220 :returns: the best match; its ``encoding`` is ``None`` when the input is empty or every candidate was ruled out. A
221 leading byte-order mark reports its own label (``UTF-8-SIG``, ``UTF-16LE``/``BE``) with ``bom`` set, so a
222 caller can strip it.
223 :raises TypeError: when ``data`` is not a bytes-like object.
224 """
225 active: Final = options or _DEFAULT
226 return EncodingMatch(*_rank(_detect(data, active.tld), active)[0])
229def detect_all(data: bytes, options: Detection | None = None, /) -> list[EncodingMatch]:
230 """
231 Detect the character encoding and rank every plausible candidate, the ``chardet.detect_all`` successor.
233 :param data: the bytes to sniff; HTML input also honors a ``<meta>`` charset declaration.
234 :param options: the detection options; defaults to :class:`Detection` (always answer, no constraints).
235 :returns: the matches best first, :func:`detect`'s result leading; ``[EncodingMatch(None, 0.0, None)]`` when the
236 input is empty or every candidate was ruled out. A leading byte-order mark collapses the ranking to one match
237 carrying its own label and ``bom``.
238 :raises TypeError: when ``data`` is not a bytes-like object.
239 """
240 return _matches(data, options or _DEFAULT)
243class EncodingDetector:
244 """
245 Incremental detection over a byte stream, mirroring chardet's ``UniversalDetector``.
247 Call :meth:`feed` with each chunk, :meth:`close` for the result, and :meth:`reset` to reuse the instance on
248 another stream. :attr:`done` turns true as soon as the result cannot change (a leading byte-order mark, or
249 :meth:`close`), so a reader loop can stop early. Every candidate carries its own state across feeds, so the
250 detector holds a fixed amount of memory whatever the stream's length; the one buffer it keeps is the leading
251 1024 bytes the byte-order-mark check and the ``<meta>`` prescan read, which the spec bounds. The result equals
252 :func:`detect` of the concatenated bytes, whatever the chunk boundaries. An instance is not thread-safe; use one
253 per stream.
255 :param options: the detection options; defaults to :class:`Detection` (always answer, no constraints).
256 """
258 done: bool
259 """Whether the result can no longer change: a leading byte-order mark was seen, or :meth:`close` ran."""
261 def __init__(self, options: Detection | None = None, /) -> None:
262 """Start an empty stream with the given options."""
263 self._options = options or _DEFAULT
264 self._stream = _DetectStream(self._options.tld)
265 self._result: EncodingMatch | None = None
266 self.done = False
268 @property
269 def result(self) -> EncodingMatch | None:
270 """The match :meth:`close` computed, or ``None`` while the stream is still open."""
271 return self._result
273 def feed(self, data: bytes) -> None:
274 """
275 Detect over one chunk of the stream; ignored once :attr:`done`.
277 :param data: the next bytes of the stream.
278 """
279 if self.done:
280 return
281 self.done = self._stream.feed(data)
283 def close(self) -> EncodingMatch:
284 """Read the detector's answer, cache it, and return it."""
285 if self._result is None:
286 self._result = EncodingMatch(*_rank(self._stream.close(), self._options)[0])
287 self.done = True
288 return self._result
290 def reset(self) -> None:
291 """Forget the stream and the result so the instance can start over."""
292 self._stream = _DetectStream(self._options.tld)
293 self._result = None
294 self.done = False
297def _matches(data: bytes, options: Detection) -> list[EncodingMatch]:
298 return list(starmap(EncodingMatch, _rank(_detect(data, options.tld), options)))
301def _rank(
302 result: tuple[str | None, bool, list[tuple[str, int]], bool] | None, options: Detection
303) -> list[tuple[str | None, float, str | None, bool, str | None]]:
304 return _detect_rank(result, options.allowed, options.excluded, options.language, options.threshold, _LANGUAGES)
307@dataclass(frozen=True, slots=True)
308class LanguageMatch:
309 """
310 One language-detection result: the ISO 639-3 code, a confidence, the Unicode script, and an English name.
312 ``language`` is the `ISO 639-3 <https://en.wikipedia.org/wiki/ISO_639-3>`__ code the text was written in
313 (``"eng"``, ``"deu"``, ``"cmn"``, ...), or ``None`` when the text carries no script the detector models (it is
314 empty, or all punctuation, digits, emoji, or symbols) or every candidate for its script was ruled out by
315 :class:`LanguageDetection`. ``confidence`` runs 0.0 to 1.0: it is 1.0 for a script only one modeled language uses
316 (Greek, Georgian, Korean, ...) and for a clear trigram winner, and drops toward 0.0 as the top two candidates for a
317 shared script (Latin, Cyrillic, Arabic, Devanagari, Hebrew) converge -- the usual signal that the text is too short
318 to separate them. ``script`` is the Unicode script name the text is written in (``"Latin"``, ``"Cyrillic"``,
319 ``"Han"`` reported as ``"Mandarin"``, ...), or ``None`` with a ``None`` language. ``name`` is the English name of
320 the language (``"English"``, ``"German"``, ``"Mandarin"``, ...), or ``None`` with a ``None`` language.
321 """
323 language: str | None
324 confidence: float
325 script: str | None
326 name: str | None = None
329@dataclass(frozen=True, slots=True)
330class LanguageDetection:
331 """
332 Options for :func:`detect_language`.
334 :param threshold: the confidence floor; a result below it is reported as :data:`LanguageMatch(None, 0.0, None)
335 <LanguageMatch>` so a short or ambiguous input does not masquerade as a confident answer. The default 0.0
336 always answers.
337 :param allowed: when set, only these languages (ISO 639-3 codes) may be returned; a text whose script has no
338 allowed language yields no match. Constrains every script, including the single-language ones.
339 :param excluded: these languages are never returned; mutually exclusive with ``allowed``.
340 """
342 threshold: float = 0.0
343 allowed: frozenset[str] | None = None
344 excluded: frozenset[str] = frozenset()
346 def __post_init__(self) -> None:
347 """Reject an out-of-range threshold and the contradictory allowed+excluded combination."""
348 if not 0.0 <= self.threshold <= 1.0:
349 msg = f"threshold must be within [0.0, 1.0], got {self.threshold}"
350 raise ValueError(msg)
351 if self.allowed is not None and self.excluded:
352 msg = "allowed and excluded are mutually exclusive"
353 raise ValueError(msg)
356_DEFAULT_LANGUAGE: Final = LanguageDetection()
359def detect_language(text: str, options: LanguageDetection | None = None, /) -> LanguageMatch:
360 """
361 Detect the natural language a string is written in, the ``whatlang`` / ``resiliparse`` content-language successor.
363 The detector finds the dominant Unicode script, then ranks the languages that share it by the similarity between
364 the text's most frequent character trigrams and each language's embedded profile. It reads the visible text only;
365 it does not consult an ``<html lang>`` attribute.
367 :param text: the text to classify; extract it from a document first (e.g. ``node.text``).
368 :param options: the detection options; defaults to :class:`LanguageDetection` (always answer, no constraints).
369 :returns: the best match; its ``language`` is ``None`` when the text has no modeled script, every candidate was
370 ruled out, or the confidence fell below ``threshold``.
371 :raises TypeError: when ``text`` is not a ``str``.
372 """
373 settings = options or _DEFAULT_LANGUAGE
374 return LanguageMatch(*_detect_language(text, settings.allowed, settings.excluded, settings.threshold))
377def normalize(form: NormalizationForm, text: str, /) -> str:
378 """
379 Return *text* in a Unicode normalization form (UAX #15), the C successor to :func:`unicodedata.normalize`.
381 The four forms pair a decomposition depth with whether the result is recomposed: ``NFD`` fully decomposes and
382 canonically orders combining marks, ``NFC`` decomposes then recomposes, and the ``NFK*`` forms decompose
383 compatibility equivalents too (folding ligatures, superscripts, and width variants onto their plain characters).
384 ``NFC`` is what you want to compare or store user text so ``"é"`` written as a base plus a combining accent equals
385 ``"é"`` written as one code point; ``NFKC`` additionally flattens presentation variants. A quick check returns
386 already-normalized text (the common case) without allocating.
388 :param form: the normalization form, one of ``"NFC"``, ``"NFD"``, ``"NFKC"``, ``"NFKD"``.
389 :param text: the string to normalize.
390 :returns: the normalized string, the same object when it was already normalized.
391 :raises ValueError: when *form* is not one of the four names.
392 :raises TypeError: when *text* is not a ``str``.
393 """
394 try:
395 code = _FORMS[form]
396 except KeyError:
397 msg = f"invalid normalization form {form!r}"
398 raise ValueError(msg) from None
399 return _normalize(code, text)
402def is_normalized(form: NormalizationForm, text: str, /) -> bool:
403 """
404 Return whether *text* is already in a Unicode normalization form, the :func:`unicodedata.is_normalized` peer.
406 This decides the same question as ``normalize(form, text) == text`` but the quick check settles almost every string
407 in a single scan, without building the normalized copy.
409 :param form: the normalization form, one of ``"NFC"``, ``"NFD"``, ``"NFKC"``, ``"NFKD"``.
410 :param text: the string to test.
411 :returns: ``True`` when *text* is unchanged by :func:`normalize` for *form*.
412 :raises ValueError: when *form* is not one of the four names.
413 :raises TypeError: when *text* is not a ``str``.
414 """
415 try:
416 code = _FORMS[form]
417 except KeyError:
418 msg = f"invalid normalization form {form!r}"
419 raise ValueError(msg) from None
420 return _is_normalized(code, text)