Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/_utils.py: 31%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Internal shared utilities for chardet."""
3from __future__ import annotations
5import codecs
6import warnings
7from collections.abc import Callable
9#: Default maximum number of bytes to examine during detection.
10DEFAULT_MAX_BYTES: int = 200_000
12#: Evidence cap: how much of the examination window the candidate-filtering,
13#: validation, and probing stages consume before their answer is considered
14#: converged (ADR-0006). Exhaustive checks (BOM, magic, UTF-8, ASCII,
15#: binary, escape presence) take no cap. Must stay >= DEFAULT_MAX_BYTES so
16#: every call using the default window is provably unaffected; a test
17#: asserts the invariant.
18EVIDENCE_CAP_BYTES: int = 256 * 1024
20#: Chunk size for whole-window validity decodes. Large enough that
21#: per-chunk overhead vanishes, small enough that the transient decoded
22#: ``str`` stays bounded regardless of input size.
23_DECODE_CHUNK_SIZE: int = 1 << 20
25#: Default minimum confidence threshold for filtering results.
26MINIMUM_THRESHOLD: float = 0.20
28#: Default chunk_size value (deprecated, kept for backward-compat signatures).
29_DEFAULT_CHUNK_SIZE: int = 65_536
31#: Cache of incremental-decoder classes, keyed by encoding name.
32#:
33#: :func:`decodes_without_error` is called once per candidate encoding per
34#: detection -- roughly 86 times for :attr:`~chardet.enums.EncodingEra.ALL` --
35#: and the factory lookup is a sixth of each call. The class is immutable and
36#: shared; only the per-call *instance* carries decoder state, so caching the
37#: class is thread-safe while caching an instance would not be.
38#:
39#: Only successful lookups are cached, which bounds the cache by the number of
40#: installed codecs. A failed lookup returns ``False`` without storing
41#: anything, so an unbounded stream of bogus names (a charset declaration in
42#: markup is attacker-controlled) cannot grow it.
43_INCREMENTAL_DECODERS: dict[str, Callable[..., codecs.IncrementalDecoder]] = {}
46def _warn_deprecated_chunk_size(chunk_size: int, stacklevel: int = 3) -> None:
47 """Emit a deprecation warning if *chunk_size* differs from the default."""
48 if chunk_size != _DEFAULT_CHUNK_SIZE:
49 warnings.warn(
50 "chunk_size is not used in this version of chardet and will be ignored",
51 DeprecationWarning,
52 stacklevel=stacklevel,
53 )
56def _incremental_decoder(
57 encoding: str,
58) -> codecs.IncrementalDecoder | None:
59 """Return a fresh incremental decoder for *encoding*, or ``None``.
61 The class lookup is cached; ``None`` means the codec does not exist.
62 Every decode below catches ``UnicodeError`` rather than
63 ``UnicodeDecodeError`` because the utf-16/utf-32 incremental decoders
64 raise a bare ``UnicodeError`` when the stream does not start with a BOM.
65 """
66 decoder_class = _INCREMENTAL_DECODERS.get(encoding)
67 if decoder_class is None:
68 try:
69 decoder_class = codecs.getincrementaldecoder(encoding)
70 except LookupError:
71 return None
72 _INCREMENTAL_DECODERS[encoding] = decoder_class
73 return decoder_class()
76def count_deleted(data: bytes, table: bytes) -> int:
77 """Count how many bytes of *data* a deletion *table* would remove.
79 ``bytes.translate`` with a deletion table allocates an output buffer
80 proportional to the input, so counting a whole large window through one
81 call spikes memory by roughly the input size. Chunking keeps the
82 transient bounded at one C call per chunk, and the count is identical:
83 deletion is per-byte and carries no state across the split.
85 :param data: The raw byte data to scan.
86 :param table: Byte values to delete.
87 :returns: The number of bytes of *data* present in *table*.
88 """
89 if len(data) <= _DECODE_CHUNK_SIZE:
90 return len(data) - len(data.translate(None, table))
91 count = 0
92 for pos in range(0, len(data), _DECODE_CHUNK_SIZE):
93 chunk = data[pos : pos + _DECODE_CHUNK_SIZE]
94 count += len(chunk) - len(chunk.translate(None, table))
95 return count
98def decodes_without_error(data: bytes, encoding: str) -> bool:
99 """Return ``True`` if *data* decodes cleanly under *encoding*.
101 Equivalent to ``data.decode(encoding, errors="strict")`` except that an
102 incomplete multi-byte sequence at the *end* of *data* is accepted instead of
103 raising.
105 Detection input is nearly always a prefix of a larger whole. Callers pass
106 the first N bytes of a file, and chardet slices further on its own.
107 For a two-byte encoding any of those cuts lands mid-character roughly half
108 the time.
110 A one-shot strict decode cannot tell a truncated tail from corrupt data: it
111 raises either way, so the candidate is discarded and every CJK encoding can
112 disappear from the candidate set over a single dangling lead byte. An
113 incremental decoder with ``final=False`` defers the partial tail instead,
114 while still raising on genuine corruption anywhere before it.
116 Large inputs are fed in chunks and the decoded text is discarded, so a
117 whole-window check never allocates a matching ``str``. Chunking is
118 transparent: the decoder carries its state across calls, so a sequence
119 straddling a chunk edge decodes exactly as it would in one call.
121 :param data: The raw byte data to test.
122 :param encoding: Name of the codec to test *data* against.
123 :returns: ``True`` if *data* decodes without error, ``False`` otherwise.
124 """
125 decoder = _incremental_decoder(encoding)
126 if decoder is None:
127 return False
128 try:
129 if len(data) <= _DECODE_CHUNK_SIZE:
130 decoder.decode(data, final=False)
131 else:
132 for pos in range(0, len(data), _DECODE_CHUNK_SIZE):
133 decoder.decode(data[pos : pos + _DECODE_CHUNK_SIZE], final=False)
134 except UnicodeError:
135 return False
136 return True
139def decodes_completely(data: bytes, encoding: str) -> bool:
140 """Return ``True`` if *data* decodes under *encoding* with nothing deferred.
142 The strict sibling of :func:`decodes_without_error`: ``final=True`` makes
143 an incomplete multi-byte sequence at the end of *data* an error rather
144 than a deferred tail. This is the question that matters when *data* is
145 the caller's entire input --- ``data.decode(encoding)`` will make exactly
146 this judgment.
148 :param data: The raw byte data to test.
149 :param encoding: Name of the codec to test *data* against.
150 :returns: ``True`` if *data* decodes completely, ``False`` otherwise.
151 """
152 decoder = _incremental_decoder(encoding)
153 if decoder is None:
154 return False
155 try:
156 decoder.decode(data, final=True)
157 except UnicodeError:
158 return False
159 return True
162def dangling_tail_with_ascii_prefix(data: bytes, encoding: str) -> bool:
163 """Return ``True`` if *data* is ASCII text plus an incomplete tail.
165 One decode pass answers both halves of the decode-safety question:
166 the tolerant (``final=False``) decode yields the text before any
167 deferred tail, and flushing the decoder afterwards raises exactly when
168 a deferred tail existed. True means the candidate decoded real ASCII
169 characters and then hit an incomplete multi-byte sequence at the end
170 --- its only *non-ASCII* evidence is the undecodable tail itself.
172 Deliberately False when the tolerant decode yields nothing at all
173 (the entire input is one dangling sequence): that candidate has zero
174 decoded evidence, not ASCII evidence, and a clipped multi-byte
175 fragment is better served by the ranking's own judgment.
177 :param data: The raw byte data to test.
178 :param encoding: Name of the codec to decode *data* with.
179 :returns: ``True`` if *data* decodes to non-empty pure ASCII with an
180 incomplete sequence deferred at the end.
181 """
182 decoder = _incremental_decoder(encoding)
183 if decoder is None:
184 return False
185 try:
186 text = decoder.decode(data, final=False)
187 except UnicodeError:
188 return False
189 if not text or not text.isascii():
190 return False
191 try:
192 decoder.decode(b"", final=True)
193 except UnicodeError:
194 return True
195 return False
198def _validate_max_bytes(max_bytes: int) -> None:
199 """Raise ValueError if *max_bytes* is not a positive integer."""
200 if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes < 1:
201 msg = "max_bytes must be a positive integer"
202 raise ValueError(msg)
205def _resolve_prefer_superset(
206 should_rename_legacy: bool, prefer_superset: bool, stacklevel: int = 3
207) -> bool:
208 """Resolve the deprecated *should_rename_legacy* into *prefer_superset*."""
209 if should_rename_legacy:
210 warnings.warn(
211 "should_rename_legacy is deprecated, use prefer_superset instead",
212 DeprecationWarning,
213 stacklevel=stacklevel,
214 )
215 return True
216 return prefer_superset
219#: Mapping from ISO 639-1 language codes to English names.
220#: Includes ``"und"`` (ISO 639-3 "Undetermined") for use when language is unknown.
221ISO_TO_LANGUAGE: dict[str, str] = {
222 "ar": "arabic",
223 "be": "belarusian",
224 "bg": "bulgarian",
225 "br": "breton",
226 "cs": "czech",
227 "cy": "welsh",
228 "da": "danish",
229 "de": "german",
230 "el": "greek",
231 "en": "english",
232 "eo": "esperanto",
233 "es": "spanish",
234 "et": "estonian",
235 "fa": "farsi",
236 "fi": "finnish",
237 "fr": "french",
238 "ga": "irish",
239 "gd": "gaelic",
240 "he": "hebrew",
241 "hr": "croatian",
242 "hu": "hungarian",
243 "id": "indonesian",
244 "is": "icelandic",
245 "it": "italian",
246 "ja": "japanese",
247 "kk": "kazakh",
248 "ko": "korean",
249 "lt": "lithuanian",
250 "lv": "latvian",
251 "mk": "macedonian",
252 "ms": "malay",
253 "mt": "maltese",
254 "nl": "dutch",
255 "no": "norwegian",
256 "pl": "polish",
257 "pt": "portuguese",
258 "ro": "romanian",
259 "ru": "russian",
260 "sk": "slovak",
261 "sl": "slovene",
262 "sr": "serbian",
263 "sv": "swedish",
264 "tg": "tajik",
265 "th": "thai",
266 "tr": "turkish",
267 "uk": "ukrainian",
268 "und": "undetermined",
269 "ur": "urdu",
270 "vi": "vietnamese",
271 "zh": "chinese",
272}