Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/__init__.py: 79%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Universal character encoding detector — 0BSD-licensed rewrite."""
3from __future__ import annotations
5from collections.abc import Iterable
7from chardet._request import _DEFAULT_CHUNK_SIZE, DetectionRequest
8from chardet._utils import DEFAULT_MAX_BYTES, MINIMUM_THRESHOLD
9from chardet._version import __version__
10from chardet.detector import UniversalDetector
11from chardet.enums import EncodingEra, LanguageFilter
12from chardet.pipeline import DetectionDict, DetectionResult
14__all__ = [
15 "DEFAULT_MAX_BYTES",
16 "MINIMUM_THRESHOLD",
17 "DetectionDict",
18 "DetectionResult",
19 "EncodingEra",
20 "LanguageFilter",
21 "UniversalDetector",
22 "__version__",
23 "detect",
24 "detect_all",
25]
28def __getattr__(name: str) -> object:
29 """Resolve the deprecated ``chardet.equivalences`` submodule on demand.
31 Before the 7.5 split, ``chardet/__init__.py`` imported that module for
32 its own use, which as a side effect made ``chardet.equivalences`` work
33 without an explicit ``import chardet.equivalences``. This package now
34 imports :mod:`chardet.output_names` instead, so the attribute is bound
35 lazily here: importing the shim eagerly would fire its deprecation
36 warning on every ``import chardet``.
37 """
38 if name == "equivalences":
39 # ``from chardet import equivalences`` would re-enter this function:
40 # the import machinery probes the parent package with ``hasattr``
41 # first. Kept local so the shim stays off the eager import path.
42 import importlib # noqa: PLC0415
44 return importlib.import_module("chardet.equivalences")
45 msg = f"module {__name__!r} has no attribute {name!r}"
46 raise AttributeError(msg)
49def detect( # noqa: PLR0913
50 byte_str: bytes | bytearray,
51 should_rename_legacy: bool = False,
52 encoding_era: EncodingEra = EncodingEra.ALL,
53 chunk_size: int = _DEFAULT_CHUNK_SIZE,
54 max_bytes: int = DEFAULT_MAX_BYTES,
55 *,
56 prefer_superset: bool = False,
57 compat_names: bool = True,
58 include_encodings: Iterable[str] | None = None,
59 exclude_encodings: Iterable[str] | None = None,
60 no_match_encoding: str = "cp1252",
61 empty_input_encoding: str = "utf-8",
62) -> DetectionDict:
63 """Detect the encoding of the given byte string.
65 :param byte_str: The byte sequence to detect encoding for.
66 :param should_rename_legacy: Deprecated alias for *prefer_superset*.
67 :param encoding_era: Restrict candidate encodings to the given era.
68 :param chunk_size: Deprecated -- accepted for backward compatibility but
69 has no effect.
70 :param max_bytes: Maximum number of bytes to examine from *byte_str*.
71 :param prefer_superset: If ``True``, remap subset encodings in the result
72 to their decode-safe Windows/CP superset equivalents (e.g.,
73 ISO-8859-1 -> Windows-1252, EUC-KR -> CP949). Recommended when the
74 result will be used to decode: detection examines at most
75 *max_bytes* of input, and only the superset is guaranteed to decode
76 bytes beyond that window. The remap is skipped when the superset
77 cannot decode the examined window itself (the Windows code pages
78 leave a few C1 positions undefined that the ISO subsets map), so
79 the reported name always decodes what was examined. If ``False``
80 (default), the detected
81 encoding is reported under its own name --- note this only skips
82 the renaming step; it is not a promise of the *smallest* matching
83 encoding, since detection may natively choose a superset that fits
84 the data better. The default will change to ``True`` in chardet
85 8.0; pass ``False`` explicitly if you depend on subset names.
86 :param compat_names: If ``True`` (default), return encoding names
87 compatible with chardet 5.x/6.x. If ``False``, return raw Python
88 codec names.
89 :param include_encodings: If given, restrict detection to only these
90 encodings (names or aliases).
91 :param exclude_encodings: If given, remove these encodings from the
92 candidate set.
93 :param no_match_encoding: Encoding to return when no candidate survives
94 the pipeline. Defaults to ``"cp1252"``. It is returned as given,
95 at confidence 0.10, and is the one answer that is not checked
96 against the data: it is the caller's sentinel for "nothing fit".
97 :param empty_input_encoding: Encoding to return for empty input. Defaults
98 to ``"utf-8"``.
99 :returns: A dictionary with keys ``"encoding"``, ``"confidence"``,
100 ``"language"``, and ``"mime_type"``.
101 """
102 request = DetectionRequest.resolve(
103 should_rename_legacy=should_rename_legacy,
104 encoding_era=encoding_era,
105 chunk_size=chunk_size,
106 max_bytes=max_bytes,
107 prefer_superset=prefer_superset,
108 compat_names=compat_names,
109 include_encodings=include_encodings,
110 exclude_encodings=exclude_encodings,
111 no_match_encoding=no_match_encoding,
112 empty_input_encoding=empty_input_encoding,
113 stacklevel=3,
114 )
115 data = byte_str if isinstance(byte_str, bytes) else bytes(byte_str)
116 return request.render(request.run(data), data[:max_bytes])[0]
119def detect_all( # noqa: PLR0913
120 byte_str: bytes | bytearray,
121 ignore_threshold: bool = False,
122 should_rename_legacy: bool = False,
123 encoding_era: EncodingEra = EncodingEra.ALL,
124 chunk_size: int = _DEFAULT_CHUNK_SIZE,
125 max_bytes: int = DEFAULT_MAX_BYTES,
126 *,
127 prefer_superset: bool = False,
128 compat_names: bool = True,
129 include_encodings: Iterable[str] | None = None,
130 exclude_encodings: Iterable[str] | None = None,
131 no_match_encoding: str = "cp1252",
132 empty_input_encoding: str = "utf-8",
133) -> list[DetectionDict]:
134 """Detect all possible encodings of the given byte string.
136 When *ignore_threshold* is False (the default), results with confidence
137 <= MINIMUM_THRESHOLD (0.20) are filtered out. If all results are below
138 the threshold, the full unfiltered list is returned as a fallback so the
139 caller always receives at least one result.
141 :param byte_str: The byte sequence to detect encoding for.
142 :param ignore_threshold: If ``True``, return all candidate encodings
143 regardless of confidence score.
144 :param should_rename_legacy: Deprecated alias for *prefer_superset*.
145 :param encoding_era: Restrict candidate encodings to the given era.
146 :param chunk_size: Deprecated -- accepted for backward compatibility but
147 has no effect.
148 :param max_bytes: Maximum number of bytes to examine from *byte_str*.
149 :param prefer_superset: If ``True``, remap subset encodings in the
150 results to their decode-safe Windows/CP superset equivalents.
151 If ``False`` (default), skip the renaming --- not a promise of the
152 smallest matching encoding. The default will change to ``True``
153 in chardet 8.0. See :func:`detect` for details.
154 :param compat_names: If ``True`` (default), return encoding names
155 compatible with chardet 5.x/6.x. If ``False``, return raw Python
156 codec names.
157 :param include_encodings: If given, restrict detection to only these
158 encodings (names or aliases).
159 :param exclude_encodings: If given, remove these encodings from the
160 candidate set.
161 :param no_match_encoding: Encoding to return when no candidate survives
162 the pipeline. Defaults to ``"cp1252"``.
163 :param empty_input_encoding: Encoding to return for empty input. Defaults
164 to ``"utf-8"``.
165 :returns: A list of dictionaries, sorted by descending confidence, one
166 per encoding name.
167 """
168 request = DetectionRequest.resolve(
169 should_rename_legacy=should_rename_legacy,
170 encoding_era=encoding_era,
171 chunk_size=chunk_size,
172 max_bytes=max_bytes,
173 prefer_superset=prefer_superset,
174 compat_names=compat_names,
175 include_encodings=include_encodings,
176 exclude_encodings=exclude_encodings,
177 no_match_encoding=no_match_encoding,
178 empty_input_encoding=empty_input_encoding,
179 stacklevel=3,
180 )
181 data = byte_str if isinstance(byte_str, bytes) else bytes(byte_str)
182 return request.render(
183 request.run(data, full_ranking=True),
184 data[:max_bytes],
185 all_results=True,
186 ignore_threshold=ignore_threshold,
187 )