Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/__init__.py: 57%

1"""Universal character encoding detector — 0BSD-licensed rewrite.""" 

2 

3from __future__ import annotations 

4 

5from collections.abc import Iterable 

6 

7from chardet._utils import ( 

8 _DEFAULT_CHUNK_SIZE, 

9 DEFAULT_MAX_BYTES, 

10 MINIMUM_THRESHOLD, 

11 _resolve_prefer_superset, 

12 _validate_max_bytes, 

13 _warn_deprecated_chunk_size, 

14) 

15from chardet._version import __version__ 

16from chardet.detector import UniversalDetector 

17from chardet.enums import EncodingEra, LanguageFilter 

18from chardet.output_names import apply_compat_names, apply_preferred_superset 

19from chardet.pipeline import DetectionDict, DetectionResult 

20from chardet.pipeline.orchestrator import run_pipeline 

21from chardet.registry import _validate_encoding, normalize_encodings 

22 

23__all__ = [ 

24 "DEFAULT_MAX_BYTES", 

25 "MINIMUM_THRESHOLD", 

26 "DetectionDict", 

27 "DetectionResult", 

28 "EncodingEra", 

29 "LanguageFilter", 

30 "UniversalDetector", 

31 "__version__", 

32 "detect", 

33 "detect_all", 

34] 

35 

36 

37def __getattr__(name: str) -> object: 

38 """Resolve the deprecated ``chardet.equivalences`` submodule on demand. 

39 

40 Before the 7.5 split, ``chardet/__init__.py`` imported that module for 

41 its own use, which as a side effect made ``chardet.equivalences`` work 

42 without an explicit ``import chardet.equivalences``. This package now 

43 imports :mod:`chardet.output_names` instead, so the attribute is bound 

44 lazily here: importing the shim eagerly would fire its deprecation 

45 warning on every ``import chardet``. 

46 """ 

47 if name == "equivalences": 

48 # ``from chardet import equivalences`` would re-enter this function: 

49 # the import machinery probes the parent package with ``hasattr`` 

50 # first. Kept local so the shim stays off the eager import path. 

51 import importlib # noqa: PLC0415 

52 

53 return importlib.import_module("chardet.equivalences") 

54 msg = f"module {__name__!r} has no attribute {name!r}" 

55 raise AttributeError(msg) 

56 

57 

58def detect( # noqa: PLR0913 

59 byte_str: bytes | bytearray, 

60 should_rename_legacy: bool = False, 

61 encoding_era: EncodingEra = EncodingEra.ALL, 

62 chunk_size: int = _DEFAULT_CHUNK_SIZE, 

63 max_bytes: int = DEFAULT_MAX_BYTES, 

64 *, 

65 prefer_superset: bool = False, 

66 compat_names: bool = True, 

67 include_encodings: Iterable[str] | None = None, 

68 exclude_encodings: Iterable[str] | None = None, 

69 no_match_encoding: str = "cp1252", 

70 empty_input_encoding: str = "utf-8", 

71) -> DetectionDict: 

72 """Detect the encoding of the given byte string. 

73 

74 :param byte_str: The byte sequence to detect encoding for. 

75 :param should_rename_legacy: Deprecated alias for *prefer_superset*. 

76 :param encoding_era: Restrict candidate encodings to the given era. 

77 :param chunk_size: Deprecated -- accepted for backward compatibility but 

78 has no effect. 

79 :param max_bytes: Maximum number of bytes to examine from *byte_str*. 

80 :param prefer_superset: If ``True``, remap subset encodings in the result 

81 to their decode-safe Windows/CP superset equivalents (e.g., 

82 ISO-8859-1 -> Windows-1252, EUC-KR -> CP949). Recommended when the 

83 result will be used to decode: detection examines at most 

84 *max_bytes* of input, and only the superset is guaranteed to decode 

85 bytes beyond that window. The remap is skipped when the superset 

86 cannot decode the examined window itself (the Windows code pages 

87 leave a few C1 positions undefined that the ISO subsets map), so 

88 the reported name always decodes what was examined. If ``False`` 

89 (default), the detected 

90 encoding is reported under its own name --- note this only skips 

91 the renaming step; it is not a promise of the *smallest* matching 

92 encoding, since detection may natively choose a superset that fits 

93 the data better. The default will change to ``True`` in chardet 

94 8.0; pass ``False`` explicitly if you depend on subset names. 

95 :param compat_names: If ``True`` (default), return encoding names 

96 compatible with chardet 5.x/6.x. If ``False``, return raw Python 

97 codec names. 

98 :param include_encodings: If given, restrict detection to only these 

99 encodings (names or aliases). 

100 :param exclude_encodings: If given, remove these encodings from the 

101 candidate set. 

102 :param no_match_encoding: Encoding to return when no candidate survives 

103 the pipeline. Defaults to ``"cp1252"``. 

104 :param empty_input_encoding: Encoding to return for empty input. Defaults 

105 to ``"utf-8"``. 

106 :returns: A dictionary with keys ``"encoding"``, ``"confidence"``, and 

107 ``"language"``. 

108 """ 

109 _warn_deprecated_chunk_size(chunk_size) 

110 _validate_max_bytes(max_bytes) 

111 prefer_superset = _resolve_prefer_superset(should_rename_legacy, prefer_superset) 

112 include = normalize_encodings(include_encodings, "include_encodings") 

113 exclude = normalize_encodings(exclude_encodings, "exclude_encodings") 

114 no_match = _validate_encoding(no_match_encoding, "no_match_encoding") 

115 empty = _validate_encoding(empty_input_encoding, "empty_input_encoding") 

116 data = byte_str if isinstance(byte_str, bytes) else bytes(byte_str) 

117 results = run_pipeline( 

118 data, 

119 encoding_era, 

120 max_bytes=max_bytes, 

121 include_encodings=include, 

122 exclude_encodings=exclude, 

123 no_match_encoding=no_match, 

124 empty_input_encoding=empty, 

125 ) 

126 result = results[0].to_dict() 

127 if prefer_superset: 

128 apply_preferred_superset(result, data[:max_bytes]) 

129 if compat_names: 

130 apply_compat_names(result) 

131 return result 

132 

133 

134def detect_all( # noqa: PLR0913 

135 byte_str: bytes | bytearray, 

136 ignore_threshold: bool = False, 

137 should_rename_legacy: bool = False, 

138 encoding_era: EncodingEra = EncodingEra.ALL, 

139 chunk_size: int = _DEFAULT_CHUNK_SIZE, 

140 max_bytes: int = DEFAULT_MAX_BYTES, 

141 *, 

142 prefer_superset: bool = False, 

143 compat_names: bool = True, 

144 include_encodings: Iterable[str] | None = None, 

145 exclude_encodings: Iterable[str] | None = None, 

146 no_match_encoding: str = "cp1252", 

147 empty_input_encoding: str = "utf-8", 

148) -> list[DetectionDict]: 

149 """Detect all possible encodings of the given byte string. 

150 

151 When *ignore_threshold* is False (the default), results with confidence 

152 <= MINIMUM_THRESHOLD (0.20) are filtered out. If all results are below 

153 the threshold, the full unfiltered list is returned as a fallback so the 

154 caller always receives at least one result. 

155 

156 :param byte_str: The byte sequence to detect encoding for. 

157 :param ignore_threshold: If ``True``, return all candidate encodings 

158 regardless of confidence score. 

159 :param should_rename_legacy: Deprecated alias for *prefer_superset*. 

160 :param encoding_era: Restrict candidate encodings to the given era. 

161 :param chunk_size: Deprecated -- accepted for backward compatibility but 

162 has no effect. 

163 :param max_bytes: Maximum number of bytes to examine from *byte_str*. 

164 :param prefer_superset: If ``True``, remap subset encodings in the 

165 results to their decode-safe Windows/CP superset equivalents. 

166 If ``False`` (default), skip the renaming --- not a promise of the 

167 smallest matching encoding. The default will change to ``True`` 

168 in chardet 8.0. See :func:`detect` for details. 

169 :param compat_names: If ``True`` (default), return encoding names 

170 compatible with chardet 5.x/6.x. If ``False``, return raw Python 

171 codec names. 

172 :param include_encodings: If given, restrict detection to only these 

173 encodings (names or aliases). 

174 :param exclude_encodings: If given, remove these encodings from the 

175 candidate set. 

176 :param no_match_encoding: Encoding to return when no candidate survives 

177 the pipeline. Defaults to ``"cp1252"``. 

178 :param empty_input_encoding: Encoding to return for empty input. Defaults 

179 to ``"utf-8"``. 

180 :returns: A list of dictionaries, sorted by descending confidence. 

181 """ 

182 _warn_deprecated_chunk_size(chunk_size) 

183 _validate_max_bytes(max_bytes) 

184 prefer_superset = _resolve_prefer_superset(should_rename_legacy, prefer_superset) 

185 include = normalize_encodings(include_encodings, "include_encodings") 

186 exclude = normalize_encodings(exclude_encodings, "exclude_encodings") 

187 no_match = _validate_encoding(no_match_encoding, "no_match_encoding") 

188 empty = _validate_encoding(empty_input_encoding, "empty_input_encoding") 

189 data = byte_str if isinstance(byte_str, bytes) else bytes(byte_str) 

190 results = run_pipeline( 

191 data, 

192 encoding_era, 

193 max_bytes=max_bytes, 

194 include_encodings=include, 

195 exclude_encodings=exclude, 

196 no_match_encoding=no_match, 

197 empty_input_encoding=empty, 

198 full_ranking=True, 

199 ) 

200 dicts = [r.to_dict() for r in results] 

201 if not ignore_threshold: 

202 filtered = [d for d in dicts if d["confidence"] > MINIMUM_THRESHOLD] 

203 if filtered: 

204 dicts = filtered 

205 window = data[:max_bytes] 

206 for d in dicts: 

207 if prefer_superset: 

208 apply_preferred_superset(d, window) 

209 if compat_names: 

210 apply_compat_names(d) 

211 return sorted(dicts, key=lambda d: d["confidence"], reverse=True)