Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from encodings.aliases import aliases
4from re import sub
5from typing import Any, Iterator, List, Tuple
7from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE
8from .utils import iana_name, is_multi_byte_encoding, unicode_range
11class CharsetMatch:
12 def __init__(
13 self,
14 payload: bytes | bytearray,
15 guessed_encoding: str,
16 mean_mess_ratio: float,
17 has_sig_or_bom: bool,
18 languages: CoherenceMatches,
19 decoded_payload: str | None = None,
20 preemptive_declaration: str | None = None,
21 ):
22 self._payload: bytes | bytearray = payload
24 self._encoding: str = guessed_encoding
25 self._mean_mess_ratio: float = mean_mess_ratio
26 self._languages: CoherenceMatches = languages
27 self._has_sig_or_bom: bool = has_sig_or_bom
28 self._unicode_ranges: list[str] | None = None
30 self._leaves: list[CharsetMatch] = []
31 self._mean_coherence_ratio: float = 0.0
33 self._output_payload: bytes | None = None
34 self._output_encoding: str | None = None
36 self._string: str | None = decoded_payload
38 self._preemptive_declaration: str | None = preemptive_declaration
40 def __eq__(self, other: object) -> bool:
41 if not isinstance(other, CharsetMatch):
42 if isinstance(other, str):
43 return iana_name(other) == self.encoding
44 return False
45 return self.encoding == other.encoding and self.fingerprint == other.fingerprint
47 def __lt__(self, other: object) -> bool:
48 """
49 Implemented to make sorted available upon CharsetMatches items.
50 """
51 if not isinstance(other, CharsetMatch):
52 raise ValueError
54 chaos_difference: float = abs(self.chaos - other.chaos)
55 coherence_difference: float = abs(self.coherence - other.coherence)
57 # Below 0.5% difference --> Use Coherence
58 if chaos_difference < 0.005 and coherence_difference > 0.02:
59 return self.coherence > other.coherence
60 elif chaos_difference < 0.005 and coherence_difference <= 0.02:
61 # When having a difficult decision, use the result that decoded as many multi-byte as possible.
62 # preserve RAM usage!
63 if len(self._payload) >= TOO_BIG_SEQUENCE:
64 return self.chaos < other.chaos
65 return self.multi_byte_usage > other.multi_byte_usage
67 return self.chaos < other.chaos
69 @property
70 def multi_byte_usage(self) -> float:
71 return 1.0 - (len(str(self)) / len(self.raw))
73 def __str__(self) -> str:
74 # Lazy Str Loading
75 if self._string is None:
76 self._string = str(self._payload, self._encoding, "strict")
77 # UTF-7 BOM is encoded in modified Base64 whose byte boundary
78 # can overlap with the next character, so raw-byte stripping
79 # is unreliable. Strip the decoded BOM character instead.
80 if (
81 self._has_sig_or_bom
82 and self._encoding == "utf_7"
83 and self._string
84 and self._string[0] == "\ufeff"
85 ):
86 self._string = self._string[1:]
87 return self._string
89 def __repr__(self) -> str:
90 return f"<CharsetMatch '{self.encoding}' fp({self.fingerprint})>"
92 def add_submatch(self, other: CharsetMatch) -> None:
93 if not isinstance(other, CharsetMatch) or other == self:
94 raise ValueError(
95 "Unable to add instance <{}> as a submatch of a CharsetMatch".format(
96 other.__class__
97 )
98 )
100 other._string = None # Unload RAM usage; dirty trick.
101 self._leaves.append(other)
103 @property
104 def encoding(self) -> str:
105 return self._encoding
107 @property
108 def encoding_aliases(self) -> list[str]:
109 """
110 Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
111 """
112 also_known_as: list[str] = []
113 for u, p in aliases.items():
114 if self.encoding == u:
115 also_known_as.append(p)
116 elif self.encoding == p:
117 also_known_as.append(u)
118 return also_known_as
120 @property
121 def bom(self) -> bool:
122 return self._has_sig_or_bom
124 @property
125 def byte_order_mark(self) -> bool:
126 return self._has_sig_or_bom
128 @property
129 def languages(self) -> list[str]:
130 """
131 Return the complete list of possible languages found in decoded sequence.
132 Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
133 """
134 return [e[0] for e in self._languages]
136 @property
137 def language(self) -> str:
138 """
139 Most probable language found in decoded sequence. If none were detected or inferred, the property will return
140 "Unknown".
141 """
142 if not self._languages:
143 # Trying to infer the language based on the given encoding
144 # Its either English or we should not pronounce ourselves in certain cases.
145 if "ascii" in self.could_be_from_charset:
146 return "English"
148 # doing it there to avoid circular import
149 from charset_normalizer.cd import encoding_languages, mb_encoding_languages
151 languages = (
152 mb_encoding_languages(self.encoding)
153 if is_multi_byte_encoding(self.encoding)
154 else encoding_languages(self.encoding)
155 )
157 if len(languages) == 0 or "Latin Based" in languages:
158 return "Unknown"
160 return languages[0]
162 return self._languages[0][0]
164 @property
165 def chaos(self) -> float:
166 return self._mean_mess_ratio
168 @property
169 def coherence(self) -> float:
170 if not self._languages:
171 return 0.0
172 return self._languages[0][1]
174 @property
175 def percent_chaos(self) -> float:
176 return round(self.chaos * 100, ndigits=3)
178 @property
179 def percent_coherence(self) -> float:
180 return round(self.coherence * 100, ndigits=3)
182 @property
183 def raw(self) -> bytes | bytearray:
184 """
185 Original untouched bytes.
186 """
187 return self._payload
189 @property
190 def submatch(self) -> list[CharsetMatch]:
191 return self._leaves
193 @property
194 def has_submatch(self) -> bool:
195 return len(self._leaves) > 0
197 @property
198 def alphabets(self) -> list[str]:
199 if self._unicode_ranges is not None:
200 return self._unicode_ranges
201 # list detected ranges
202 detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)]
203 # filter and sort
204 self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
205 return self._unicode_ranges
207 @property
208 def could_be_from_charset(self) -> list[str]:
209 """
210 The complete list of encoding that output the exact SAME str result and therefore could be the originating
211 encoding.
212 This list does include the encoding available in property 'encoding'.
213 """
214 return [self._encoding] + [m.encoding for m in self._leaves]
216 def output(self, encoding: str = "utf_8") -> bytes:
217 """
218 Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
219 Any errors will be simply ignored by the encoder NOT replaced.
220 """
221 if self._output_encoding is None or self._output_encoding != encoding:
222 self._output_encoding = encoding
223 decoded_string = str(self)
224 if (
225 self._preemptive_declaration is not None
226 and self._preemptive_declaration.lower()
227 not in ["utf-8", "utf8", "utf_8"]
228 ):
229 patched_header = sub(
230 RE_POSSIBLE_ENCODING_INDICATION,
231 lambda m: m.string[m.span()[0] : m.span()[1]].replace(
232 m.groups()[0],
233 iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type]
234 ),
235 decoded_string[:8192],
236 count=1,
237 )
239 decoded_string = patched_header + decoded_string[8192:]
241 self._output_payload = decoded_string.encode(encoding, "replace")
243 return self._output_payload # type: ignore
245 @property
246 def fingerprint(self) -> int:
247 """
248 Retrieve a hash fingerprint of the decoded payload, used for deduplication.
249 """
250 return hash(str(self))
253class CharsetMatches:
254 """
255 Container with every CharsetMatch items ordered by default from most probable to the less one.
256 Act like a list(iterable) but does not implements all related methods.
257 """
259 def __init__(self, results: list[CharsetMatch] | None = None):
260 self._results: list[CharsetMatch] = sorted(results) if results else []
262 def __iter__(self) -> Iterator[CharsetMatch]:
263 yield from self._results
265 def __getitem__(self, item: int | str) -> CharsetMatch:
266 """
267 Retrieve a single item either by its position or encoding name (alias may be used here).
268 Raise KeyError upon invalid index or encoding not present in results.
269 """
270 if isinstance(item, int):
271 return self._results[item]
272 if isinstance(item, str):
273 item = iana_name(item, False)
274 for result in self._results:
275 if item in result.could_be_from_charset:
276 return result
277 raise KeyError
279 def __len__(self) -> int:
280 return len(self._results)
282 def __bool__(self) -> bool:
283 return len(self._results) > 0
285 def append(self, item: CharsetMatch) -> None:
286 """
287 Insert a single match. Will be inserted accordingly to preserve sort.
288 Can be inserted as a submatch.
289 """
290 if not isinstance(item, CharsetMatch):
291 raise ValueError(
292 "Cannot append instance '{}' to CharsetMatches".format(
293 str(item.__class__)
294 )
295 )
296 # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
297 if len(item.raw) < TOO_BIG_SEQUENCE:
298 for match in self._results:
299 if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
300 match.add_submatch(item)
301 return
302 self._results.append(item)
303 self._results = sorted(self._results)
305 def best(self) -> CharsetMatch | None:
306 """
307 Simply return the first match. Strict equivalent to matches[0].
308 """
309 if not self._results:
310 return None
311 return self._results[0]
313 def first(self) -> CharsetMatch | None:
314 """
315 Redundant method, call the method best(). Kept for BC reasons.
316 """
317 return self.best()
320CoherenceMatch = Tuple[str, float]
321CoherenceMatches = List[CoherenceMatch]
324class CliDetectionResult:
325 def __init__(
326 self,
327 path: str,
328 encoding: str | None,
329 encoding_aliases: list[str],
330 alternative_encodings: list[str],
331 language: str,
332 alphabets: list[str],
333 has_sig_or_bom: bool,
334 chaos: float,
335 coherence: float,
336 unicode_path: str | None,
337 is_preferred: bool,
338 ):
339 self.path: str = path
340 self.unicode_path: str | None = unicode_path
341 self.encoding: str | None = encoding
342 self.encoding_aliases: list[str] = encoding_aliases
343 self.alternative_encodings: list[str] = alternative_encodings
344 self.language: str = language
345 self.alphabets: list[str] = alphabets
346 self.has_sig_or_bom: bool = has_sig_or_bom
347 self.chaos: float = chaos
348 self.coherence: float = coherence
349 self.is_preferred: bool = is_preferred
351 @property
352 def __dict__(self) -> dict[str, Any]: # type: ignore
353 return {
354 "path": self.path,
355 "encoding": self.encoding,
356 "encoding_aliases": self.encoding_aliases,
357 "alternative_encodings": self.alternative_encodings,
358 "language": self.language,
359 "alphabets": self.alphabets,
360 "has_sig_or_bom": self.has_sig_or_bom,
361 "chaos": self.chaos,
362 "coherence": self.coherence,
363 "unicode_path": self.unicode_path,
364 "is_preferred": self.is_preferred,
365 }
367 def to_json(self) -> str:
368 from json import dumps
370 return dumps(self.__dict__, ensure_ascii=True, indent=4)