Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import importlib
4import logging
5import unicodedata
6from bisect import bisect_right
7from codecs import IncrementalDecoder
8from encodings.aliases import aliases
9from functools import lru_cache
10from re import findall
11from typing import Generator
13from .constant import (
14 ENCODING_MARKS,
15 IANA_SUPPORTED_SIMILAR,
16 RE_POSSIBLE_ENCODING_INDICATION,
17 UNICODE_RANGES_COMBINED,
18 _SECONDARY_RANGE_NAMES,
19 UTF8_MAXIMAL_ALLOCATION,
20 COMMON_CJK_CHARACTERS,
21 _LATIN,
22 _CJK,
23 _HANGUL,
24 _KATAKANA,
25 _HIRAGANA,
26 _THAI,
27 _ARABIC,
28 _ARABIC_ISOLATED_FORM,
29 _ACCENT_KEYWORDS,
30 _ACCENTUATED,
31)
34def _character_flags(character: str) -> int:
35 """Compute all name-based classification flags with a single unicodedata.name() call."""
36 try:
37 desc: str = unicodedata.name(character)
38 except ValueError:
39 return 0
41 flags: int = 0
43 if "LATIN" in desc:
44 flags |= _LATIN
45 if "CJK" in desc:
46 flags |= _CJK
47 if "HANGUL" in desc:
48 flags |= _HANGUL
49 if "KATAKANA" in desc:
50 flags |= _KATAKANA
51 if "HIRAGANA" in desc:
52 flags |= _HIRAGANA
53 if "THAI" in desc:
54 flags |= _THAI
55 if "ARABIC" in desc:
56 flags |= _ARABIC
57 if "ISOLATED FORM" in desc:
58 flags |= _ARABIC_ISOLATED_FORM
60 for kw in _ACCENT_KEYWORDS:
61 if kw in desc:
62 flags |= _ACCENTUATED
63 break
65 return flags
68def is_accentuated(character: str) -> bool:
69 return bool(_character_flags(character) & _ACCENTUATED)
72@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
73def remove_accent(character: str) -> str:
74 decomposed: str = unicodedata.decomposition(character)
75 if not decomposed:
76 return character
78 codes: list[str] = decomposed.split(" ")
80 return chr(int(codes[0], 16))
83# Pre-built sorted lookup table for O(log n) binary search in unicode_range().
84# Each entry is (range_start, range_end_exclusive, range_name).
85_UNICODE_RANGES_SORTED: list[tuple[int, int, str]] = sorted(
86 (ord_range.start, ord_range.stop, name)
87 for name, ord_range in UNICODE_RANGES_COMBINED.items()
88)
89_UNICODE_RANGE_STARTS: list[int] = [e[0] for e in _UNICODE_RANGES_SORTED]
92def unicode_range(character: str) -> str | None:
93 """
94 Retrieve the Unicode range official name from a single character.
95 """
96 character_ord: int = ord(character)
98 # Binary search: find the rightmost range whose start <= character_ord
99 idx = bisect_right(_UNICODE_RANGE_STARTS, character_ord) - 1
100 if idx >= 0:
101 start, stop, name = _UNICODE_RANGES_SORTED[idx]
102 if character_ord < stop:
103 return name
105 return None
108def is_latin(character: str) -> bool:
109 return bool(_character_flags(character) & _LATIN)
112def is_punctuation(character: str) -> bool:
113 character_category: str = unicodedata.category(character)
115 if "P" in character_category:
116 return True
118 character_range: str | None = unicode_range(character)
120 if character_range is None:
121 return False
123 return "Punctuation" in character_range
126def is_symbol(character: str) -> bool:
127 character_category: str = unicodedata.category(character)
129 if "S" in character_category or "N" in character_category:
130 return True
132 character_range: str | None = unicode_range(character)
134 if character_range is None:
135 return False
137 return "Forms" in character_range and character_category != "Lo"
140def is_emoticon(character: str) -> bool:
141 character_range: str | None = unicode_range(character)
143 if character_range is None:
144 return False
146 return "Emoticons" in character_range or "Pictographs" in character_range
149def is_separator(character: str) -> bool:
150 if character.isspace() or character in {"｜", "+", "<", ">"}:
151 return True
153 character_category: str = unicodedata.category(character)
155 return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
158@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
159def is_case_variable(character: str) -> bool:
160 return character.islower() != character.isupper()
163@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
164def is_cjk(character: str) -> bool:
165 return bool(_character_flags(character) & _CJK)
168@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
169def is_hiragana(character: str) -> bool:
170 return bool(_character_flags(character) & _HIRAGANA)
173@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
174def is_katakana(character: str) -> bool:
175 return bool(_character_flags(character) & _KATAKANA)
178@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
179def is_hangul(character: str) -> bool:
180 return bool(_character_flags(character) & _HANGUL)
183@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
184def is_thai(character: str) -> bool:
185 return bool(_character_flags(character) & _THAI)
188@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
189def is_arabic(character: str) -> bool:
190 return bool(_character_flags(character) & _ARABIC)
193@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
194def is_arabic_isolated_form(character: str) -> bool:
195 return bool(_character_flags(character) & _ARABIC_ISOLATED_FORM)
198@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
199def is_cjk_uncommon(character: str) -> bool:
200 return character not in COMMON_CJK_CHARACTERS
203def is_unicode_range_secondary(range_name: str) -> bool:
204 return range_name in _SECONDARY_RANGE_NAMES
207@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
208def is_unprintable(character: str) -> bool:
209 return (
210 not character.isspace() # includes \n \t \r \v
211 and not character.isprintable()
212 and character != "\x1a" # Why? Its the ASCII substitute character.
213 and character != "\ufeff" # bug discovered in Python,
214 # Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
215 )
218def any_specified_encoding(
219 sequence: bytes | bytearray, search_zone: int = 8192
220) -> str | None:
221 """
222 Extract using ASCII-only decoder any specified encoding in the first n-bytes.
223 """
224 if not isinstance(sequence, (bytes, bytearray)):
225 raise TypeError
227 seq_len: int = len(sequence)
229 decoded_zone: str = sequence[: min(seq_len, search_zone)].decode(
230 "ascii", errors="ignore"
231 )
233 # Cheap literal pre-filter.
234 lowered_zone: str = decoded_zone.lower()
235 if "coding" not in lowered_zone and "charset" not in lowered_zone:
236 return None
238 results: list[str] = findall(
239 RE_POSSIBLE_ENCODING_INDICATION,
240 decoded_zone,
241 )
243 if len(results) == 0:
244 return None
246 for specified_encoding in results:
247 specified_encoding = specified_encoding.lower().replace("-", "_")
249 encoding_alias: str
250 encoding_iana: str
252 for encoding_alias, encoding_iana in aliases.items():
253 if encoding_alias == specified_encoding:
254 return encoding_iana
255 if encoding_iana == specified_encoding:
256 return encoding_iana
258 return None
261@lru_cache(maxsize=128)
262def is_multi_byte_encoding(name: str) -> bool:
263 """
264 Verify is a specific encoding is a multi byte one based on it IANA name
265 """
266 if name in {
267 "utf_8",
268 "utf_8_sig",
269 "utf_16",
270 "utf_16_be",
271 "utf_16_le",
272 "utf_32",
273 "utf_32_le",
274 "utf_32_be",
275 "utf_7",
276 }:
277 return True
279 # Besides the Unicode family above, every multibyte codec shipped with
280 # Python is implemented by _multibytecodec through exactly one of the six
281 # cjkcodecs providers below. Probing those providers directly (getcodec)
282 # classifies a name without importing its "encodings.<name>" module:
283 # classifying the whole IANA_SUPPORTED list would otherwise import many
284 # modules and dominate "import charset_normalizer" wall time.
285 # see https://github.com/jawah/charset_normalizer/issues/742
286 for provider in (
287 "_codecs_cn",
288 "_codecs_hk",
289 "_codecs_iso2022",
290 "_codecs_jp",
291 "_codecs_kr",
292 "_codecs_tw",
293 ):
294 try:
295 importlib.import_module(provider).getcodec(name) # type: ignore[attr-defined]
296 except (ImportError, AttributeError, LookupError): # Defensive: edge cases
297 continue
298 return True
300 return False
303def identify_sig_or_bom(sequence: bytes | bytearray) -> tuple[str | None, bytes]:
304 """
305 Identify and extract SIG/BOM in given sequence.
306 """
308 for iana_encoding in ENCODING_MARKS:
309 marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
311 if isinstance(marks, bytes):
312 marks = [marks]
314 for mark in marks:
315 if sequence.startswith(mark):
316 return iana_encoding, mark
318 return None, b""
321def should_strip_sig_or_bom(iana_encoding: str) -> bool:
322 return iana_encoding not in {"utf_16", "utf_32"}
325def iana_name(cp_name: str, strict: bool = True) -> str:
326 """Returns the Python normalized encoding name (Not the IANA official name)."""
327 cp_name = cp_name.lower().replace("-", "_")
329 encoding_alias: str
330 encoding_iana: str
332 for encoding_alias, encoding_iana in aliases.items():
333 if cp_name in [encoding_alias, encoding_iana]:
334 return encoding_iana
336 if strict:
337 raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
339 return cp_name
342def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
343 if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
344 return 0.0
346 decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
347 decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
349 id_a: IncrementalDecoder = decoder_a(errors="ignore")
350 id_b: IncrementalDecoder = decoder_b(errors="ignore")
352 character_match_count: int = 0
354 for i in range(256):
355 to_be_decoded: bytes = bytes([i])
356 if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
357 character_match_count += 1
359 return character_match_count / 256
362def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
363 """
364 Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
365 the function cp_similarity.
366 """
367 return (
368 iana_name_a in IANA_SUPPORTED_SIMILAR
369 and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
370 )
373def set_logging_handler(
374 name: str = "charset_normalizer",
375 level: int = logging.INFO,
376 format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
377) -> None:
378 logger = logging.getLogger(name)
379 logger.setLevel(level)
381 handler = logging.StreamHandler()
382 handler.setFormatter(logging.Formatter(format_string))
383 logger.addHandler(handler)
386def cut_sequence_chunks(
387 sequences: bytes | bytearray,
388 encoding_iana: str,
389 offsets: range,
390 chunk_size: int,
391 bom_or_sig_available: bool,
392 strip_sig_or_bom: bool,
393 sig_payload: bytes,
394 is_multi_byte_decoder: bool,
395 decoded_payload: str | None = None,
396 deferred_decoding: bool = False,
397) -> Generator[str, None, None]:
398 if decoded_payload and not is_multi_byte_decoder:
399 for i in offsets:
400 chunk = decoded_payload[i : i + chunk_size]
401 if not chunk:
402 break
403 yield chunk
404 elif deferred_decoding:
405 # Deferred single-byte probing: the whole payload is not decoded
406 # yet. Single-byte codecs are stateless (1 byte == 1 char), hence
407 # decode(base)[i:j] == decode(base[i:j]): slicing the raw bytes
408 # yields exactly the chunks the branch above would have produced,
409 # short trailing chunks included, and raises UnicodeDecodeError on
410 # invalid bytes just like the whole-payload decode would.
411 base_bytes = (
412 sequences if not strip_sig_or_bom else sequences[len(sig_payload) :]
413 )
414 for i in offsets:
415 cut_sequence = base_bytes[i : i + chunk_size]
416 if not cut_sequence:
417 break
418 yield str(cut_sequence, encoding_iana)
419 else:
420 for i in offsets:
421 chunk_end = i + chunk_size
422 if chunk_end > len(sequences) + 8:
423 continue
425 cut_sequence = sequences[i : i + chunk_size]
427 if bom_or_sig_available and not strip_sig_or_bom:
428 cut_sequence = sig_payload + cut_sequence
430 chunk = cut_sequence.decode(
431 encoding_iana,
432 errors="ignore" if is_multi_byte_decoder else "strict",
433 )
435 # multi-byte bad cutting detector and adjustment
436 # not the cleanest way to perform that fix but clever enough for now.
437 if is_multi_byte_decoder and i > 0:
438 chunk_partial_size_chk: int = min(chunk_size, 16)
440 if (
441 decoded_payload
442 and chunk[:chunk_partial_size_chk] not in decoded_payload
443 ):
444 for j in range(i, i - 4, -1):
445 cut_sequence = sequences[j:chunk_end]
447 if bom_or_sig_available and not strip_sig_or_bom:
448 cut_sequence = sig_payload + cut_sequence
450 chunk = cut_sequence.decode(encoding_iana, errors="ignore")
452 if chunk[:chunk_partial_size_chk] in decoded_payload:
453 break
455 yield chunk