Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import importlib
4import logging
5import unicodedata
6from codecs import IncrementalDecoder
7from encodings.aliases import aliases
8from functools import lru_cache
9from re import findall
10from typing import Generator
12from _multibytecodec import ( # type: ignore[import-not-found,import]
13 MultibyteIncrementalDecoder,
14)
16from .constant import (
17 ENCODING_MARKS,
18 IANA_SUPPORTED_SIMILAR,
19 RE_POSSIBLE_ENCODING_INDICATION,
20 UNICODE_RANGES_COMBINED,
21 UNICODE_SECONDARY_RANGE_KEYWORD,
22 UTF8_MAXIMAL_ALLOCATION,
23 COMMON_CJK_CHARACTERS,
24)
27@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
28def is_accentuated(character: str) -> bool:
29 try:
30 description: str = unicodedata.name(character)
31 except ValueError: # Defensive: unicode database outdated?
32 return False
33 return (
34 "WITH GRAVE" in description
35 or "WITH ACUTE" in description
36 or "WITH CEDILLA" in description
37 or "WITH DIAERESIS" in description
38 or "WITH CIRCUMFLEX" in description
39 or "WITH TILDE" in description
40 or "WITH MACRON" in description
41 or "WITH RING ABOVE" in description
42 )
45@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
46def remove_accent(character: str) -> str:
47 decomposed: str = unicodedata.decomposition(character)
48 if not decomposed:
49 return character
51 codes: list[str] = decomposed.split(" ")
53 return chr(int(codes[0], 16))
56@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
57def unicode_range(character: str) -> str | None:
58 """
59 Retrieve the Unicode range official name from a single character.
60 """
61 character_ord: int = ord(character)
63 for range_name, ord_range in UNICODE_RANGES_COMBINED.items():
64 if character_ord in ord_range:
65 return range_name
67 return None
70@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
71def is_latin(character: str) -> bool:
72 try:
73 description: str = unicodedata.name(character)
74 except ValueError: # Defensive: unicode database outdated?
75 return False
76 return "LATIN" in description
79@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
80def is_punctuation(character: str) -> bool:
81 character_category: str = unicodedata.category(character)
83 if "P" in character_category:
84 return True
86 character_range: str | None = unicode_range(character)
88 if character_range is None:
89 return False
91 return "Punctuation" in character_range
94@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
95def is_symbol(character: str) -> bool:
96 character_category: str = unicodedata.category(character)
98 if "S" in character_category or "N" in character_category:
99 return True
101 character_range: str | None = unicode_range(character)
103 if character_range is None:
104 return False
106 return "Forms" in character_range and character_category != "Lo"
109@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
110def is_emoticon(character: str) -> bool:
111 character_range: str | None = unicode_range(character)
113 if character_range is None:
114 return False
116 return "Emoticons" in character_range or "Pictographs" in character_range
119@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
120def is_separator(character: str) -> bool:
121 if character.isspace() or character in {"｜", "+", "<", ">"}:
122 return True
124 character_category: str = unicodedata.category(character)
126 return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
129@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
130def is_case_variable(character: str) -> bool:
131 return character.islower() != character.isupper()
134@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
135def is_cjk(character: str) -> bool:
136 try:
137 character_name = unicodedata.name(character)
138 except ValueError: # Defensive: unicode database outdated?
139 return False
141 return "CJK" in character_name
144@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
145def is_hiragana(character: str) -> bool:
146 try:
147 character_name = unicodedata.name(character)
148 except ValueError: # Defensive: unicode database outdated?
149 return False
151 return "HIRAGANA" in character_name
154@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
155def is_katakana(character: str) -> bool:
156 try:
157 character_name = unicodedata.name(character)
158 except ValueError: # Defensive: unicode database outdated?
159 return False
161 return "KATAKANA" in character_name
164@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
165def is_hangul(character: str) -> bool:
166 try:
167 character_name = unicodedata.name(character)
168 except ValueError: # Defensive: unicode database outdated?
169 return False
171 return "HANGUL" in character_name
174@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
175def is_thai(character: str) -> bool:
176 try:
177 character_name = unicodedata.name(character)
178 except ValueError: # Defensive: unicode database outdated?
179 return False
181 return "THAI" in character_name
184@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
185def is_arabic(character: str) -> bool:
186 try:
187 character_name = unicodedata.name(character)
188 except ValueError: # Defensive: unicode database outdated?
189 return False
191 return "ARABIC" in character_name
194@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
195def is_arabic_isolated_form(character: str) -> bool:
196 try:
197 character_name = unicodedata.name(character)
198 except ValueError: # Defensive: unicode database outdated?
199 return False
201 return "ARABIC" in character_name and "ISOLATED FORM" in character_name
204@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
205def is_cjk_uncommon(character: str) -> bool:
206 return character not in COMMON_CJK_CHARACTERS
209@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
210def is_unicode_range_secondary(range_name: str) -> bool:
211 return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
214@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
215def is_unprintable(character: str) -> bool:
216 return (
217 character.isspace() is False # includes \n \t \r \v
218 and character.isprintable() is False
219 and character != "\x1a" # Why? Its the ASCII substitute character.
220 and character != "\ufeff" # bug discovered in Python,
221 # Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
222 )
225def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> str | None:
226 """
227 Extract using ASCII-only decoder any specified encoding in the first n-bytes.
228 """
229 if not isinstance(sequence, bytes):
230 raise TypeError
232 seq_len: int = len(sequence)
234 results: list[str] = findall(
235 RE_POSSIBLE_ENCODING_INDICATION,
236 sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"),
237 )
239 if len(results) == 0:
240 return None
242 for specified_encoding in results:
243 specified_encoding = specified_encoding.lower().replace("-", "_")
245 encoding_alias: str
246 encoding_iana: str
248 for encoding_alias, encoding_iana in aliases.items():
249 if encoding_alias == specified_encoding:
250 return encoding_iana
251 if encoding_iana == specified_encoding:
252 return encoding_iana
254 return None
257@lru_cache(maxsize=128)
258def is_multi_byte_encoding(name: str) -> bool:
259 """
260 Verify is a specific encoding is a multi byte one based on it IANA name
261 """
262 return name in {
263 "utf_8",
264 "utf_8_sig",
265 "utf_16",
266 "utf_16_be",
267 "utf_16_le",
268 "utf_32",
269 "utf_32_le",
270 "utf_32_be",
271 "utf_7",
272 } or issubclass(
273 importlib.import_module(f"encodings.{name}").IncrementalDecoder,
274 MultibyteIncrementalDecoder,
275 )
278def identify_sig_or_bom(sequence: bytes) -> tuple[str | None, bytes]:
279 """
280 Identify and extract SIG/BOM in given sequence.
281 """
283 for iana_encoding in ENCODING_MARKS:
284 marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
286 if isinstance(marks, bytes):
287 marks = [marks]
289 for mark in marks:
290 if sequence.startswith(mark):
291 return iana_encoding, mark
293 return None, b""
296def should_strip_sig_or_bom(iana_encoding: str) -> bool:
297 return iana_encoding not in {"utf_16", "utf_32"}
300def iana_name(cp_name: str, strict: bool = True) -> str:
301 """Returns the Python normalized encoding name (Not the IANA official name)."""
302 cp_name = cp_name.lower().replace("-", "_")
304 encoding_alias: str
305 encoding_iana: str
307 for encoding_alias, encoding_iana in aliases.items():
308 if cp_name in [encoding_alias, encoding_iana]:
309 return encoding_iana
311 if strict:
312 raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
314 return cp_name
317def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
318 if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
319 return 0.0
321 decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
322 decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
324 id_a: IncrementalDecoder = decoder_a(errors="ignore")
325 id_b: IncrementalDecoder = decoder_b(errors="ignore")
327 character_match_count: int = 0
329 for i in range(255):
330 to_be_decoded: bytes = bytes([i])
331 if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
332 character_match_count += 1
334 return character_match_count / 254
337def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
338 """
339 Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
340 the function cp_similarity.
341 """
342 return (
343 iana_name_a in IANA_SUPPORTED_SIMILAR
344 and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
345 )
348def set_logging_handler(
349 name: str = "charset_normalizer",
350 level: int = logging.INFO,
351 format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
352) -> None:
353 logger = logging.getLogger(name)
354 logger.setLevel(level)
356 handler = logging.StreamHandler()
357 handler.setFormatter(logging.Formatter(format_string))
358 logger.addHandler(handler)
361def cut_sequence_chunks(
362 sequences: bytes,
363 encoding_iana: str,
364 offsets: range,
365 chunk_size: int,
366 bom_or_sig_available: bool,
367 strip_sig_or_bom: bool,
368 sig_payload: bytes,
369 is_multi_byte_decoder: bool,
370 decoded_payload: str | None = None,
371) -> Generator[str, None, None]:
372 if decoded_payload and is_multi_byte_decoder is False:
373 for i in offsets:
374 chunk = decoded_payload[i : i + chunk_size]
375 if not chunk:
376 break
377 yield chunk
378 else:
379 for i in offsets:
380 chunk_end = i + chunk_size
381 if chunk_end > len(sequences) + 8:
382 continue
384 cut_sequence = sequences[i : i + chunk_size]
386 if bom_or_sig_available and strip_sig_or_bom is False:
387 cut_sequence = sig_payload + cut_sequence
389 chunk = cut_sequence.decode(
390 encoding_iana,
391 errors="ignore" if is_multi_byte_decoder else "strict",
392 )
394 # multi-byte bad cutting detector and adjustment
395 # not the cleanest way to perform that fix but clever enough for now.
396 if is_multi_byte_decoder and i > 0:
397 chunk_partial_size_chk: int = min(chunk_size, 16)
399 if (
400 decoded_payload
401 and chunk[:chunk_partial_size_chk] not in decoded_payload
402 ):
403 for j in range(i, i - 4, -1):
404 cut_sequence = sequences[j:chunk_end]
406 if bom_or_sig_available and strip_sig_or_bom is False:
407 cut_sequence = sig_payload + cut_sequence
409 chunk = cut_sequence.decode(encoding_iana, errors="ignore")
411 if chunk[:chunk_partial_size_chk] in decoded_payload:
412 break
414 yield chunk