Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from functools import lru_cache
4from logging import getLogger
7from .constant import (
8 COMMON_CJK_CHARACTERS,
9 COMMON_SAFE_ASCII_CHARACTERS,
10 TRACE,
11 CompatibleFamillyRange,
12 _ACCENTUATED,
13 _ARABIC,
14 _ARABIC_ISOLATED_FORM,
15 _BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES,
16 _CJK,
17 _COMPATIBLE_RANGE_FAMILIES,
18 _COMPATIBLE_WITH_ANY_RANGE_FAMILIES,
19 _HANGUL,
20 _HALFWIDTH_KATAKANA,
21 _HIRAGANA,
22 _KATAKANA,
23 _LATIN,
24 _LIGATURE,
25 _RANGE_FAMILIES,
26 _SENTENCE_OPEN_PUNCTUATION,
27 _SUPERSCRIPT,
28 _THAI,
29)
30from .utils import (
31 _character_flags,
32 is_emoticon,
33 is_punctuation,
34 is_separator,
35 is_symbol,
36 remove_accent,
37 unicode_range,
38)
40# Combined bitmask for CJK/Hangul/Katakana/Hiragana/Thai glyph detection.
41_GLYPH_MASK: int = _CJK | _HANGUL | _KATAKANA | _HIRAGANA | _THAI
44class CharInfo:
45 """Pre-computed character properties shared across all detectors."""
47 __slots__ = (
48 "character",
49 "printable",
50 "alpha",
51 "upper",
52 "lower",
53 "space",
54 "digit",
55 "is_ascii",
56 "case_variable",
57 "flags",
58 "accentuated",
59 "latin",
60 "is_cjk",
61 "is_katakana",
62 "is_halfwidth_katakana",
63 "is_arabic",
64 "is_ligature",
65 "is_superscript",
66 "is_sentence_open_punctuation",
67 "is_glyph",
68 "punct",
69 "sym",
70 "range",
71 "sep",
72 "emoticon",
73 "safe",
74 "common_cjk",
75 "unaccented",
76 )
78 character: str
79 printable: bool
80 alpha: bool
81 upper: bool
82 lower: bool
83 space: bool
84 digit: bool
85 is_ascii: bool
86 case_variable: bool
87 flags: int
88 accentuated: bool
89 latin: bool
90 is_cjk: bool
91 is_katakana: bool
92 is_halfwidth_katakana: bool
93 is_arabic: bool
94 is_ligature: bool
95 is_superscript: bool
96 is_sentence_open_punctuation: bool
97 is_glyph: bool
98 punct: bool
99 sym: bool
100 range: str | None
101 sep: bool
102 emoticon: bool
103 safe: bool
104 common_cjk: bool
105 unaccented: str
107 def __init__(self, character: str) -> None:
108 """Compute all properties for *character* (built once per codepoint,
109 every branch assigns every slot)."""
110 self.character = character
112 # ASCII fast-path: for characters with ord < 128, we can skip
113 # _character_flags() entirely and derive most properties from ord.
114 o: int = ord(character)
115 if o < 128:
116 self.is_ascii = True
117 self.accentuated = False
118 self.unaccented = character
119 self.emoticon = False
120 self.common_cjk = False
121 self.safe = character in COMMON_SAFE_ASCII_CHARACTERS
122 self.is_cjk = False
123 self.is_katakana = False
124 self.is_halfwidth_katakana = False
125 self.is_arabic = False
126 self.is_ligature = False
127 self.is_superscript = False
128 self.is_sentence_open_punctuation = False
129 self.is_glyph = False
130 # ASCII alpha: a-z (97-122) or A-Z (65-90)
131 if 65 <= o <= 90:
132 # Uppercase ASCII letter
133 self.alpha = True
134 self.upper = True
135 self.lower = False
136 self.space = False
137 self.digit = False
138 self.printable = True
139 self.case_variable = True
140 self.flags = _LATIN
141 self.latin = True
142 self.punct = False
143 self.sym = False
144 elif 97 <= o <= 122:
145 # Lowercase ASCII letter
146 self.alpha = True
147 self.upper = False
148 self.lower = True
149 self.space = False
150 self.digit = False
151 self.printable = True
152 self.case_variable = True
153 self.flags = _LATIN
154 self.latin = True
155 self.punct = False
156 self.sym = False
157 elif 48 <= o <= 57:
158 # ASCII digit 0-9
159 self.alpha = False
160 self.upper = False
161 self.lower = False
162 self.space = False
163 self.digit = True
164 self.printable = True
165 self.case_variable = False
166 self.flags = 0
167 self.latin = False
168 self.punct = False
169 self.sym = False
170 elif o == 32 or (9 <= o <= 13):
171 # Space, tab, newline, etc.
172 self.alpha = False
173 self.upper = False
174 self.lower = False
175 self.space = True
176 self.digit = False
177 self.printable = o == 32
178 self.case_variable = False
179 self.flags = 0
180 self.latin = False
181 self.punct = False
182 self.sym = False
183 else:
184 # Other ASCII (punctuation, symbols, control chars)
185 self.printable = character.isprintable()
186 self.alpha = False
187 self.upper = False
188 self.lower = False
189 self.space = False
190 self.digit = False
191 self.case_variable = False
192 self.flags = 0
193 self.latin = False
194 self.punct = is_punctuation(character) if self.printable else False
195 self.sym = is_symbol(character) if self.printable else False
196 else:
197 # Non-ASCII path
198 self.is_ascii = False
199 self.safe = False
200 self.printable = character.isprintable()
201 self.alpha = character.isalpha()
202 self.upper = character.isupper()
203 self.lower = character.islower()
204 self.space = character.isspace()
205 self.digit = character.isdigit()
206 self.case_variable = self.lower != self.upper
208 # Flag-based classification (single unicodedata.name() call, lru-cached)
209 flags: int = _character_flags(character)
210 if self.alpha:
211 self.emoticon = False
212 else:
213 self.emoticon = is_emoticon(character)
214 self.flags = flags
215 self.accentuated = bool(flags & _ACCENTUATED)
216 self.latin = bool(flags & _LATIN)
217 self.is_cjk = bool(flags & _CJK)
218 self.is_katakana = bool(flags & _KATAKANA)
219 self.is_halfwidth_katakana = bool(flags & _HALFWIDTH_KATAKANA)
220 self.is_arabic = bool(flags & _ARABIC)
221 self.is_ligature = bool(flags & _LIGATURE)
222 self.is_superscript = bool(flags & _SUPERSCRIPT)
223 self.is_sentence_open_punctuation = bool(flags & _SENTENCE_OPEN_PUNCTUATION)
224 self.is_glyph = bool(flags & _GLYPH_MASK)
226 if self.latin and self.accentuated:
227 self.unaccented = remove_accent(character)
228 else:
229 self.unaccented = character
231 self.common_cjk = self.is_cjk and character in COMMON_CJK_CHARACTERS
233 # Eagerly compute punct and sym (avoids property dispatch overhead
234 # on 300K+ accesses in the hot loop).
235 if self.printable:
236 self.punct = is_punctuation(character)
237 self.sym = is_symbol(character)
238 else:
239 self.punct = False
240 self.sym = False
242 self.range = unicode_range(character)
243 self.sep = is_separator(character)
246# Per-codepoint cache of CharInfo instances
247# At most UTF-8 size allocated.
248@lru_cache(maxsize=None)
249def _char_info(character: str) -> CharInfo:
250 """Build (once per codepoint) and cache the CharInfo for *character*."""
251 return CharInfo(character)
254# ASCII table indexed by codepoint.
255_ASCII_CHAR_INFO: list[CharInfo] = [
256 CharInfo(chr(_codepoint)) for _codepoint in range(128)
257]
260class MessDetectorPlugin:
261 """
262 Base abstract class used for mess detection plugins.
263 All detectors MUST extend and implement given methods.
264 """
266 __slots__ = ()
268 def feed_info(self, character: str, info: CharInfo) -> None:
269 """
270 The main routine to be executed upon character.
271 Insert the logic in witch the text would be considered chaotic.
272 """
273 raise NotImplementedError # Defensive:
275 def reset(self) -> None: # Defensive:
276 """
277 Permit to reset the plugin to the initial state.
278 """
279 raise NotImplementedError
281 @property
282 def ratio(self) -> float:
283 """
284 Compute the chaos ratio based on what your feed() has seen.
285 Must NOT be lower than 0.; No restriction gt 0.
286 """
287 raise NotImplementedError # Defensive:
290class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin):
291 __slots__ = (
292 "_punctuation_count",
293 "_symbol_count",
294 "_character_count",
295 "_last_printable_char",
296 )
298 def __init__(self) -> None:
299 self._punctuation_count: int = 0
300 self._symbol_count: int = 0
301 self._character_count: int = 0
303 self._last_printable_char: str | None = None
305 def feed_info(self, character: str, info: CharInfo) -> None:
306 """Optimized feed using pre-computed character info."""
307 self._character_count += 1
309 if character != self._last_printable_char and not info.safe:
310 if info.punct:
311 self._punctuation_count += 1
312 elif not info.digit and info.sym and not info.emoticon:
313 self._symbol_count += 2
315 self._last_printable_char = character
317 def reset(self) -> None: # Abstract
318 self._punctuation_count = 0
319 self._character_count = 0
320 self._symbol_count = 0
322 @property
323 def ratio(self) -> float:
324 if self._character_count == 0:
325 return 0.0
327 ratio_of_punctuation: float = (
328 self._punctuation_count + self._symbol_count
329 ) / self._character_count
331 return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0
334class TooManyAccentuatedPlugin(MessDetectorPlugin):
335 __slots__ = ("_character_count", "_accentuated_count")
337 def __init__(self) -> None:
338 self._character_count: int = 0
339 self._accentuated_count: int = 0
341 def feed_info(self, character: str, info: CharInfo) -> None:
342 """Optimized feed using pre-computed character info."""
343 self._character_count += 1
345 if info.accentuated:
346 self._accentuated_count += 1
348 def reset(self) -> None: # Abstract
349 self._character_count = 0
350 self._accentuated_count = 0
352 @property
353 def ratio(self) -> float:
354 if self._character_count < 8:
355 return 0.0
357 ratio_of_accentuation: float = self._accentuated_count / self._character_count
358 return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
361class UnprintablePlugin(MessDetectorPlugin):
362 __slots__ = ("_unprintable_count", "_character_count", "_has_escape")
364 def __init__(self) -> None:
365 self._unprintable_count: int = 0
366 self._character_count: int = 0
367 self._has_escape: bool = False
369 def feed_info(self, character: str, info: CharInfo) -> None:
370 """Optimized feed using pre-computed character info."""
371 if character == "\x1b":
372 self._has_escape = True
374 if (
375 not info.printable
376 and not info.space
377 and character != "\x1a"
378 and character != "\ufeff"
379 ):
380 self._unprintable_count += 1
381 self._character_count += 1
383 def reset(self) -> None: # Abstract
384 self._unprintable_count = 0
385 self._has_escape = False
387 @property
388 def ratio(self) -> float:
389 if self._character_count == 0: # Defensive:
390 return 0.0
392 if self._has_escape:
393 return 1.0
395 return (self._unprintable_count * 8) / self._character_count
398class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin):
399 __slots__ = (
400 "_successive_count",
401 "_character_count",
402 "_last_latin_character",
403 "_last_was_accentuated",
404 )
406 def __init__(self) -> None:
407 self._successive_count: int = 0
408 self._character_count: int = 0
410 self._last_latin_character: CharInfo | None = None
411 self._last_was_accentuated: bool = False
413 def feed_info(self, character: str, info: CharInfo) -> None:
414 """Optimized feed using pre-computed character info."""
415 self._character_count += 1
416 if (
417 self._last_latin_character is not None
418 and info.accentuated
419 and self._last_was_accentuated
420 ):
421 if info.upper and self._last_latin_character.upper:
422 self._successive_count += 1
423 if info.unaccented == self._last_latin_character.unaccented:
424 self._successive_count += 1
425 self._last_latin_character = info
426 self._last_was_accentuated = info.accentuated
428 def reset(self) -> None: # Abstract
429 self._successive_count = 0
430 self._character_count = 0
431 self._last_latin_character = None
432 self._last_was_accentuated = False
434 @property
435 def ratio(self) -> float:
436 if self._character_count == 0:
437 return 0.0
439 return (self._successive_count * 2) / self._character_count
442class SuspiciousRange(MessDetectorPlugin):
443 __slots__ = (
444 "_suspicious_successive_range_count",
445 "_character_count",
446 "_last_printable_seen",
447 "_last_printable_range",
448 )
450 def __init__(self) -> None:
451 self._suspicious_successive_range_count: int = 0
452 self._character_count: int = 0
453 self._last_printable_seen: str | None = None
454 self._last_printable_range: str | None = None
456 def feed_info(self, character: str, info: CharInfo) -> None:
457 """Optimized feed using pre-computed character info."""
458 self._character_count += 1
460 if info.space or info.punct or info.safe:
461 self._last_printable_seen = None
462 self._last_printable_range = None
463 return
465 if self._last_printable_seen is None:
466 self._last_printable_seen = character
467 self._last_printable_range = info.range
468 return
470 unicode_range_a: str | None = self._last_printable_range
471 unicode_range_b: str | None = info.range
473 # Identical non-None ranges can never be suspicious.
474 if unicode_range_a != unicode_range_b or unicode_range_a is None:
475 if is_suspiciously_successive_range(unicode_range_a, unicode_range_b):
476 self._suspicious_successive_range_count += 1
478 self._last_printable_seen = character
479 self._last_printable_range = unicode_range_b
481 def reset(self) -> None: # Abstract
482 self._character_count = 0
483 self._suspicious_successive_range_count = 0
484 self._last_printable_seen = None
485 self._last_printable_range = None
487 @property
488 def ratio(self) -> float:
489 if self._character_count <= 13:
490 return 0.0
492 ratio_of_suspicious_range_usage: float = (
493 self._suspicious_successive_range_count * 2
494 ) / self._character_count
496 return ratio_of_suspicious_range_usage
499class SuperWeirdWordPlugin(MessDetectorPlugin):
500 __slots__ = (
501 "_word_count",
502 "_bad_word_count",
503 "_foreign_long_count",
504 "_is_current_word_bad",
505 "_foreign_long_watch",
506 "_character_count",
507 "_bad_character_count",
508 "_buffer_length",
509 "_buffer_last_char",
510 "_buffer_last_char_accentuated",
511 "_buffer_accent_count",
512 "_buffer_glyph_count",
513 "_buffer_upper_count",
514 "_buffer_first_lower",
515 "_buffer_has_non_ascii",
516 "_buffer_last_char_ligature",
517 "_buffer_has_internal_ligature",
518 "_is_current_word_invalid",
519 "_invalid_word_count",
520 )
522 def __init__(self) -> None:
523 self._word_count: int = 0
524 self._bad_word_count: int = 0
525 self._foreign_long_count: int = 0
527 self._is_current_word_bad: bool = False
528 self._foreign_long_watch: bool = False
530 self._character_count: int = 0
531 self._bad_character_count: int = 0
533 self._buffer_length: int = 0
534 self._buffer_last_char: str | None = None
535 self._buffer_last_char_accentuated: bool = False
536 self._buffer_accent_count: int = 0
537 self._buffer_glyph_count: int = 0
538 self._buffer_upper_count: int = 0
539 self._buffer_first_lower: bool = False
540 self._buffer_has_non_ascii: bool = False
541 self._buffer_last_char_ligature: bool = False
542 self._buffer_has_internal_ligature: bool = False
543 self._is_current_word_invalid: bool = False
544 self._invalid_word_count: int = 0
546 def feed_info(self, character: str, info: CharInfo) -> None:
547 """Optimized feed using pre-computed character info."""
548 if info.alpha:
549 if self._buffer_last_char_ligature:
550 self._buffer_has_internal_ligature = True
551 self._buffer_last_char_ligature = info.is_ligature
552 if self._buffer_length == 0:
553 self._buffer_first_lower = info.lower
554 self._buffer_length += 1
555 self._buffer_last_char = character
557 if info.upper:
558 self._buffer_upper_count += 1
559 if not info.is_ascii:
560 self._buffer_has_non_ascii = True
562 self._buffer_last_char_accentuated = info.accentuated
564 if info.accentuated:
565 self._buffer_accent_count += 1
566 if info.is_glyph:
567 self._buffer_glyph_count += 1
568 elif not self._foreign_long_watch and (not info.latin or info.accentuated):
569 self._foreign_long_watch = True
570 return
571 if not self._buffer_length:
572 return
573 if info.is_sentence_open_punctuation or (
574 info.is_superscript and self._buffer_has_internal_ligature
575 ):
576 self._is_current_word_bad = True
577 self._is_current_word_invalid = True
578 if info.space or info.punct or info.sep:
579 self._word_count += 1
580 buffer_length: int = self._buffer_length
582 self._character_count += buffer_length
584 if buffer_length >= 4:
585 if self._buffer_accent_count / buffer_length >= 0.5:
586 self._is_current_word_bad = True
587 elif (
588 self._buffer_last_char_accentuated
589 and self._buffer_last_char.isupper() # type: ignore[union-attr]
590 and self._buffer_upper_count != buffer_length
591 ):
592 self._foreign_long_count += 1
593 self._is_current_word_bad = True
594 elif self._buffer_glyph_count == 1:
595 self._is_current_word_bad = True
596 self._foreign_long_count += 1
597 elif (
598 self._buffer_has_non_ascii
599 and self._buffer_first_lower
600 and self._buffer_upper_count == buffer_length - 1
601 ):
602 # Inverse capitalization detector.
603 # No natural writing produces such words.
604 # see https://github.com/jawah/charset_normalizer/issues/731
605 self._foreign_long_count += 1
606 self._is_current_word_bad = True
607 if buffer_length >= 24 and self._foreign_long_watch:
608 probable_camel_cased: bool = (
609 self._buffer_upper_count > 0
610 and self._buffer_upper_count / buffer_length <= 0.3
611 )
613 if not probable_camel_cased:
614 self._foreign_long_count += 1
615 self._is_current_word_bad = True
617 if self._is_current_word_bad:
618 self._bad_word_count += 1
619 self._bad_character_count += buffer_length
620 self._is_current_word_bad = False
621 if self._is_current_word_invalid:
622 self._invalid_word_count += 1
623 self._is_current_word_invalid = False
625 self._foreign_long_watch = False
626 self._buffer_length = 0
627 self._buffer_last_char = None
628 self._buffer_last_char_accentuated = False
629 self._buffer_accent_count = 0
630 self._buffer_glyph_count = 0
631 self._buffer_upper_count = 0
632 self._buffer_first_lower = False
633 self._buffer_has_non_ascii = False
634 self._buffer_last_char_ligature = False
635 self._buffer_has_internal_ligature = False
636 elif (
637 character not in {"<", ">", "-", "=", "~", "|", "_"}
638 and not info.digit
639 and info.sym
640 ):
641 self._is_current_word_bad = True
642 self._buffer_length += 1
643 self._buffer_last_char = character
644 self._buffer_last_char_accentuated = False
646 def reset(self) -> None: # Abstract
647 self._buffer_length = 0
648 self._buffer_last_char = None
649 self._buffer_last_char_accentuated = False
650 self._is_current_word_bad = False
651 self._foreign_long_watch = False
652 self._bad_word_count = 0
653 self._word_count = 0
654 self._character_count = 0
655 self._bad_character_count = 0
656 self._foreign_long_count = 0
657 self._buffer_accent_count = 0
658 self._buffer_glyph_count = 0
659 self._buffer_upper_count = 0
660 self._buffer_first_lower = False
661 self._buffer_has_non_ascii = False
662 self._buffer_last_char_ligature = False
663 self._buffer_has_internal_ligature = False
664 self._is_current_word_invalid = False
665 self._invalid_word_count = 0
667 @property
668 def ratio(self) -> float:
669 if self._invalid_word_count:
670 return 1.0
672 if self._word_count <= 10 and self._foreign_long_count == 0:
673 return 0.0
675 return self._bad_character_count / self._character_count
678class CjkUncommonPlugin(MessDetectorPlugin):
679 """
680 Detect messy CJK text that probably means nothing.
681 """
683 __slots__ = ("_character_count", "_uncommon_count")
685 def __init__(self) -> None:
686 self._character_count: int = 0
687 self._uncommon_count: int = 0
689 def feed_info(self, character: str, info: CharInfo) -> None:
690 """Optimized feed using pre-computed character info."""
691 self._character_count += 1
693 if not info.common_cjk:
694 self._uncommon_count += 1
696 def reset(self) -> None: # Abstract
697 self._character_count = 0
698 self._uncommon_count = 0
700 @property
701 def ratio(self) -> float:
702 if self._character_count < 4:
703 return 0.0
705 uncommon_form_usage: float = (
706 2 * self._uncommon_count - self._character_count
707 ) / (5 * max(self._character_count, 16))
709 # we can be pretty sure it's garbage when uncommon characters are widely
710 # used. otherwise it could just be traditional chinese for example.
711 return max(0.0, uncommon_form_usage)
714class SuspiciousKatakanaPlugin(MessDetectorPlugin):
715 """Detect implausible halfwidth Katakana and uncommon CJK combinations."""
717 __slots__ = (
718 "_katakana_count",
719 "_halfwidth_katakana_count",
720 "_cjk_count",
721 "_uncommon_cjk_count",
722 )
724 def __init__(self) -> None:
725 self._katakana_count: int = 0
726 self._halfwidth_katakana_count: int = 0
727 self._cjk_count: int = 0
728 self._uncommon_cjk_count: int = 0
730 def feed_info(self, character: str, info: CharInfo) -> None:
731 """Optimized feed using pre-computed character info."""
732 if info.is_katakana:
733 self._katakana_count += 1
734 if info.is_halfwidth_katakana:
735 self._halfwidth_katakana_count += 1
736 return
738 self._cjk_count += 1
739 if not info.common_cjk:
740 self._uncommon_cjk_count += 1
742 def reset(self) -> None: # Abstract
743 self._katakana_count = 0
744 self._halfwidth_katakana_count = 0
745 self._cjk_count = 0
746 self._uncommon_cjk_count = 0
748 @property
749 def ratio(self) -> float:
750 if (
751 self._halfwidth_katakana_count >= 4
752 and self._halfwidth_katakana_count == self._katakana_count
753 and 3 <= self._cjk_count == self._uncommon_cjk_count
754 ):
755 return 1.0
757 return 0.0
760class ArchaicUpperLowerPlugin(MessDetectorPlugin):
761 __slots__ = (
762 "_buf",
763 "_character_count_since_last_sep",
764 "_successive_upper_lower_count",
765 "_successive_upper_lower_count_final",
766 "_character_count",
767 "_last_alpha_seen_upper",
768 "_last_alpha_seen_lower",
769 "_current_ascii_only",
770 )
772 def __init__(self) -> None:
773 self._buf: bool = False
775 self._character_count_since_last_sep: int = 0
777 self._successive_upper_lower_count: int = 0
778 self._successive_upper_lower_count_final: int = 0
780 self._character_count: int = 0
782 self._last_alpha_seen_upper: bool = False
783 self._last_alpha_seen_lower: bool = False
784 self._current_ascii_only: bool = True
786 def feed_info(self, character: str, info: CharInfo) -> None:
787 """Optimized feed using pre-computed character info."""
788 is_concerned: bool = info.alpha and info.case_variable
789 chunk_sep: bool = not is_concerned
791 if chunk_sep and self._character_count_since_last_sep > 0:
792 if (
793 self._character_count_since_last_sep <= 64
794 and not info.digit
795 and not self._current_ascii_only
796 ):
797 self._successive_upper_lower_count_final += (
798 self._successive_upper_lower_count
799 )
801 self._successive_upper_lower_count = 0
802 self._character_count_since_last_sep = 0
803 self._buf = False
804 self._character_count += 1
805 self._current_ascii_only = True
807 return
809 if self._current_ascii_only and not info.is_ascii:
810 self._current_ascii_only = False
812 if self._character_count_since_last_sep > 0:
813 if (info.upper and self._last_alpha_seen_lower) or (
814 info.lower and self._last_alpha_seen_upper
815 ):
816 if self._buf:
817 self._successive_upper_lower_count += 2
818 self._buf = False
819 else:
820 self._buf = True
821 else:
822 self._buf = False
824 self._character_count += 1
825 self._character_count_since_last_sep += 1
826 self._last_alpha_seen_upper = info.upper
827 self._last_alpha_seen_lower = info.lower
829 def reset(self) -> None: # Abstract
830 self._character_count = 0
831 self._character_count_since_last_sep = 0
832 self._successive_upper_lower_count = 0
833 self._successive_upper_lower_count_final = 0
834 self._last_alpha_seen_upper = False
835 self._last_alpha_seen_lower = False
836 self._buf = False
837 self._current_ascii_only = True
839 @property
840 def ratio(self) -> float:
841 if self._character_count == 0: # Defensive:
842 return 0.0
844 return self._successive_upper_lower_count_final / self._character_count
847class ArabicIsolatedFormPlugin(MessDetectorPlugin):
848 __slots__ = ("_character_count", "_isolated_form_count")
850 def __init__(self) -> None:
851 self._character_count: int = 0
852 self._isolated_form_count: int = 0
854 def reset(self) -> None: # Abstract
855 self._character_count = 0
856 self._isolated_form_count = 0
858 def feed_info(self, character: str, info: CharInfo) -> None:
859 """Optimized feed using pre-computed character info."""
860 self._character_count += 1
862 if info.flags & _ARABIC_ISOLATED_FORM:
863 self._isolated_form_count += 1
865 @property
866 def ratio(self) -> float:
867 if self._character_count < 8:
868 return 0.0
870 isolated_form_usage: float = self._isolated_form_count / self._character_count
872 return isolated_form_usage
875@lru_cache(maxsize=None)
876def is_suspiciously_successive_range(
877 unicode_range_a: str | None, unicode_range_b: str | None
878) -> bool:
879 """
880 Determine if two Unicode ranges seen next to each other can be considered suspicious.
881 """
882 if unicode_range_a is None or unicode_range_b is None:
883 return True
885 familly_a: str = _RANGE_FAMILIES[unicode_range_a]
886 familly_b: str = _RANGE_FAMILIES[unicode_range_b]
888 if familly_a == familly_b:
889 return False
891 if (
892 familly_a in _COMPATIBLE_WITH_ANY_RANGE_FAMILIES
893 or familly_b in _COMPATIBLE_WITH_ANY_RANGE_FAMILIES
894 ):
895 return False
897 if CompatibleFamillyRange(familly_a, familly_b) in _COMPATIBLE_RANGE_FAMILIES:
898 return False
900 # Basic Latin is commonly interspersed with East Asian scripts, but the
901 # compatibility must not extend to every range in the Latin family.
902 if unicode_range_a == "Basic Latin":
903 return familly_b not in _BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES
905 if unicode_range_b == "Basic Latin":
906 return familly_a not in _BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES
908 return True
911def mess_ratio(
912 decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False
913) -> float:
914 """
915 Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
916 """
918 seq_len: int = len(decoded_sequence)
920 if seq_len < 511:
921 step: int = 32
922 elif seq_len < 1024:
923 step = 64
924 else:
925 step = 128
927 # str.isascii() is O(1) (the flag lives in the str header). Seven of the
928 # ten detectors provably keep a 0.0 ratio on ASCII-only input and are
929 # therefore not fed at all.
930 is_pure_ascii: bool = decoded_sequence.isascii()
932 # Cached per-codepoint character properties (see CharInfo). ASCII
933 # characters resolve through the immutable import-time table; anything
934 # else goes through the lru_cache-backed slow path.
935 ascii_info = _ASCII_CHAR_INFO
936 char_info = _char_info
938 mean_mess_ratio: float
939 info: CharInfo
941 # Create each detector as a named local variable (unrolled from the generic loop).
942 # This eliminates per-character iteration over the detector list and
943 # per-character eligible() virtual dispatch, while keeping every plugin class
944 # intact and fully readable.
945 d_sp: TooManySymbolOrPunctuationPlugin = TooManySymbolOrPunctuationPlugin()
946 d_ta: TooManyAccentuatedPlugin = TooManyAccentuatedPlugin()
947 d_up: UnprintablePlugin = UnprintablePlugin()
948 d_sda: SuspiciousDuplicateAccentPlugin = SuspiciousDuplicateAccentPlugin()
949 d_sr: SuspiciousRange = SuspiciousRange()
950 d_sw: SuperWeirdWordPlugin = SuperWeirdWordPlugin()
951 d_cu: CjkUncommonPlugin = CjkUncommonPlugin()
952 d_sk: SuspiciousKatakanaPlugin = SuspiciousKatakanaPlugin()
953 d_au: ArchaicUpperLowerPlugin = ArchaicUpperLowerPlugin()
954 d_ai: ArabicIsolatedFormPlugin = ArabicIsolatedFormPlugin()
956 # Local references avoid repeated bound-method creation in the hot loop.
957 d_sp_feed = d_sp.feed_info
958 d_ta_feed = d_ta.feed_info
959 d_up_feed = d_up.feed_info
960 d_sda_feed = d_sda.feed_info
961 d_sr_feed = d_sr.feed_info
962 d_sw_feed = d_sw.feed_info
963 d_cu_feed = d_cu.feed_info
964 d_sk_feed = d_sk.feed_info
965 d_au_feed = d_au.feed_info
966 d_ai_feed = d_ai.feed_info
968 for block_start in range(0, seq_len, step):
969 for character in decoded_sequence[block_start : block_start + step]:
970 # Character properties computed once per distinct codepoint
971 # (shared across all plugins and all mess_ratio calls).
972 # ord() doubles as the ASCII table index.
973 codepoint: int = ord(character)
974 if codepoint < 128:
975 info = ascii_info[codepoint]
976 else:
977 info = char_info(character)
979 # Detectors with eligible() == always True
980 d_up_feed(character, info)
981 d_sw_feed(character, info)
983 if is_pure_ascii:
984 # The seven remaining detectors provably stay at 0.0 (see above).
985 if info.printable:
986 d_sp_feed(character, info)
987 continue
989 d_au_feed(character, info)
991 # Detectors with eligible() == isprintable
992 if info.printable:
993 d_sp_feed(character, info)
994 d_sr_feed(character, info)
996 # Detectors with eligible() == isalpha
997 if info.alpha:
998 d_ta_feed(character, info)
999 # SuspiciousDuplicateAccent: isalpha() and is_latin()
1000 if info.latin:
1001 d_sda_feed(character, info)
1002 # CjkUncommon and SuspiciousKatakana: is_cjk()
1003 if info.is_cjk:
1004 d_cu_feed(character, info)
1005 d_sk_feed(character, info)
1006 elif info.is_katakana:
1007 d_sk_feed(character, info)
1008 # ArabicIsolatedForm: is_arabic()
1009 if info.is_arabic:
1010 d_ai_feed(character, info)
1012 mean_mess_ratio = (
1013 d_sp.ratio
1014 + d_ta.ratio
1015 + d_up.ratio
1016 + d_sda.ratio
1017 + d_sr.ratio
1018 + d_sw.ratio
1019 + d_cu.ratio
1020 + d_sk.ratio
1021 + d_au.ratio
1022 + d_ai.ratio
1023 )
1025 if mean_mess_ratio >= maximum_threshold:
1026 break
1027 else:
1028 # Flush last word buffer in SuperWeirdWordPlugin via trailing newline.
1029 nl_info = ascii_info[10] # "\n"
1030 d_sw_feed("\n", nl_info)
1031 if not is_pure_ascii:
1032 d_au_feed("\n", nl_info)
1033 d_up_feed("\n", nl_info)
1035 mean_mess_ratio = (
1036 d_sp.ratio
1037 + d_ta.ratio
1038 + d_up.ratio
1039 + d_sda.ratio
1040 + d_sr.ratio
1041 + d_sw.ratio
1042 + d_cu.ratio
1043 + d_sk.ratio
1044 + d_au.ratio
1045 + d_ai.ratio
1046 )
1048 if debug: # Defensive:
1049 logger = getLogger("charset_normalizer")
1051 logger.log(
1052 TRACE,
1053 "Mess-detector extended-analysis start. "
1054 f"intermediary_mean_mess_ratio_calc={step} mean_mess_ratio={mean_mess_ratio} "
1055 f"maximum_threshold={maximum_threshold}",
1056 )
1058 if seq_len > 16:
1059 logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}")
1060 logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}")
1062 for dt in [d_sp, d_ta, d_up, d_sda, d_sr, d_sw, d_cu, d_sk, d_au, d_ai]:
1063 logger.log(TRACE, f"{dt.__class__}: {dt.ratio}")
1065 return round(mean_mess_ratio, 3)