Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/ftfy/badness.py: 73%
11 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:33 +0000
1"""
2`ftfy.badness` contains a heuristic that detects likely mojibake.
4This heuristic signals to ftfy which segments of text need to be fixed, and
5also indicates when the text can stop being fixed.
7The design of this heuristic is that we categorize the approximately 400
8Unicode characters that occur in UTF-8 mojibake, specifically the characters
9that come from mixing up UTF-8 with the other encodings we support. We
10identify sequences and contexts of these characters that are much more likely
11to be mojibake than intended strings, such as lowercase accented letters
12followed immediately by currency symbols.
13"""
15import warnings
16import re
19# There are only 403 characters that occur in known UTF-8 mojibake, and we can
20# characterize them:
22MOJIBAKE_CATEGORIES = {
23 # Characters that appear in many different contexts. Sequences that contain
24 # them are not inherently mojibake
25 "common": (
26 "\N{NO-BREAK SPACE}"
27 "\N{SOFT HYPHEN}"
28 "\N{MIDDLE DOT}"
29 "\N{ACUTE ACCENT}"
30 "\N{EN DASH}"
31 "\N{EM DASH}"
32 "\N{HORIZONTAL BAR}"
33 "\N{HORIZONTAL ELLIPSIS}"
34 "\N{RIGHT SINGLE QUOTATION MARK}"
35 ),
36 # the C1 control character range, which have no uses outside of mojibake anymore
37 "c1": "\x80-\x9f",
38 # Characters that are nearly 100% used in mojibake
39 "bad": (
40 "\N{BROKEN BAR}"
41 "\N{CURRENCY SIGN}"
42 "\N{DIAERESIS}"
43 "\N{NOT SIGN}"
44 "\N{MACRON}"
45 "\N{PILCROW SIGN}"
46 "\N{SECTION SIGN}"
47 "\N{CEDILLA}"
48 "\N{LATIN SMALL LETTER F WITH HOOK}"
49 "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # it's not a modifier
50 "\N{CARON}"
51 "\N{BREVE}"
52 "\N{OGONEK}"
53 "\N{SMALL TILDE}"
54 "\N{DAGGER}"
55 "\N{DOUBLE DAGGER}"
56 "\N{PER MILLE SIGN}"
57 "\N{REVERSED NOT SIGN}"
58 "\N{LOZENGE}"
59 "\ufffd"
60 # Theoretically these would appear in 'numeric' contexts, but when they
61 # co-occur with other mojibake characters, it's not really ambiguous
62 "\N{FEMININE ORDINAL INDICATOR}"
63 "\N{MASCULINE ORDINAL INDICATOR}"
64 ),
65 "currency": (
66 "\N{CENT SIGN}"
67 "\N{POUND SIGN}"
68 "\N{YEN SIGN}"
69 "\N{PESETA SIGN}"
70 "\N{EURO SIGN}"
71 ),
72 "start_punctuation": (
73 "\N{INVERTED EXCLAMATION MARK}"
74 "\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}"
75 "\N{INVERTED QUESTION MARK}"
76 "\N{COPYRIGHT SIGN}"
77 "\N{GREEK TONOS}"
78 "\N{GREEK DIALYTIKA TONOS}"
79 "\N{LEFT SINGLE QUOTATION MARK}"
80 "\N{SINGLE LOW-9 QUOTATION MARK}"
81 "\N{LEFT DOUBLE QUOTATION MARK}"
82 "\N{DOUBLE LOW-9 QUOTATION MARK}"
83 "\N{BULLET}"
84 "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}"
85 "\uf8ff" # OS-specific symbol, usually the Apple logo
86 ),
87 "end_punctuation": (
88 "\N{REGISTERED SIGN}"
89 "\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
90 "\N{DOUBLE ACUTE ACCENT}"
91 "\N{RIGHT DOUBLE QUOTATION MARK}"
92 "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}"
93 "\N{TRADE MARK SIGN}"
94 ),
95 "numeric": (
96 "\N{SUPERSCRIPT TWO}"
97 "\N{SUPERSCRIPT THREE}"
98 "\N{SUPERSCRIPT ONE}"
99 "\N{PLUS-MINUS SIGN}"
100 "\N{VULGAR FRACTION ONE QUARTER}"
101 "\N{VULGAR FRACTION ONE HALF}"
102 "\N{VULGAR FRACTION THREE QUARTERS}"
103 "\N{MULTIPLICATION SIGN}"
104 "\N{MICRO SIGN}"
105 "\N{DIVISION SIGN}"
106 "\N{FRACTION SLASH}"
107 "\N{PARTIAL DIFFERENTIAL}"
108 "\N{INCREMENT}"
109 "\N{N-ARY PRODUCT}"
110 "\N{N-ARY SUMMATION}"
111 "\N{SQUARE ROOT}"
112 "\N{INFINITY}"
113 "\N{INTERSECTION}"
114 "\N{INTEGRAL}"
115 "\N{ALMOST EQUAL TO}"
116 "\N{NOT EQUAL TO}"
117 "\N{IDENTICAL TO}"
118 "\N{LESS-THAN OR EQUAL TO}"
119 "\N{GREATER-THAN OR EQUAL TO}"
120 "\N{NUMERO SIGN}"
121 ),
122 # Letters that might be used to make emoticon faces (kaomoji), and
123 # therefore might need to appear in more improbable-looking contexts.
124 #
125 # These are concatenated character ranges for use in a regex. I know
126 # they look like faces themselves. I think expressing the ranges like
127 # this helps to illustrate why we need to be careful with these
128 # characters.
129 "kaomoji": (
130 "Ò-Ö"
131 "Ù-Ü"
132 "ò-ö"
133 "ø-ü"
134 "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}"
135 "\N{DEGREE SIGN}"
136 ),
137 "upper_accented": (
138 # LATIN CAPITAL LETTER A WITH GRAVE - LATIN CAPITAL LETTER N WITH TILDE
139 "\xc0-\xd1"
140 # skip capital O's and U's that could be used in kaomoji, but
141 # include Ø because it's very common in Arabic mojibake:
142 "\N{LATIN CAPITAL LETTER O WITH STROKE}"
143 "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}"
144 "\N{LATIN CAPITAL LETTER Y WITH ACUTE}"
145 "\N{LATIN CAPITAL LETTER A WITH BREVE}"
146 "\N{LATIN CAPITAL LETTER A WITH OGONEK}"
147 "\N{LATIN CAPITAL LETTER C WITH ACUTE}"
148 "\N{LATIN CAPITAL LETTER C WITH CARON}"
149 "\N{LATIN CAPITAL LETTER D WITH CARON}"
150 "\N{LATIN CAPITAL LETTER D WITH STROKE}"
151 "\N{LATIN CAPITAL LETTER E WITH OGONEK}"
152 "\N{LATIN CAPITAL LETTER E WITH CARON}"
153 "\N{LATIN CAPITAL LETTER G WITH BREVE}"
154 "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}"
155 "\N{LATIN CAPITAL LETTER L WITH ACUTE}"
156 "\N{LATIN CAPITAL LETTER L WITH CARON}"
157 "\N{LATIN CAPITAL LETTER L WITH STROKE}"
158 "\N{LATIN CAPITAL LETTER N WITH ACUTE}"
159 "\N{LATIN CAPITAL LETTER N WITH CARON}"
160 "\N{LATIN CAPITAL LIGATURE OE}"
161 "\N{LATIN CAPITAL LETTER R WITH CARON}"
162 "\N{LATIN CAPITAL LETTER S WITH ACUTE}"
163 "\N{LATIN CAPITAL LETTER S WITH CEDILLA}"
164 "\N{LATIN CAPITAL LETTER S WITH CARON}"
165 "\N{LATIN CAPITAL LETTER T WITH CEDILLA}"
166 "\N{LATIN CAPITAL LETTER T WITH CARON}"
167 "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}"
168 "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}"
169 "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}"
170 "\N{LATIN CAPITAL LETTER Z WITH ACUTE}"
171 "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}"
172 "\N{LATIN CAPITAL LETTER Z WITH CARON}"
173 "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}"
174 ),
175 "lower_accented": (
176 "\N{LATIN SMALL LETTER SHARP S}"
177 # LATIN SMALL LETTER A WITH GRAVE - LATIN SMALL LETTER N WITH TILDE
178 "\xe0-\xf1"
179 # skip o's and u's that could be used in kaomoji
180 "\N{LATIN SMALL LETTER A WITH BREVE}"
181 "\N{LATIN SMALL LETTER A WITH OGONEK}"
182 "\N{LATIN SMALL LETTER C WITH ACUTE}"
183 "\N{LATIN SMALL LETTER C WITH CARON}"
184 "\N{LATIN SMALL LETTER D WITH CARON}"
185 "\N{LATIN SMALL LETTER D WITH STROKE}"
186 "\N{LATIN SMALL LETTER E WITH OGONEK}"
187 "\N{LATIN SMALL LETTER E WITH CARON}"
188 "\N{LATIN SMALL LETTER G WITH BREVE}"
189 "\N{LATIN SMALL LETTER L WITH ACUTE}"
190 "\N{LATIN SMALL LETTER L WITH CARON}"
191 "\N{LATIN SMALL LETTER L WITH STROKE}"
192 "\N{LATIN SMALL LIGATURE OE}"
193 "\N{LATIN SMALL LETTER R WITH ACUTE}"
194 "\N{LATIN SMALL LETTER S WITH ACUTE}"
195 "\N{LATIN SMALL LETTER S WITH CEDILLA}"
196 "\N{LATIN SMALL LETTER S WITH CARON}"
197 "\N{LATIN SMALL LETTER T WITH CARON}"
198 "\N{LATIN SMALL LETTER U WITH DIAERESIS}"
199 "\N{LATIN SMALL LETTER Z WITH ACUTE}"
200 "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}"
201 "\N{LATIN SMALL LETTER Z WITH CARON}"
202 "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}"
203 "\N{LATIN SMALL LIGATURE FI}"
204 "\N{LATIN SMALL LIGATURE FL}"
205 ),
206 "upper_common": (
207 "\N{LATIN CAPITAL LETTER THORN}"
208 "\N{GREEK CAPITAL LETTER ALPHA}-\N{GREEK CAPITAL LETTER OMEGA}"
209 # not included under 'accented' because these can commonly
210 # occur at ends of words, in positions where they'd be detected
211 # as mojibake
212 "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}"
213 "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}"
214 "\N{GREEK CAPITAL LETTER ETA WITH TONOS}"
215 "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}"
216 "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}"
217 "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}"
218 "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}"
219 "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}"
220 "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}"
221 "\N{CYRILLIC CAPITAL LETTER IO}-\N{CYRILLIC CAPITAL LETTER YA}"
222 ),
223 "lower_common": (
224 # lowercase thorn does not appear in mojibake
225 "\N{GREEK SMALL LETTER ALPHA}-\N{GREEK SMALL LETTER OMEGA}"
226 "\N{GREEK SMALL LETTER ALPHA WITH TONOS}"
227 "\N{GREEK SMALL LETTER EPSILON WITH TONOS}"
228 "\N{GREEK SMALL LETTER ETA WITH TONOS}"
229 "\N{GREEK SMALL LETTER IOTA WITH TONOS}"
230 "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}"
231 "\N{CYRILLIC SMALL LETTER A}-\N{CYRILLIC SMALL LETTER DZHE}"
232 ),
233 "box": (
234 # omit the single horizontal line, might be used in kaomoji
235 "│┌┐┘├┤┬┼"
236 "\N{BOX DRAWINGS DOUBLE HORIZONTAL}-\N{BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL}"
237 "▀▄█▌▐░▒▓"
238 ),
239}
242# We can now build a regular expression that detects unlikely juxtapositions
243# of characters, mostly based on their categories.
244#
245# Another regular expression, which detects sequences that look more specifically
246# like UTF-8 mojibake, appears in chardata.py.
247#
248# This is a verbose regular expression, with whitespace added for somewhat more
249# readability. Remember that the only spaces that count as literal spaces in this
250# expression are ones inside character classes (square brackets).
252BADNESS_RE = re.compile(
253 r"""
254 [{c1}]
255 |
256 [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}]
257 |
258 [a-zA-Z] [{lower_common}{upper_common}] [{bad}]
259 |
260 [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}]
261 |
262 [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}]
263 |
264 [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}]
265 |
266 # leave out [upper_accented][currency] without further info, because it's used in some
267 # fancy leetspeak-esque writing
268 [{lower_accented}{box}{end_punctuation}] [{currency}]
269 |
270 \s [{upper_accented}] [{currency}]
271 |
272 [{upper_accented}{box}] [{numeric}]
273 |
274 [{lower_accented}{upper_accented}{box}{currency}{end_punctuation}] [{start_punctuation}] [{numeric}]
275 |
276 [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}]
277 |
278 [{currency}{numeric}{box}] [{start_punctuation}]
279 |
280 [a-z] [{upper_accented}] [{start_punctuation}{currency}]
281 |
282 [{box}] [{kaomoji}]
283 |
284 [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}]
285 |
286 [{box}] [{end_punctuation}]
287 |
288 [{lower_accented}{upper_accented}] [{end_punctuation}] \w
289 |
291 # The ligature œ when not followed by an unaccented Latin letter
292 [Œœ][^A-Za-z]
293 |
295 # Common Windows-1252 2-character mojibake that isn't covered by the cases above
296 [ÂÃÎÐ][€Šš¢£Ÿž\xa0\xad®©°·»{start_punctuation}{end_punctuation}–—´]
297 |
298 × [²³]
299 |
300 # Windows-1252 mojibake of Arabic words needs to include the 'common' characters.
301 # To compensate, we require four characters to be matched.
302 [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»]
303 [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»]
304 |
306 # Windows-1252 mojibake that starts 3-character sequences for some South Asian
307 # alphabets
308 à[²µ¹¼½¾]
309 |
311 # MacRoman mojibake that isn't covered by the cases above
312 √[±∂†≠®™´≤≥¥µø]
313 |
314 ≈[°¢]
315 |
316 ‚Ä[ìîïòôúùû†°¢π]
317 |
318 ‚[âó][àä°ê]
319 |
321 # Windows-1251 mojibake of characters in the U+2000 range
322 вЂ
323 |
325 # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet.
326 # Because the 2-character sequences involved here may be common, we require
327 # seeing a 3-character sequence.
328 [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС]
329 |
330 # A distinctive five-character sequence of Cyrillic letters, which can be
331 # Windows-1251 mojibake on top of Latin-1 mojibake of Windows-1252 characters.
332 # Require a Latin letter nearby.
333 ГўВЂВ.[A-Za-z ]
334 |
336 # Windows-1252 encodings of 'à' and 'á', as well as \xa0 itself
337 Ã[\xa0¡]
338 |
339 [a-z]\s?[ÃÂ][ ]
340 |
341 ^[ÃÂ][ ]
342 |
344 # Cases where  precedes a character as an encoding of exactly the same
345 # character, and the character is common enough
346 [a-z.,?!{end_punctuation}] Â [ {start_punctuation}{end_punctuation}]
347 |
349 # Windows-1253 mojibake of characters in the U+2000 range
350 β€[™\xa0Ά\xad®°]
351 |
353 # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet
354 [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ]
355""".format(
356 **MOJIBAKE_CATEGORIES
357 ),
358 re.VERBOSE,
359)
362def sequence_weirdness(text: str) -> int:
363 """
364 This was the name of the heuristic used in ftfy 2.x through 5.x. As an
365 attempt at compatibility with external code that calls the heuristic
366 directly, we redirect to our new heuristic, :func:`badness`.
367 """
368 warnings.warn(
369 "`sequence_weirdness()` is an old heuristic, and the current "
370 "closest equivalent is `ftfy.badness.badness()`"
371 )
372 return badness(text)
375def badness(text: str) -> int:
376 """
377 Get the 'badness' of a sequence of text, counting the number of unlikely
378 character sequences. A badness greater than 0 indicates that some of it
379 seems to be mojibake.
380 """
381 return len(BADNESS_RE.findall(text))
384def is_bad(text: str) -> bool:
385 """
386 Returns true iff the given text looks like it contains mojibake.
388 This can be faster than `badness`, because it returns when the first match
389 is found to a regex instead of counting matches. Note that as strings get
390 longer, they have a higher chance of returning True for `is_bad(string)`.
391 """
392 return bool(BADNESS_RE.search(text))