Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import importlib
4from codecs import IncrementalDecoder
5from functools import lru_cache
7from .constant import (
8 FREQUENCIES,
9 KO_NAMES,
10 LANGUAGE_SUPPORTED_COUNT,
11 TOO_SMALL_SEQUENCE,
12 ZH_NAMES,
13 _FREQUENCIES_SET,
14 _FREQUENCIES_RANK,
15)
16from .md import _ASCII_CHAR_INFO, _char_info, is_suspiciously_successive_range
17from .models import CoherenceMatches
18from .utils import (
19 is_multi_byte_encoding,
20 is_unicode_range_secondary,
21)
24def encoding_unicode_range(iana_name: str) -> list[str]:
25 """
26 Return associated unicode ranges in a single byte code page.
27 """
28 if is_multi_byte_encoding(iana_name):
29 raise OSError( # Defensive:
30 "Function not supported on multi-byte code page"
31 )
33 decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
35 p: IncrementalDecoder = decoder(errors="ignore")
36 seen_ranges: dict[str, int] = {}
37 character_count: int = 0
39 for i in range(0x40, 0xFF):
40 chunk: str = p.decode(bytes([i]))
42 if chunk:
43 chunk_codepoint = ord(chunk)
44 character_range: str | None = (
45 _ASCII_CHAR_INFO[chunk_codepoint].range
46 if chunk_codepoint < 128
47 else _char_info(chunk).range
48 )
50 if character_range is None:
51 continue
53 if not is_unicode_range_secondary(character_range):
54 if character_range not in seen_ranges:
55 seen_ranges[character_range] = 0
56 seen_ranges[character_range] += 1
57 character_count += 1
59 return sorted(
60 [
61 character_range
62 for character_range in seen_ranges
63 if seen_ranges[character_range] / character_count >= 0.15
64 ]
65 )
68def unicode_range_languages(primary_range: str) -> list[str]:
69 """
70 Return inferred languages used with a unicode range.
71 """
72 languages: list[str] = []
74 for language, characters in FREQUENCIES.items():
75 for character in characters:
76 codepoint = ord(character)
77 info = (
78 _ASCII_CHAR_INFO[codepoint]
79 if codepoint < 128
80 else _char_info(character)
81 )
82 if info.range == primary_range:
83 languages.append(language)
84 break
86 return languages
89@lru_cache()
90def encoding_languages(iana_name: str) -> list[str]:
91 """
92 Single-byte encoding language association. Some code page are heavily linked to particular language(s).
93 This function does the correspondence.
94 """
95 try:
96 unicode_ranges: list[str] = encoding_unicode_range(iana_name)
97 except ImportError: # Defensive: encoding unavailable on this build.
98 return []
100 primary_range: str | None = None
102 for specified_range in unicode_ranges:
103 if "Latin" not in specified_range:
104 primary_range = specified_range
105 break
107 if primary_range is None:
108 return ["Latin Based"]
110 return unicode_range_languages(primary_range)
113@lru_cache()
114def mb_encoding_languages(iana_name: str) -> list[str]:
115 """
116 Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
117 This function does the correspondence.
118 """
119 if (
120 iana_name.startswith("shift_")
121 or iana_name.startswith("iso2022_jp")
122 or iana_name.startswith("euc_j")
123 or iana_name == "cp932"
124 ):
125 return ["Japanese"]
126 if iana_name.startswith("gb") or iana_name in ZH_NAMES:
127 return ["Chinese"]
128 if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
129 return ["Korean"]
131 return []
134@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
135def get_target_features(language: str) -> tuple[bool, bool]:
136 """
137 Determine main aspects from a supported language if it contains accents and if is pure Latin.
138 """
139 target_have_accents: bool = False
140 target_pure_latin: bool = True
142 for character in FREQUENCIES[language]:
143 codepoint = ord(character)
144 info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character)
145 if not target_have_accents and info.accentuated:
146 target_have_accents = True
147 if target_pure_latin and not info.latin:
148 target_pure_latin = False
150 return target_have_accents, target_pure_latin
153def alphabet_languages(
154 characters: list[str], ignore_non_latin: bool = False
155) -> list[str]:
156 """
157 Return associated languages associated to given characters.
158 """
159 languages: list[tuple[str, float]] = []
161 characters_set: frozenset[str] = frozenset(characters)
162 source_have_accents = False
163 for character in characters:
164 codepoint = ord(character)
165 info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character)
166 if info.accentuated:
167 source_have_accents = True
168 break
170 for language, language_characters in FREQUENCIES.items():
171 target_have_accents, target_pure_latin = get_target_features(language)
173 if ignore_non_latin and not target_pure_latin:
174 continue
176 if not target_have_accents and source_have_accents:
177 continue
179 character_count: int = len(language_characters)
181 character_match_count: int = len(_FREQUENCIES_SET[language] & characters_set)
183 ratio: float = character_match_count / character_count
185 if ratio >= 0.2:
186 languages.append((language, ratio))
188 languages = sorted(languages, key=lambda x: x[1], reverse=True)
190 return [compatible_language[0] for compatible_language in languages]
193def characters_popularity_compare(
194 language: str, ordered_characters: list[str]
195) -> float:
196 """
197 Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
198 The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
199 Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
200 """
201 if language not in FREQUENCIES:
202 raise ValueError(f"{language} not available") # Defensive:
204 character_approved_count: int = 0
205 lang_rank: dict[str, int] = _FREQUENCIES_RANK[language]
207 ordered_characters_count: int = len(ordered_characters)
208 target_language_characters_count: int = len(FREQUENCIES[language])
210 large_alphabet: bool = target_language_characters_count > 26
211 large_alphabet_threshold: float = target_language_characters_count / 3
213 expected_projection_ratio: float = (
214 target_language_characters_count / ordered_characters_count
215 )
217 # Single pass: characters present in the language vocabulary, as
218 # (language rank, popularity rank) pairs. The scoring below only ever
219 # needs ranks, never the characters themselves.
220 common_lr: list[int] = []
221 common_orr: list[int] = []
222 for popularity_rank, character in enumerate(ordered_characters):
223 language_rank = lang_rank.get(character)
224 if language_rank is not None:
225 common_lr.append(language_rank)
226 common_orr.append(popularity_rank)
228 for character_rank_in_language, character_rank in zip(common_lr, common_orr):
229 character_rank_projection: int = int(character_rank * expected_projection_ratio)
231 if (
232 not large_alphabet
233 and abs(character_rank_projection - character_rank_in_language) > 4
234 ):
235 continue
237 if (
238 large_alphabet
239 and abs(character_rank_projection - character_rank_in_language)
240 < large_alphabet_threshold
241 ):
242 character_approved_count += 1
243 continue
245 if character_rank_in_language == 0:
246 # before_match_count is structurally 0 here (no pair can have a
247 # smaller language rank): the historic "before <= 4" acceptance
248 # always holds. (The symmetric "after_len == 0" case is
249 # impossible: language ranks are strictly below the language
250 # character count, hence after_len >= 1.)
251 character_approved_count += 1
252 continue
254 after_len: int = target_language_characters_count - character_rank_in_language
256 # Count how many characters appear "before" in both orderings, and
257 # how many appear "at or after" in both orderings. Both counts grow
258 # monotonically and the approval thresholds
259 # (before / rank >= 0.4 or after / after_len >= 0.4) are known
260 # upfront, expressed below as exact integer comparisons: exit as
261 # soon as one is crossed.
262 before_match_count: int = 0
263 after_match_count: int = 0
265 for lr_i, orr_i in zip(common_lr, common_orr):
266 if lr_i < character_rank_in_language:
267 if orr_i < character_rank:
268 before_match_count += 1
269 if 5 * before_match_count >= 2 * character_rank_in_language:
270 character_approved_count += 1
271 break
272 else:
273 if orr_i >= character_rank:
274 after_match_count += 1
275 if 5 * after_match_count >= 2 * after_len:
276 character_approved_count += 1
277 break
279 return character_approved_count / len(ordered_characters)
282def alpha_unicode_split(decoded_sequence: str) -> list[str]:
283 """
284 Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
285 Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
286 One containing the latin letters and the other hebrew.
287 """
288 layers: dict[str, list[str]] = {}
290 # Fast path: track single-layer key to skip dict iteration for single-script text.
291 single_layer_key: str | None = None
292 multi_layer: bool = False
294 # Cache the last character_range and its resolved layer to avoid repeated
295 # is_suspiciously_successive_range calls for consecutive same-range chars.
296 prev_character_range: str | None = None
297 prev_layer_target: str | None = None
299 for character in decoded_sequence:
300 # Reuse the per-codepoint CharInfo cache: info.alpha and info.range
301 # are computed with the very same str.isalpha() / unicode_range()
302 # calls this loop historically made per character occurrence.
303 codepoint: int = ord(character)
304 if codepoint < 128:
305 info = _ASCII_CHAR_INFO[codepoint]
306 else:
307 info = _char_info(character)
309 if not info.alpha:
310 continue
312 character_range: str | None = info.range
314 if character_range is None:
315 continue
317 # Fast path: same range as previous character → reuse cached layer target.
318 if character_range == prev_character_range:
319 if prev_layer_target is not None:
320 layers[prev_layer_target].append(character)
321 continue
323 layer_target_range: str | None = None
325 if multi_layer:
326 for discovered_range in layers:
327 if not is_suspiciously_successive_range(
328 discovered_range, character_range
329 ):
330 layer_target_range = discovered_range
331 break
332 elif single_layer_key is not None:
333 if not is_suspiciously_successive_range(single_layer_key, character_range):
334 layer_target_range = single_layer_key
336 if layer_target_range is None:
337 layer_target_range = character_range
339 if layer_target_range not in layers:
340 layers[layer_target_range] = []
341 if single_layer_key is None:
342 single_layer_key = layer_target_range
343 else:
344 multi_layer = True
346 layers[layer_target_range].append(character)
348 # Cache for next iteration
349 prev_character_range = character_range
350 prev_layer_target = layer_target_range
352 return ["".join(chars).lower() for chars in layers.values()]
355def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
356 """
357 This function merge results previously given by the function coherence_ratio.
358 The return type is the same as coherence_ratio.
359 """
360 per_language_ratios: dict[str, list[float]] = {}
361 for result in results:
362 for sub_result in result:
363 language, ratio = sub_result
364 if language not in per_language_ratios:
365 per_language_ratios[language] = [ratio]
366 continue
367 per_language_ratios[language].append(ratio)
369 merge = [
370 (
371 language,
372 round(
373 sum(per_language_ratios[language]) / len(per_language_ratios[language]),
374 4,
375 ),
376 )
377 for language in per_language_ratios
378 ]
380 return sorted(merge, key=lambda x: x[1], reverse=True)
383def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
384 """
385 We shall NOT return "English—" in CoherenceMatches because it is an alternative
386 of "English". This function only keeps the best match and remove the em-dash in it.
387 """
388 index_results: dict[str, list[float]] = dict()
390 for result in results:
391 language, ratio = result
392 no_em_name: str = language.replace("—", "")
394 if no_em_name not in index_results:
395 index_results[no_em_name] = []
397 index_results[no_em_name].append(ratio)
399 if any(len(index_results[e]) > 1 for e in index_results):
400 filtered_results: CoherenceMatches = []
402 for language in index_results:
403 filtered_results.append((language, max(index_results[language])))
405 return filtered_results
407 return results
410def coherence_ratio(
411 decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
412) -> CoherenceMatches:
413 """
414 Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
415 A layer = Character extraction by alphabets/ranges.
416 """
418 results: list[tuple[str, float]] = []
419 ignore_non_latin: bool = False
421 sufficient_match_count: int = 0
423 lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
424 if "Latin Based" in lg_inclusion_list:
425 ignore_non_latin = True
426 lg_inclusion_list.remove("Latin Based")
428 for layer in alpha_unicode_split(decoded_sequence):
429 # Native counting + stable sort reproduce Counter.most_common()
430 # ordering exactly (ties keep first-appearance order) without the
431 # interpreted Counter machinery in the compiled hot path.
432 char_counts: dict[str, int] = {}
433 for layer_character in layer:
434 char_counts[layer_character] = char_counts.get(layer_character, 0) + 1
436 character_count: int = len(layer)
438 if character_count <= TOO_SMALL_SEQUENCE:
439 continue
441 popular_character_ordered: list[str] = [
442 item[0]
443 for item in sorted(
444 char_counts.items(), key=lambda item: item[1], reverse=True
445 )
446 ]
448 for language in lg_inclusion_list or alphabet_languages(
449 popular_character_ordered, ignore_non_latin
450 ):
451 ratio: float = characters_popularity_compare(
452 language, popular_character_ordered
453 )
455 if ratio < threshold:
456 continue
457 elif ratio >= 0.8:
458 sufficient_match_count += 1
460 results.append((language, round(ratio, 4)))
462 if sufficient_match_count >= 3:
463 break
465 return sorted(
466 filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
467 )