Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/output_names.py: 62%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Public-API encoding-name remapping.
3Two output transforms applied to detection results before they cross the
4public API:
6* :func:`apply_preferred_superset` -- when the ``prefer_superset`` API option
7 is enabled, replaces detected ISO/subset encoding names with their
8 Windows/CP supersets that modern software actually uses
9 (e.g., ISO-8859-1 -> Windows-1252).
11* :func:`apply_compat_names` -- when the default ``compat_names=True`` mode
12 is enabled, maps internal Python codec names to the names chardet 5.x/6.x
13 returned, preserving backward compatibility for callers that compare
14 encoding strings directly.
16Both transforms operate in-place on a :class:`~chardet.pipeline.DetectionDict`
17and return the same dict for fluent chaining.
18"""
20from __future__ import annotations
22from chardet._utils import decodes_without_error
23from chardet.pipeline import DetectionDict
25# Preferred superset name for each encoding, used by the ``prefer_superset``
26# API option. When enabled, detected encoding names are replaced with the
27# Windows/CP superset that modern software actually uses (browsers, editors,
28# etc. treat these ISO subsets as their Windows counterparts).
29# Values are lowercase codec names, like the keys: this table only swaps one
30# codec for another. Display casing ("Windows-1252") is a separate step that
31# :data:`_COMPAT_NAMES` applies afterwards, and a display-cased value here
32# would reach it under a name it cannot map and no codec accepts.
33PREFERRED_SUPERSET: dict[str, str] = {
34 "ascii": "cp1252",
35 "euc_kr": "cp949",
36 "iso8859-1": "cp1252",
37 "iso8859-2": "cp1250",
38 "iso8859-5": "cp1251",
39 "iso8859-6": "cp1256",
40 "iso8859-7": "cp1253",
41 "iso8859-8": "cp1255",
42 "iso8859-9": "cp1254",
43 "iso8859-11": "cp874",
44 "iso8859-13": "cp1257",
45 "tis-620": "cp874",
46}
49# Mapping from Python codec names to chardet 5.x/6.x compatible display names.
50# Only entries where codec name differs from the compat output are listed.
51# Encodings where codec name == compat name (e.g., "ascii", "utf-8") and
52# encodings new to v7 have no entry — the codec name passes through unchanged.
53_COMPAT_NAMES: dict[str, str] = {
54 # 5.x compat — these encodings existed in chardet 5.x with different names
55 "big5hkscs": "Big5",
56 "cp855": "IBM855",
57 "cp866": "IBM866",
58 "cp874": "CP874",
59 "cp932": "CP932",
60 "cp949": "CP949",
61 "euc_jis_2004": "EUC-JP",
62 "euc_kr": "EUC-KR",
63 "gb18030": "GB18030",
64 "hz": "HZ-GB-2312",
65 "iso2022_jp_2": "ISO-2022-JP",
66 "iso2022_kr": "ISO-2022-KR",
67 "iso8859-1": "ISO-8859-1",
68 "iso8859-2": "ISO-8859-2",
69 "iso8859-5": "ISO-8859-5",
70 "iso8859-6": "ISO-8859-6",
71 "iso8859-7": "ISO-8859-7",
72 "iso8859-8": "ISO-8859-8",
73 "iso8859-9": "ISO-8859-9",
74 "iso8859-13": "ISO-8859-13",
75 "johab": "Johab",
76 "koi8-r": "KOI8-R",
77 "mac-cyrillic": "MacCyrillic",
78 "mac-roman": "MacRoman",
79 "shift_jis_2004": "SHIFT_JIS",
80 "tis-620": "TIS-620",
81 "utf-16": "UTF-16",
82 "utf-32": "UTF-32",
83 "utf-8-sig": "UTF-8-SIG",
84 "cp1250": "Windows-1250",
85 "cp1251": "Windows-1251",
86 "cp1252": "Windows-1252",
87 "cp1253": "Windows-1253",
88 "cp1254": "Windows-1254",
89 "cp1255": "Windows-1255",
90 "cp1256": "Windows-1256",
91 "cp1257": "Windows-1257",
92 # 6.x compat — new in chardet 6.x with different names
93 "kz1048": "KZ1048",
94 "mac-greek": "MacGreek",
95 "mac-iceland": "MacIceland",
96 "mac-latin2": "MacLatin2",
97 "mac-turkish": "MacTurkish",
98}
101def _remap_encoding(result: DetectionDict, mapping: dict[str, str]) -> DetectionDict:
102 """Replace the encoding name using *mapping*, modifying *result* in-place."""
103 enc = result.get("encoding")
104 if isinstance(enc, str):
105 result["encoding"] = mapping.get(enc, enc)
106 return result
109def apply_preferred_superset(
110 result: DetectionDict,
111 data: bytes | None = None,
112) -> DetectionDict:
113 """Replace the encoding name with its preferred Windows/CP superset.
115 Modifies the ``"encoding"`` value in *result* in-place and returns *result*
116 for fluent chaining.
118 The Windows code pages leave a few C1 positions undefined that their ISO
119 subsets map (0x81, 0x8D, 0x8F, 0x90, 0x9D under cp1252), so the remap is
120 decode-safe only for data that avoids them. When *data* is given, the
121 remap applies only if the superset decodes it; otherwise the detected
122 name stands, being the one that does.
124 :param result: A detection result dict containing an ``"encoding"`` key.
125 :param data: The bytes *result* was detected from, when known.
126 :returns: The same *result* dict, modified in-place.
127 """
128 enc = result.get("encoding")
129 if not isinstance(enc, str):
130 return result
131 superset = PREFERRED_SUPERSET.get(enc)
132 if superset is None:
133 return result
134 if data is None or decodes_without_error(data, superset):
135 result["encoding"] = superset
136 return result
139# Deprecated alias — kept for external consumers.
140apply_legacy_rename = apply_preferred_superset
143def apply_compat_names(
144 result: DetectionDict,
145) -> DetectionDict:
146 """Convert internal codec names to chardet 5.x/6.x compatible names.
148 Modifies the ``"encoding"`` value in *result* in-place and returns *result*
149 for fluent chaining.
151 :param result: A detection result dict containing an ``"encoding"`` key.
152 :returns: The same *result* dict, modified in-place.
153 """
154 return _remap_encoding(result, _COMPAT_NAMES)