Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/output_names.py: 59%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Public-API encoding-name remapping.
3Two output transforms applied to detection results before they cross the
4public API:
6* :func:`apply_preferred_superset` -- when the ``prefer_superset`` API option
7 is enabled, replaces detected ISO/subset encoding names with their
8 Windows/CP supersets that modern software actually uses
9 (e.g., ISO-8859-1 -> Windows-1252).
11* :func:`apply_compat_names` -- when the default ``compat_names=True`` mode
12 is enabled, maps internal Python codec names to the names chardet 5.x/6.x
13 returned, preserving backward compatibility for callers that compare
14 encoding strings directly.
16Both transforms operate in-place on a :class:`~chardet.pipeline.DetectionDict`
17and return the same dict for fluent chaining.
18"""
20from __future__ import annotations
22import codecs
24from chardet._utils import decodes_completely, decodes_without_error
25from chardet.pipeline import DetectionDict
27# Preferred superset name for each encoding, used by the ``prefer_superset``
28# API option. When enabled, detected encoding names are replaced with the
29# Windows/CP superset that modern software actually uses (browsers, editors,
30# etc. treat these ISO subsets as their Windows counterparts).
31# This is a *selection*, not the codec fact: the registry owns which
32# superset each encoding has (``EncodingInfo.superset``), and this table is
33# the frozen subset of those pairs the option applies. It is public through
34# ``UniversalDetector.LEGACY_MAP``, so adding a pair changes
35# ``prefer_superset=True`` output (ADR-0007). ``iso8859-11`` is an alias of
36# the registry's ``tis-620``, which the pipeline reports instead.
37# Values are lowercase codec names, like the keys: this table only swaps one
38# codec for another. Display casing ("Windows-1252") is a separate step that
39# :data:`_COMPAT_NAMES` applies afterwards, and a display-cased value here
40# would reach it under a name it cannot map and no codec accepts.
41PREFERRED_SUPERSET: dict[str, str] = {
42 "ascii": "cp1252",
43 "euc_kr": "cp949",
44 "iso8859-1": "cp1252",
45 "iso8859-2": "cp1250",
46 "iso8859-5": "cp1251",
47 "iso8859-6": "cp1256",
48 "iso8859-7": "cp1253",
49 "iso8859-8": "cp1255",
50 "iso8859-9": "cp1254",
51 "iso8859-11": "cp874",
52 "iso8859-13": "cp1257",
53 "tis-620": "cp874",
54}
57# Mapping from Python codec names to chardet 5.x/6.x compatible display names.
58# Only entries where codec name differs from the compat output are listed.
59# Encodings where codec name == compat name (e.g., "ascii", "utf-8") and
60# encodings new to v7 have no entry — the codec name passes through unchanged.
61_COMPAT_NAMES: dict[str, str] = {
62 # 5.x compat — these encodings existed in chardet 5.x with different names
63 "big5hkscs": "Big5",
64 "cp855": "IBM855",
65 "cp866": "IBM866",
66 "cp874": "CP874",
67 "cp932": "CP932",
68 "cp949": "CP949",
69 "euc_jis_2004": "EUC-JP",
70 "euc_kr": "EUC-KR",
71 "gb18030": "GB18030",
72 "hz": "HZ-GB-2312",
73 "iso2022_jp_2": "ISO-2022-JP",
74 "iso2022_kr": "ISO-2022-KR",
75 "iso8859-1": "ISO-8859-1",
76 "iso8859-2": "ISO-8859-2",
77 "iso8859-5": "ISO-8859-5",
78 "iso8859-6": "ISO-8859-6",
79 "iso8859-7": "ISO-8859-7",
80 "iso8859-8": "ISO-8859-8",
81 "iso8859-9": "ISO-8859-9",
82 "iso8859-13": "ISO-8859-13",
83 "johab": "Johab",
84 "koi8-r": "KOI8-R",
85 "mac-cyrillic": "MacCyrillic",
86 "mac-roman": "MacRoman",
87 "shift_jis_2004": "SHIFT_JIS",
88 "tis-620": "TIS-620",
89 "utf-16": "UTF-16",
90 "utf-32": "UTF-32",
91 "utf-8-sig": "UTF-8-SIG",
92 "cp1250": "Windows-1250",
93 "cp1251": "Windows-1251",
94 "cp1252": "Windows-1252",
95 "cp1253": "Windows-1253",
96 "cp1254": "Windows-1254",
97 "cp1255": "Windows-1255",
98 "cp1256": "Windows-1256",
99 "cp1257": "Windows-1257",
100 # 6.x compat — new in chardet 6.x with different names
101 "kz1048": "KZ1048",
102 "mac-greek": "MacGreek",
103 "mac-iceland": "MacIceland",
104 "mac-latin2": "MacLatin2",
105 "mac-turkish": "MacTurkish",
106}
109def _remap_encoding(result: DetectionDict, mapping: dict[str, str]) -> DetectionDict:
110 """Replace the encoding name using *mapping*, modifying *result* in-place."""
111 enc = result.get("encoding")
112 if isinstance(enc, str):
113 result["encoding"] = mapping.get(enc, enc)
114 return result
117def apply_preferred_superset(
118 result: DetectionDict,
119 data: bytes | None = None,
120 *,
121 include_encodings: frozenset[str] | None = None,
122 exclude_encodings: frozenset[str] | None = None,
123) -> DetectionDict:
124 """Replace the encoding name with its preferred Windows/CP superset.
126 Modifies the ``"encoding"`` value in *result* in-place and returns *result*
127 for fluent chaining.
129 The Windows code pages leave a few C1 positions undefined that their ISO
130 subsets map (0x81, 0x8D, 0x8F, 0x90, 0x9D under cp1252), so the remap is
131 decode-safe only for data that avoids them. When *data* is given, the
132 remap applies only if the superset decodes it; otherwise the detected
133 name stands, being the one that does.
135 :param result: A detection result dict containing an ``"encoding"`` key.
136 :param data: The bytes *result* was detected from, when known.
137 :param include_encodings: Canonical encoding names (as returned by
138 :func:`~chardet.registry.normalize_encodings`) allowed in the result.
139 The superset is skipped when it is not one of them.
140 :param exclude_encodings: Canonical encoding names forbidden in the
141 result. The superset is skipped when it is one of them.
142 :returns: The same *result* dict, modified in-place.
143 """
144 enc = result.get("encoding")
145 if not isinstance(enc, str):
146 return result
147 superset = PREFERRED_SUPERSET.get(enc)
148 if superset is None:
149 return result
150 if include_encodings is not None and superset not in include_encodings:
151 return result
152 if exclude_encodings is not None and superset in exclude_encodings:
153 return result
154 if data is None or decodes_without_error(data, superset):
155 result["encoding"] = superset
156 return result
159# Deprecated alias — kept for external consumers.
160apply_legacy_rename = apply_preferred_superset
163#: Internal names whose display name resolves to a *narrower* Python codec.
164#: ``EUC-JP`` is ``euc_jp``, which lacks the JIS X 0213 rows that
165#: ``euc_jis_2004`` validated; ``SHIFT_JIS``, ``Big5`` and ``ISO-2022-JP``
166#: narrow ``shift_jis_2004``, ``big5hkscs`` and ``iso2022_jp_2`` the same
167#: way. A test pins this set to what :mod:`codecs` reports, so a new
168#: mapping cannot narrow silently.
169NARROWER_DISPLAY_NAMES: frozenset[str] = frozenset(
170 {"big5hkscs", "euc_jis_2004", "iso2022_jp_2", "shift_jis_2004"}
171)
174def display_codec(encoding: str) -> str:
175 """Return the codec that *encoding*'s display name resolves to.
177 ``compat_names=True`` reports ``shift_jis_2004`` as ``SHIFT_JIS``, which
178 :mod:`codecs` resolves to plain ``shift_jis``; that narrower codec is
179 the one a caller decodes with. Names without a display entry resolve
180 to themselves.
182 :param encoding: An internal (canonical registry) encoding name.
183 :returns: The Python codec name behind the name the caller sees.
184 """
185 return codecs.lookup(_COMPAT_NAMES.get(encoding, encoding)).name
188def decodes_under_public_name(data: bytes, encoding: str, *, complete: bool) -> bool:
189 """Return whether *data* decodes under the name the caller will see.
191 The display codec is checked only when it is narrower than *encoding*
192 (:data:`NARROWER_DISPLAY_NAMES`); every other display name is the same
193 codec under another spelling.
195 :param data: The bytes the caller will decode.
196 :param encoding: The internal codec name of the result.
197 :param complete: ``True`` when *data* is the caller's whole input: then
198 *encoding* itself is checked too, and both checks are strict
199 (:func:`~chardet._utils.decodes_completely`), the question the
200 decode-safety flip asks. ``False`` when *encoding* already passed
201 the tolerant validity check: then only the display codec is
202 checked, tolerating a cut sequence at the end as validity does.
203 :returns: ``True`` if the caller's decode under the public name works.
204 """
205 if complete:
206 if not decodes_completely(data, encoding):
207 return False
208 return encoding not in NARROWER_DISPLAY_NAMES or decodes_completely(
209 data, _COMPAT_NAMES[encoding]
210 )
211 return encoding not in NARROWER_DISPLAY_NAMES or decodes_without_error(
212 data, _COMPAT_NAMES[encoding]
213 )
216def apply_compat_names(
217 result: DetectionDict,
218 data: bytes | None = None,
219) -> DetectionDict:
220 """Convert internal codec names to chardet 5.x/6.x compatible names.
222 Modifies the ``"encoding"`` value in *result* in-place and returns *result*
223 for fluent chaining.
225 The internal name is the codec the pipeline validated the data under.
226 For the names in :data:`NARROWER_DISPLAY_NAMES` the display name is a
227 smaller codec, so when *data* is given the rename applies only if the
228 display codec decodes it; otherwise the internal name stands, which
229 Python's codecs accept as is. Without *data* the rename is
230 unconditional, as before.
232 :param result: A detection result dict containing an ``"encoding"`` key.
233 :param data: The bytes *result* was detected from, when known.
234 :returns: The same *result* dict, modified in-place.
235 """
236 enc = result.get("encoding")
237 if (
238 data is not None
239 and isinstance(enc, str)
240 and not decodes_under_public_name(data, enc, complete=False)
241 ):
242 return result
243 return _remap_encoding(result, _COMPAT_NAMES)