Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1b: charset declaration extraction (HTML/XML/PEP 263) and promotion.
3Note: ``from __future__ import annotations`` is intentionally omitted because
4this module is compiled with mypyc, which does not support PEP 563 string
5annotations.
6"""
8import re
10from chardet._utils import decodes_without_error
11from chardet.enums import EncodingEra
12from chardet.pipeline import DETERMINISTIC_CONFIDENCE, DetectionResult, PipelineContext
13from chardet.pipeline.structural import compute_structural_score
14from chardet.registry import REGISTRY, lookup_encoding
16# Markup charset declarations that commonly refer to a Windows superset
17# encoding rather than the strict standard encoding. Japanese web content
18# almost universally declares "Shift_JIS" but actually uses CP932 extensions;
19# similarly, Korean web content declares "EUC-KR" but uses CP949/UHC.
20# When the declared encoding resolves to the base (key), we check whether
21# the superset (second element) is a better answer. The first element is
22# the codec that the *reported* encoding name resolves to for callers:
23# shift_jis_2004 is displayed as "SHIFT_JIS", which standard codec lookup
24# resolves to plain shift_jis, so that is the codec whose decode must not
25# break for the un-promoted name to be safe to report.
26_MARKUP_SUPERSET_PROMOTIONS: dict[str, tuple[str, str]] = {
27 "shift_jis_2004": ("shift_jis", "cp932"),
28 "euc_kr": ("euc_kr", "cp949"),
29}
31_SCAN_LIMIT = 4096
33_XML_ENCODING_RE = re.compile(
34 rb"""<\?xml[^>]+encoding\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE
35)
36_HTML5_CHARSET_RE = re.compile(
37 rb"""<meta[^>]+charset\s*=\s*['"]?\s*([^\s'">;]+)""", re.IGNORECASE
38)
39_HTML4_CONTENT_TYPE_RE = re.compile(
40 rb"""<meta[^>]+content\s*=\s*['"][^'"]*charset=([^\s'">;]+)""", re.IGNORECASE
41)
43# PEP 263: encoding declaration in the first two lines of a Python file.
44# https://peps.python.org/pep-0263/
45_PEP263_RE = re.compile(rb"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.MULTILINE)
47# Charset declarations in EBCDIC-encoded markup, matched against a cp037
48# decode of the head. Letters, digits, and the anchor characters ``<``,
49# ``>``, ``?``, ``=``, and ``/`` sit at the same code points in every
50# supported EBCDIC code page, so the ``<meta``/``<?xml`` tag anchor, the
51# ``charset=``/``encoding=`` label, and the encoding name itself decode
52# correctly through cp037 regardless of which EBCDIC variant the data
53# actually uses. The tag anchor is required so plain EBCDIC prose that
54# merely mentions ``encoding=NAME`` is not treated as a declaration; the
55# tag span and the declaration tokens are matched by separate regexes so
56# a bogus earlier ``encoding=`` token inside the same tag cannot consume
57# the anchor away from the genuine ``charset=`` that follows it. Quote
58# characters are NOT invariant (e.g. cp1026 moves ``"``), so an optional
59# single junk character stands in for the opening quote.
60_EBCDIC_TAG_RE = re.compile(r"<(?:meta|\?xml)[^>]*", re.IGNORECASE)
61_EBCDIC_DECL_RE = re.compile(
62 r"(?:charset|encoding)\s*=\s*[^\sA-Za-z0-9._-]?\s*([A-Za-z][A-Za-z0-9._-]+)",
63 re.IGNORECASE,
64)
66# High bytes: EBCDIC text is dominated by bytes >= 0x80 (Latin lowercase
67# letters all sit at 0x81+; other scripts likewise), while ASCII-compatible
68# markup is dominated by bytes < 0x80.
69_MARKUP_HIGH_BYTES = bytes(range(0x80, 0x100))
71# Minimum fraction of high bytes in the head for an EBCDIC scan to be
72# worth attempting.
73_EBCDIC_SCAN_MIN_HIGH_FRACTION = 0.25
76def _detect_ebcdic_declaration(head: bytes) -> DetectionResult | None:
77 """Look for a charset declaration in EBCDIC-encoded markup.
79 The ASCII regexes cannot see declarations in EBCDIC bytes, so when the
80 head looks like EBCDIC text (dominated by high bytes), decode it through
81 cp037 — the EBCDIC page whose letter and digit positions are shared by
82 all variants — and scan the decoded text. Only declarations naming a
83 MAINFRAME-era encoding are honoured, and the declared encoding must
84 actually decode the head.
85 """
86 high_count = len(head) - len(head.translate(None, _MARKUP_HIGH_BYTES))
87 if high_count < len(head) * _EBCDIC_SCAN_MIN_HIGH_FRACTION:
88 return None
89 decoded = head.decode("cp037", errors="replace")
90 # Scan every declaration token inside every anchor tag: an unrelated
91 # earlier ``charset=``/``encoding=`` token (a query string in an href,
92 # a bogus attribute in the same tag) must not mask a genuine EBCDIC
93 # declaration after it.
94 for tag in _EBCDIC_TAG_RE.finditer(decoded):
95 for match in _EBCDIC_DECL_RE.finditer(tag.group(0)):
96 encoding = lookup_encoding(match.group(1).strip())
97 if (
98 encoding is not None
99 and REGISTRY[encoding].era & EncodingEra.MAINFRAME
100 and decodes_without_error(head, encoding)
101 ):
102 return DetectionResult(
103 encoding=encoding,
104 confidence=DETERMINISTIC_CONFIDENCE,
105 language=None,
106 mime_type="text/html",
107 )
108 return None
111def _detect_pep263(data: bytes) -> DetectionResult | None:
112 """Check the first two lines of *data* for a PEP 263 encoding declaration.
114 PEP 263 declarations (e.g. ``# -*- coding: utf-8 -*-``) are only valid
115 on line 1 or line 2 of a Python source file.
117 :param data: The raw byte data to scan.
118 :returns: A :class:`DetectionResult` with confidence 0.95, or ``None``.
119 """
120 # PEP 263 requires a '#' comment marker on line 1 or 2.
121 if b"#" not in data[:200]:
122 return None
123 # Extract first two lines only.
124 first_two_lines = b"\n".join(data.split(b"\n", 2)[:2])
125 match = _PEP263_RE.search(first_two_lines)
126 if match:
127 try:
128 raw_name = match.group(1).decode("ascii").strip()
129 except (UnicodeDecodeError, ValueError):
130 return None
131 encoding = lookup_encoding(raw_name)
132 if encoding is not None and _validate_bytes(data, encoding):
133 return DetectionResult(
134 encoding=encoding,
135 confidence=DETERMINISTIC_CONFIDENCE,
136 language=None,
137 mime_type="text/x-python",
138 )
139 return None
142def detect_markup_charset(data: bytes) -> DetectionResult | None:
143 """Scan the first bytes of *data* for a charset declaration.
145 Checks for:
147 1. ``<?xml ... encoding="..."?>``
148 2. ``<meta charset="...">``
149 3. ``<meta http-equiv="Content-Type" content="...; charset=...">``
150 4. PEP 263 ``# -*- coding: ... -*-`` (first two lines only)
152 :param data: The raw byte data to scan.
153 :returns: A :class:`DetectionResult` with confidence 0.95, or ``None``.
154 """
155 if not data:
156 return None
158 head = data[:_SCAN_LIMIT]
160 for pattern in (_XML_ENCODING_RE, _HTML5_CHARSET_RE, _HTML4_CONTENT_TYPE_RE):
161 match = pattern.search(head)
162 if match:
163 try:
164 raw_name = match.group(1).decode("ascii").strip()
165 except (UnicodeDecodeError, ValueError):
166 continue
167 encoding = lookup_encoding(raw_name)
168 if encoding is not None and _validate_bytes(data, encoding):
169 mime_type = "text/xml" if pattern is _XML_ENCODING_RE else "text/html"
170 return DetectionResult(
171 encoding=encoding,
172 confidence=DETERMINISTIC_CONFIDENCE,
173 language=None,
174 mime_type=mime_type,
175 )
177 ebcdic_result = _detect_ebcdic_declaration(head)
178 if ebcdic_result is not None:
179 return ebcdic_result
181 return _detect_pep263(data)
184def promote_markup_superset(
185 data: bytes,
186 markup_result: DetectionResult,
187 allowed: frozenset[str],
188) -> DetectionResult:
189 """Promote a markup-declared encoding to its superset when structural evidence supports it.
191 If the declared encoding has a known superset (per
192 :data:`_MARKUP_SUPERSET_PROMOTIONS`), the superset validates the data,
193 and the superset's structural score is materially better, return a new
194 result using the superset encoding. Otherwise return *markup_result*
195 unchanged.
197 The two decode checks read the whole input, the structural comparison
198 only the first :data:`_SCAN_LIMIT` bytes. That split is deliberate:
199 what a caller can actually ``.decode()`` is a fact about their entire
200 input --- capping it costs real answers, since a declared-Shift_JIS page
201 typically reaches its first CP932-only byte well past the header --- but
202 the structural score is a ranking heuristic that has converged by then,
203 and running it over 200 kB twice on every declared page is the expensive
204 part of this function.
205 """
206 if markup_result.encoding is None:
207 return markup_result
208 promotion = _MARKUP_SUPERSET_PROMOTIONS.get(markup_result.encoding)
209 if promotion is None:
210 return markup_result
211 reported_codec, superset_name = promotion
212 if superset_name not in allowed:
213 return markup_result
214 superset_info = REGISTRY[superset_name]
215 # Validate: superset must be able to decode the data
216 if not decodes_without_error(data, superset_name):
217 return markup_result
218 # Decode-safety: if the codec the reported name resolves to cannot
219 # decode the data (e.g. a declared-Shift_JIS page using CP932 NEC/IBM
220 # extensions), the superset is the only answer a caller can actually
221 # use with ``.decode()`` -- promote unconditionally.
222 if not decodes_without_error(data, reported_codec):
223 return DetectionResult(
224 superset_name,
225 markup_result.confidence,
226 markup_result.language,
227 markup_result.mime_type,
228 )
229 # Compare structural scores
230 # Scored on the head only. Multi-byte structure is uniform enough that
231 # the ranking converges long before 200 kB, and this is the expensive
232 # half: two full-buffer passes on every declared page, where
233 # ``_validate_bytes`` caps the same kind of scan at _SCAN_LIMIT.
234 head = data[:_SCAN_LIMIT]
235 ctx = PipelineContext()
236 base_score = compute_structural_score(head, REGISTRY[markup_result.encoding], ctx)
237 superset_score = compute_structural_score(head, superset_info, ctx)
238 if superset_score > base_score:
239 return DetectionResult(
240 superset_name,
241 markup_result.confidence,
242 markup_result.language,
243 markup_result.mime_type,
244 )
245 return markup_result
248def _validate_bytes(data: bytes, encoding: str) -> bool:
249 """Check that *data* can be decoded under *encoding* without errors.
251 Only validates the first ``_SCAN_LIMIT`` bytes to avoid decoding a
252 full 200 kB input just to verify a charset declaration found in the
253 header.
254 """
255 return decodes_without_error(data[:_SCAN_LIMIT], encoding)