Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1b: charset declarations (HTML/XML/PEP 263) and the answer they give.
3The module has two seams. :func:`iter_declarations` is the parser: it
4reads the head of the data and yields every charset declaration it finds,
5resolved to a registry name, without decoding the window.
6:func:`detect_markup_charset` is the stage: it walks those declarations
7and decides which one, if any, is the answer, including any promotion to
8a Windows superset.
10Note: ``from __future__ import annotations`` is intentionally omitted because
11this module is compiled with mypyc, which does not support PEP 563 string
12annotations.
13"""
15import re
16from collections.abc import Iterator
17from typing import NamedTuple
19from chardet._utils import decodes_without_error
20from chardet.enums import EncodingEra
21from chardet.output_names import display_codec
22from chardet.pipeline import DETERMINISTIC_CONFIDENCE, DetectionResult, PipelineContext
23from chardet.pipeline.structural import compute_structural_score
24from chardet.registry import CJK_SUPERSETS, REGISTRY, lookup_encoding
26# Markup charset declarations that commonly refer to a Windows superset
27# encoding rather than the strict standard encoding. Japanese web content
28# almost universally declares "Shift_JIS" but actually uses CP932 extensions;
29# similarly, Korean web content declares "EUC-KR" but uses CP949/UHC.
30# When the declared encoding resolves to the base (key), we check whether
31# ``superset`` is a better answer. ``reported_codec`` is the codec that the
32# *reported* encoding name resolves to for callers: shift_jis_2004 is
33# displayed as "SHIFT_JIS", which standard codec lookup resolves to plain
34# shift_jis, so that is the codec whose decode must not break for the
35# un-promoted name to be safe to report. The pairs come from the
36# registry's superset fact, limited to the multi-byte CJK encodings
37# (ADR-0007).
40class _Promotion(NamedTuple):
41 """The two names a declared CJK encoding can be reported under."""
43 reported_codec: str
44 superset: str
47_MARKUP_SUPERSET_PROMOTIONS: dict[str, _Promotion] = {
48 name: _Promotion(display_codec(name), superset)
49 for name, superset in CJK_SUPERSETS.items()
50}
51_SCAN_LIMIT = 4096
53# Labels that cannot describe a document whose declaration was read as
54# ASCII bytes: UTF-16 and UTF-32 text would have put null bytes between the
55# letters of the tag. The WHATWG prescan reads such a label as UTF-8, and
56# ``x-user-defined`` as windows-1252, so the declared page still has a
57# chance of decoding under the name it is given.
58_SELF_CONTRADICTING_LABELS = frozenset(
59 {"utf-16", "utf-16-be", "utf-16-le", "utf-32", "utf-32-be", "utf-32-le"}
60)
62# A tag scan stops at the next ``<`` as well as at ``>``. A tag that never
63# closes is not a declaration either way, but with ``[^>]`` every unclosed
64# ``<meta`` ran to the end of the head before failing, and a head made of
65# nothing else cost a thousand times a normal one. The price is a literal
66# ``<`` inside an attribute value, which hides that one tag.
67_XML_ENCODING_RE = re.compile(
68 rb"""<\?xml\s[^<>]*encoding\s*=\s*['"]([^'"]+)['"]""", re.IGNORECASE
69)
70_META_TAG_RE = re.compile(rb"<meta\b([^<>]*)>", re.IGNORECASE)
71# DOTALL so a quoted value may span lines.
72_ATTR_RE = re.compile(
73 rb"""([^\s=/>]+)(?:\s*=\s*(?:(["'])(.*?)\2|([^\s>"']+)))?""",
74 re.IGNORECASE | re.DOTALL,
75)
76_CONTENT_CHARSET_RE = re.compile(
77 rb"""charset[\t\n\f\r ]*=[\t\n\f\r ]*(?:(["'])([^"']+)\1|([^\s;"']+))""",
78 re.IGNORECASE,
79)
81# PEP 263: encoding declaration in the first two lines of a Python file.
82# https://peps.python.org/pep-0263/
83_PEP263_RE = re.compile(rb"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)")
84_PEP263_BLANK_RE = re.compile(rb"^[ \t\f]*(?:[#\r]|$)")
86# Charset declarations in EBCDIC-encoded markup, matched against a cp037
87# decode of the head. Letters, digits, and the anchor characters ``<``,
88# ``>``, ``?``, ``=``, and ``/`` sit at the same code points in every
89# supported EBCDIC code page, so the ``<meta``/``<?xml`` tag anchor, the
90# ``charset=``/``encoding=`` label, and the encoding name itself decode
91# correctly through cp037 regardless of which EBCDIC variant the data
92# actually uses. The tag anchor is required so plain EBCDIC prose that
93# merely mentions ``encoding=NAME`` is not treated as a declaration; the
94# tag span and the declaration tokens are matched by separate regexes so
95# a bogus earlier ``encoding=`` token inside the same tag cannot consume
96# the anchor away from the genuine ``charset=`` that follows it. Quote
97# characters are NOT invariant (e.g. cp1026 moves ``"``), so an optional
98# single junk character stands in for the opening quote. That junk
99# character is grouped with its own trailing whitespace rather than left
100# between two ``\s*`` runs: ``\s*X?\s*`` lets a whitespace byte land in
101# either run, so a value that never arrives (``charset=`` followed by a
102# long space run) backtracks over every split and makes the match
103# quadratic in the whitespace length. ``\s*(?:X\s*)?`` matches the same
104# strings with one way to place each space.
105_EBCDIC_TAG_RE = re.compile(r"<(?:meta|\?xml)[^<>]*", re.IGNORECASE)
106_EBCDIC_DECL_RE = re.compile(
107 r"(?:charset|encoding)\s*=\s*(?:[^\sA-Za-z0-9._-]\s*)?([A-Za-z][A-Za-z0-9._-]+)",
108 re.IGNORECASE,
109)
111# High bytes: EBCDIC text is dominated by bytes >= 0x80 (Latin lowercase
112# letters all sit at 0x81+; other scripts likewise), while ASCII-compatible
113# markup is dominated by bytes < 0x80.
114_MARKUP_HIGH_BYTES = bytes(range(0x80, 0x100))
116# Minimum fraction of high bytes in the head for an EBCDIC scan to be
117# worth attempting.
118_EBCDIC_SCAN_MIN_HIGH_FRACTION = 0.25
121class Declaration(NamedTuple):
122 """One charset declaration found in the head of the data.
124 :ivar encoding: The registry name the label resolves to under its
125 scanner's rule, or ``None`` when no codec answers to the label.
126 :ivar mime_type: ``text/xml`` for an XML prolog, ``text/html`` for a
127 meta tag, ``text/x-python`` for a PEP 263 coding cookie.
128 """
130 encoding: str | None
131 mime_type: str
134def _ascii_label(raw: bytes) -> str | None:
135 """Return *raw* as a stripped ASCII string, or ``None`` if it is not ASCII."""
136 try:
137 return raw.decode("ascii").strip()
138 except UnicodeDecodeError:
139 return None
142def _iter_ebcdic_declarations(data: bytes) -> Iterator[Declaration]:
143 """Yield the charset declarations in EBCDIC-encoded markup.
145 The ASCII regexes cannot see declarations in EBCDIC bytes, so when the
146 head looks like EBCDIC text (dominated by high bytes), decode it through
147 cp037, the EBCDIC page whose letter and digit positions are shared by
148 all variants, and scan the decoded text. Only declarations naming a
149 MAINFRAME-era encoding are yielded. The MIME type follows the tag the
150 declaration sits in.
151 """
152 head = data[:_SCAN_LIMIT]
153 high_count = len(head) - len(head.translate(None, _MARKUP_HIGH_BYTES))
154 if high_count < len(head) * _EBCDIC_SCAN_MIN_HIGH_FRACTION:
155 return
156 decoded = head.decode("cp037", errors="replace")
157 # Scan every declaration token inside every anchor tag: an unrelated
158 # earlier ``charset=``/``encoding=`` token (a query string in an href,
159 # a bogus attribute in the same tag) must not mask a genuine EBCDIC
160 # declaration after it.
161 for tag in _EBCDIC_TAG_RE.finditer(decoded):
162 for match in _EBCDIC_DECL_RE.finditer(tag.group(0)):
163 encoding = lookup_encoding(match.group(1).strip())
164 if encoding is not None and REGISTRY[encoding].era & EncodingEra.MAINFRAME:
165 is_xml = tag.group(0)[1] == "?"
166 yield Declaration(encoding, "text/xml" if is_xml else "text/html")
169def _pep263_encoding(raw_name: str) -> str | None:
170 """Resolve a coding cookie the way :func:`tokenize.detect_encoding` does.
172 Python folds the ``utf-8-*`` spellings to UTF-8 and ``latin-1``,
173 ``iso-8859-1``, and ``iso-latin-1`` (with or without a suffix) to
174 Latin-1 before it consults the codec registry, so ``iso-latin-1`` is a
175 valid cookie even though no codec answers to it, and ``utf-8-sig``
176 names plain UTF-8 rather than a file that starts with a mark.
177 """
178 name = raw_name[:12].lower().replace("_", "-")
179 if name == "utf-8" or name.startswith("utf-8-"):
180 return "utf-8"
181 if name in ("latin-1", "iso-8859-1", "iso-latin-1") or name.startswith(
182 ("latin-1-", "iso-8859-1-", "iso-latin-1-")
183 ):
184 return "iso8859-1"
185 return lookup_encoding(raw_name)
188def _pep263_declaration(data: bytes) -> Declaration | None:
189 """Return the PEP 263 coding cookie in the first two lines of *data*.
191 PEP 263 declarations (e.g. ``# -*- coding: utf-8 -*-``) are only valid
192 on line 1 or line 2 of a Python source file. Both lines have to start
193 inside the first :data:`_SCAN_LIMIT` bytes, the same bound the markup
194 scan uses.
195 """
196 head = data[:_SCAN_LIMIT]
197 # PEP 263 requires a '#' comment marker on line 1 or 2.
198 if b"#" not in head:
199 return None
200 # Python only examines line 2 when line 1 is a comment or blank.
201 lines = head.split(b"\n", 2)
202 match = _PEP263_RE.match(lines[0])
203 if match is None and len(lines) > 1 and _PEP263_BLANK_RE.match(lines[0]):
204 match = _PEP263_RE.match(lines[1])
205 if match is None:
206 return None
207 raw_name = _ascii_label(match.group(1))
208 if raw_name is None:
209 return None
210 return Declaration(_pep263_encoding(raw_name), "text/x-python")
213def _strip_comments(head: bytes) -> bytes:
214 """Return *head* without its ``<!-- -->`` comments.
216 A declaration inside a comment is not a declaration, and the WHATWG
217 prescan skips comments for that reason. As in the prescan, the dashes
218 that open a comment may also close it (``<!-->`` is a whole comment),
219 and a comment that never closes runs to the end of the head.
220 """
221 start = head.find(b"<!--")
222 if start < 0:
223 return head
224 parts: list[bytes] = []
225 pos = 0
226 while start >= 0:
227 parts.append(head[pos:start])
228 end = head.find(b"-->", start + 2)
229 if end < 0:
230 return b"".join(parts)
231 pos = end + 3
232 start = head.find(b"<!--", pos)
233 parts.append(head[pos:])
234 return b"".join(parts)
237def _markup_encoding(raw_name: str) -> str | None:
238 """Resolve a charset label found in ASCII-compatible markup."""
239 if raw_name.lower() == "x-user-defined":
240 return "cp1252"
241 encoding = lookup_encoding(raw_name)
242 if encoding in _SELF_CONTRADICTING_LABELS:
243 return "utf-8"
244 return encoding
247def _meta_attributes(body: bytes) -> dict[bytes, bytes]:
248 """Return the attributes of one meta tag, lowercased by name."""
249 attrs: dict[bytes, bytes] = {}
250 for match in _ATTR_RE.finditer(body):
251 name = match.group(1).lower()
252 if name in attrs:
253 continue
254 value = match.group(3)
255 if value is None:
256 value = match.group(4)
257 if value is not None:
258 attrs[name] = value
259 return attrs
262def _iter_html_charsets(head: bytes) -> Iterator[bytes]:
263 """Yield charset labels from real meta declarations, in document order.
265 ``charset=`` inside some other attribute (a refresh URL, a description)
266 is not a declaration. A declaration is a ``charset`` attribute, or the
267 ``charset`` parameter of an ``http-equiv="content-type"`` tag.
268 ``name="content-type"`` is wrong by the spec but common enough in the
269 wild that it counts too.
270 """
271 for tag in _META_TAG_RE.finditer(head):
272 attrs = _meta_attributes(tag.group(1))
273 charset = attrs.get(b"charset")
274 if charset is not None and charset.strip():
275 yield charset
276 continue
277 equiv = attrs.get(b"http-equiv")
278 if equiv is None:
279 equiv = attrs.get(b"name")
280 content = attrs.get(b"content")
281 if (
282 equiv is not None
283 and content is not None
284 and equiv.strip().lower() == b"content-type"
285 ):
286 found = _CONTENT_CHARSET_RE.search(content)
287 if found:
288 yield found.group(2) if found.group(1) else found.group(3)
291def iter_declarations(data: bytes) -> Iterator[Declaration]:
292 """Yield every charset declaration in the head of *data*, in scan order.
294 The scan order is the order a reader honors them in:
296 1. ``<?xml ... encoding="..."?>``
297 2. ``<meta charset="...">`` and ``<meta http-equiv="Content-Type"
298 content="...; charset=...">`` (also with ``name`` in place of
299 ``http-equiv``), in document order
300 3. declarations inside EBCDIC-encoded markup, when the head is
301 dominated by high bytes
302 4. PEP 263 ``# -*- coding: ... -*-`` (first two lines only)
304 Markup inside ``<!-- -->`` comments is not scanned. A UTF-16 or UTF-32
305 label is read as UTF-8 and ``x-user-defined`` as windows-1252, as in the
306 WHATWG prescan, since a declaration found in ASCII bytes cannot be
307 describing UTF-16 text. A label that is not ASCII is skipped.
309 This is parsing only: nothing here decodes the window, so whether a
310 declaration is believable is left to :func:`detect_markup_charset`.
311 The scans run lazily, so a consumer that stops at the first answer
312 never pays for the EBCDIC or PEP 263 scan.
314 :param data: The raw byte data to scan.
315 :returns: An iterator of :class:`Declaration`.
316 """
317 if not data:
318 return
320 head = _strip_comments(data[:_SCAN_LIMIT])
322 match = _XML_ENCODING_RE.search(head)
323 if match:
324 raw_name = _ascii_label(match.group(1))
325 if raw_name is not None:
326 yield Declaration(
327 _markup_encoding(raw_name) if raw_name else None, "text/xml"
328 )
330 for raw in _iter_html_charsets(head):
331 raw_name = _ascii_label(raw)
332 if raw_name is not None:
333 yield Declaration(_markup_encoding(raw_name), "text/html")
335 yield from _iter_ebcdic_declarations(data)
337 pep263 = _pep263_declaration(data)
338 if pep263 is not None:
339 yield pep263
342class _Verdict(NamedTuple):
343 """What one declaration says about the window.
345 :ivar answer: The name to report, or ``None``.
346 :ivar settled: Whether the declared encoding or its Windows superset
347 decodes the window, allowed or not. A settled declaration ends
348 the scan even when it has no answer to give: the page named an
349 encoding that fits its bytes, and when the caller's filters rule
350 that out, a later declaration is not more trustworthy for it.
351 Often the later one is a Latin-1 default that decodes any bytes.
352 """
354 answer: str | None
355 settled: bool
358_UNSETTLED = _Verdict(None, settled=False)
361def _declared_answer(data: bytes, encoding: str, allowed: frozenset[str]) -> _Verdict:
362 """Return what a declaration of *encoding* says about *data*.
364 A declaration is an answer only if the declared encoding, or its Windows
365 superset, decodes the whole of *data*, and only if that name is in
366 *allowed*. (When the declared encoding decodes but the narrower codec
367 its display name resolves to does not, the superset is preferred; if
368 that fails too, the declared name stands and rendering keeps the
369 internal name, which decodes.) The decoder stops at the first bad
370 sequence, so a declaration that lies about the header fails as fast as
371 a head-only check would, and a cut multi-byte sequence at the very end
372 is tolerated as everywhere else. A declaration that decodes but names
373 nothing in *allowed* is settled without an answer (see :class:`_Verdict`).
375 For a declared CJK encoding with a Windows superset in *allowed*
376 (Shift_JIS and CP932, EUC-KR and CP949), the superset is reported when
377 its structural score on the head is strictly higher and it decodes, or
378 when it decodes and the name the caller would see for the declared
379 encoding does not (a declared Shift_JIS page using NEC extensions, a
380 declared EUC-KR page whose first CP949-only hangul sits past the
381 header). The decode checks read the whole window: what a caller can
382 ``.decode()`` is a fact about their entire input. The structural
383 comparison reads only the first :data:`_SCAN_LIMIT` bytes, since
384 multi-byte structure is uniform enough that the ranking has converged
385 by then. It runs first because it is cheap and settles which decode
386 the answer needs: a 200 kB window decodes in a few hundred
387 microseconds, and every declared page of a promotable encoding comes
388 through here.
389 """
390 promotion = _MARKUP_SUPERSET_PROMOTIONS.get(encoding)
391 if promotion is None or promotion.superset not in allowed:
392 if decodes_without_error(data, encoding):
393 return _Verdict(encoding if encoding in allowed else None, settled=True)
394 # A failed subset is still settled when the superset the caller
395 # ruled out decodes, as a declared EUC-KR page with CP949 hangul is.
396 settled = promotion is not None and decodes_without_error(
397 data, promotion.superset
398 )
399 return _Verdict(None, settled)
400 reported_codec, superset = promotion
401 if encoding not in allowed:
402 # The declared name cannot be reported. The superset stands in for
403 # it only when the declared encoding itself fails the window; a
404 # declaration that decodes is an answer, just not an allowed one.
405 if decodes_without_error(data, encoding):
406 return _Verdict(None, settled=True)
407 if decodes_without_error(data, superset):
408 return _Verdict(superset, settled=True)
409 return _UNSETTLED
410 head = data[:_SCAN_LIMIT]
411 # The orchestrator's context cannot be shared: its structural cache is
412 # keyed by encoding name only, and this scores a different slice.
413 ctx = PipelineContext()
414 base_score = compute_structural_score(head, REGISTRY[encoding], ctx)
415 superset_score = compute_structural_score(head, REGISTRY[superset], ctx)
416 if superset_score > base_score:
417 # The superset fits the structure better; take it if it decodes.
418 if decodes_without_error(data, superset):
419 return _Verdict(superset, settled=True)
420 if decodes_without_error(data, encoding):
421 return _Verdict(encoding, settled=True)
422 return _UNSETTLED
423 # The reported codec is a subset of the declared one: plain shift_jis
424 # decodes nothing that shift_jis_2004 cannot, and euc_kr reports as
425 # itself. So when the reported codec decodes the window, the declared
426 # encoding does too, and a clean declared page costs this one decode.
427 if decodes_without_error(data, reported_codec):
428 return _Verdict(encoding, settled=True)
429 # Decode-safety: if the reported name cannot decode the data, the
430 # superset is the only answer a caller can actually use with
431 # ``.decode()``, regardless of the structural tie.
432 if decodes_without_error(data, superset):
433 return _Verdict(superset, settled=True)
434 # Neither the reported codec nor the superset decodes. The declared
435 # encoding is still the answer if it decodes (a JIS X 0213 character
436 # that only shift_jis_2004 maps); when the reported codec is the same
437 # codec, its failure above already settled that it does not.
438 if reported_codec != encoding and decodes_without_error(data, encoding):
439 return _Verdict(encoding, settled=True)
440 return _UNSETTLED
443def detect_markup_charset(
444 data: bytes,
445 allowed: frozenset[str],
446 utf8_precheck: DetectionResult | None = None,
447) -> DetectionResult | None:
448 """Return the answer the charset declarations in *data* give, if any.
450 Runs before the ASCII and UTF-8 stages so explicit declarations like
451 ``<?xml encoding="iso-8859-1"?>`` are honored even when the bytes
452 happen to be pure ASCII. Declarations from :func:`iter_declarations`
453 are tried in order. One counts only if the name it resolves to, or its
454 Windows superset (see :func:`_declared_answer`), decodes the whole of
455 *data* and is in *allowed*. A declaration that fails to decode is
456 skipped and the next one is tried, so a stale prolog does not hide a
457 correct meta. A declaration that decodes but is filtered out ends the
458 scan with no answer, leaving the page to the later stages, the same as
459 a page without one. Each failed encoding is decoded once: the head can
460 repeat one label hundreds of times, and every try reads the window.
462 A declaration is honored over pure ASCII, since the declared encoding
463 decodes those bytes identically, but not over genuine UTF-8 structure:
464 data containing valid multi-byte UTF-8 sequences is UTF-8 regardless of
465 what the (frequently stale) declaration claims, and decoding it as the
466 declared encoding would produce mojibake. So when *utf8_precheck* is
467 a UTF-8 result in *allowed* and the first accepted declaration names
468 something else, that UTF-8 result is returned instead, keeping the
469 declaration's MIME type; only the encoding wins.
471 :param data: The raw byte data to scan.
472 :param allowed: The encoding names the caller's filters permit.
473 :param utf8_precheck: The UTF-8 stage's result for *data*, if any.
474 :returns: A :class:`DetectionResult` with confidence 0.95 (or the UTF-8
475 precheck's), or ``None`` when no declaration is accepted.
476 """
477 rejected: set[str] = set()
478 for declaration in iter_declarations(data):
479 encoding = declaration.encoding
480 if encoding is None or encoding in rejected:
481 continue
482 answer, settled = _declared_answer(data, encoding, allowed)
483 if answer is None:
484 if settled:
485 return None
486 rejected.add(encoding)
487 continue
488 if (
489 utf8_precheck is not None
490 and utf8_precheck.encoding != answer
491 and utf8_precheck.encoding in allowed
492 ):
493 return DetectionResult(
494 utf8_precheck.encoding,
495 utf8_precheck.confidence,
496 utf8_precheck.language,
497 declaration.mime_type,
498 )
499 return DetectionResult(
500 encoding=answer,
501 confidence=DETERMINISTIC_CONFIDENCE,
502 language=None,
503 mime_type=declaration.mime_type,
504 )
505 return None