Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1c: Pure ASCII detection (with null-separator tolerance).
3Note: ``from __future__ import annotations`` is intentionally omitted because
4this module is compiled with mypyc, which does not support PEP 563 string
5annotations.
6"""
8from chardet._utils import count_deleted
9from chardet.pipeline import ASCII_TEXT_BYTES, DetectionResult
11# Maximum fraction of null bytes to still classify data as ASCII.
12# Null-separated CLI output (find -print0, git ls-tree -z) typically has
13# 1-3.5% nulls. 5% covers all realistic cases. The UTF-16 stage runs
14# first and its null-separator guard rejects any all-ASCII data with
15# fewer nulls than real UTF-16 carries, so nothing accepted here can
16# have been claimed as UTF-16.
17_MAX_NULL_FRACTION = 0.05
20def detect_ascii(data: bytes) -> DetectionResult | None:
21 r"""Return an ASCII result if all bytes are printable ASCII plus common whitespace.
23 Tolerates sparse null bytes (``\x00``) up to ``_MAX_NULL_FRACTION`` of
24 the data, returning confidence 0.99 instead of 1.0 to distinguish from
25 pure ASCII.
27 :param data: The raw byte data to examine.
28 :returns: A :class:`DetectionResult` for ASCII, or ``None``.
29 """
30 if not data:
31 return None
32 # Non-ASCII data can never pass: the disallowed set includes every high
33 # byte. ``isascii`` answers that in one allocation-free C scan, so
34 # large non-ASCII inputs skip the counting pass entirely.
35 if not data.isascii():
36 return None
37 # Count rather than materialize the remainder: on a large all-ASCII
38 # window the deletion translate would allocate an input-sized buffer
39 # just to hand back an empty result.
40 # ASCII_TEXT_BYTES is the *allowed* set, so what the deletion leaves is
41 # the disallowed bytes: count them as the complement.
42 disallowed = len(data) - count_deleted(data, ASCII_TEXT_BYTES)
43 if disallowed == 0:
44 return DetectionResult(encoding="ascii", confidence=1.0, language=None)
45 null_count = data.count(0)
46 # Any disallowed byte that is not a null disqualifies the data.
47 if disallowed != null_count:
48 return None
49 # All non-allowed bytes are nulls — accept if sparse enough
50 null_fraction = null_count / len(data)
51 if null_fraction <= _MAX_NULL_FRACTION:
52 return DetectionResult(encoding="ascii", confidence=0.99, language=None)
53 return None