Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage 1c: Pure ASCII detection (with null-separator tolerance).
3Note: ``from __future__ import annotations`` is intentionally omitted because
4this module is compiled with mypyc, which does not support PEP 563 string
5annotations.
6"""
8from chardet._utils import count_deleted
9from chardet.pipeline import ASCII_TEXT_BYTES, DetectionResult
11# Maximum fraction of null bytes to still classify data as ASCII.
12# Null-separated CLI output (find -print0, git ls-tree -z) typically has
13# 1-3.5% nulls. 5% covers all realistic cases while staying well below
14# the UTF-16 guard threshold (15%).
15_MAX_NULL_FRACTION = 0.05
18def detect_ascii(data: bytes) -> DetectionResult | None:
19 r"""Return an ASCII result if all bytes are printable ASCII plus common whitespace.
21 Tolerates sparse null bytes (``\x00``) up to ``_MAX_NULL_FRACTION`` of
22 the data, returning confidence 0.99 instead of 1.0 to distinguish from
23 pure ASCII.
25 :param data: The raw byte data to examine.
26 :returns: A :class:`DetectionResult` for ASCII, or ``None``.
27 """
28 if not data:
29 return None
30 # Non-ASCII data can never pass: the disallowed set includes every high
31 # byte. ``isascii`` answers that in one allocation-free C scan, so
32 # large non-ASCII inputs skip the counting pass entirely.
33 if not data.isascii():
34 return None
35 # Count rather than materialize the remainder: on a large all-ASCII
36 # window the deletion translate would allocate an input-sized buffer
37 # just to hand back an empty result.
38 # ASCII_TEXT_BYTES is the *allowed* set, so what the deletion leaves is
39 # the disallowed bytes: count them as the complement.
40 disallowed = len(data) - count_deleted(data, ASCII_TEXT_BYTES)
41 if disallowed == 0:
42 return DetectionResult(encoding="ascii", confidence=1.0, language=None)
43 null_count = data.count(0)
44 # Any disallowed byte that is not a null disqualifies the data.
45 if disallowed != null_count:
46 return None
47 # All non-allowed bytes are nulls — accept if sparse enough
48 null_fraction = null_count / len(data)
49 if null_fraction <= _MAX_NULL_FRACTION:
50 return DetectionResult(encoding="ascii", confidence=0.99, language=None)
51 return None