Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/pipeline/utf8.py: 18%

1"""Stage 1d: UTF-8 structural validation. 

2 

3Validation is decode-based: CPython's strict UTF-8 decoder enforces exactly 

4the rules the old hand-rolled loop checked (overlong encodings, surrogates, 

5codepoints above U+10FFFF) at C speed on every build flavor, where a per-byte 

6loop runs an order of magnitude slower even compiled. See ADR-0006. 

7 

8The input is fed to an incremental decoder in chunks so a large input never 

9allocates a matching ``str``; decoded output is discarded. ``final=False`` 

10reproduces the truncated-tail tolerance (an incomplete multi-byte sequence at 

11the very end is fine, the input is usually a prefix of a larger whole), the 

12same trick :func:`chardet._utils.decodes_without_error` uses. 

13 

14Note: ``from __future__ import annotations`` is intentionally omitted because 

15this module is compiled with mypyc, which does not support PEP 563 string 

16annotations. 

17""" 

18 

19import codecs 

20 

21from chardet.pipeline import DetectionResult 

22 

23# Confidence curve parameters for UTF-8 detection. 

24# Even a small fraction of valid multi-byte sequences is strong evidence. 

25_BASE_CONFIDENCE = 0.80 

26_MAX_CONFIDENCE = 0.99 

27# Scale factor for the multi-byte byte ratio: mb_ratio * 6 saturates the 

28# confidence ramp at ~17% multi-byte content. 

29_MB_RATIO_SCALE = 6 

30 

31# Chunk size for incremental decoding: large enough that per-chunk overhead 

32# vanishes, small enough that the transient decoded ``str`` stays negligible. 

33_CHUNK_SIZE = 1 << 20 

34 

35# Deleting the ASCII range leaves exactly the high bytes, so 

36# ``len(chunk.translate(None, _ASCII_DELETE))`` counts them at C speed. 

37_ASCII_DELETE = bytes(range(0x80)) 

38 

39_utf8_decoder = codecs.getincrementaldecoder("utf-8") 

40 

41 

42def _expected_seq_len(byte: int) -> int: 

43 """Sequence length a UTF-8 lead byte declares, or 0 for a non-lead. 

44 

45 0xC0-0xC1 are overlong 2-byte encodings of ASCII, so leads start at 0xC2. 

46 """ 

47 if 0xC2 <= byte <= 0xDF: 

48 return 2 

49 if 0xE0 <= byte <= 0xEF: 

50 return 3 

51 if 0xF0 <= byte <= 0xF4: 

52 return 4 

53 return 0 

54 

55 

56def detect_utf8(data: bytes) -> DetectionResult | None: 

57 """Validate UTF-8 byte structure. 

58 

59 Returns a result only if multi-byte sequences are found (pure ASCII 

60 is handled by the ASCII stage). 

61 

62 :param data: The raw byte data to examine. 

63 :returns: A :class:`DetectionResult` for UTF-8, or ``None``. 

64 """ 

65 return scan_utf8(data)[1] 

66 

67 

68def scan_utf8(data: bytes) -> "tuple[bool, DetectionResult | None]": 

69 """Validate UTF-8 structure, separating validity from evidence. 

70 

71 ``None`` from :func:`detect_utf8` is two different verdicts: the data 

72 is *not* UTF-8, or it is valid UTF-8 carrying no multi-byte evidence 

73 (pure ASCII, ASCII plus control bytes, ASCII plus a truncated tail). 

74 Only the first justifies ruling UTF-8 out downstream, so callers that 

75 act on a rejection need both halves of the answer. 

76 

77 :param data: The raw byte data to examine. 

78 :returns: ``(window_is_valid_utf8, result_or_None)``. The flag covers 

79 the whole of *data*; the result is present only when complete 

80 multi-byte sequences were found. 

81 """ 

82 if not data: 

83 return (True, None) 

84 # Pure ASCII is valid UTF-8 with no multi-byte evidence — let the ASCII 

85 # detector handle it. ``isascii`` is a single C scan and skips the 

86 # decode entirely for the common all-ASCII case. 

87 if data.isascii(): 

88 return (True, None) 

89 

90 length = len(data) 

91 decoder = _utf8_decoder() 

92 multibyte_bytes = 0 

93 pending_len = 0 

94 pos = 0 

95 # Start of the incomplete sequence at the very end (== length when the 

96 # data ends on a sequence boundary). Bytes from here on are tolerated, 

97 # not validated, and excluded from the multi-byte counts — matching the 

98 # old validator, which stopped at a truncated final sequence without 

99 # examining it. 

100 tail_start = length 

101 counted_end = 0 

102 while pos < length: 

103 chunk = data[pos : pos + _CHUNK_SIZE] 

104 chunk_end = pos + len(chunk) 

105 counted_end = chunk_end 

106 if pending_len == 0 and chunk.isascii(): 

107 # No pending sequence and no high bytes: trivially valid, 

108 # nothing to count. 

109 pos = chunk_end 

110 continue 

111 multibyte_bytes += len(chunk.translate(None, _ASCII_DELETE)) 

112 try: 

113 decoder.decode(chunk, final=False) 

114 except UnicodeError as exc: 

115 # Only one rejection is tolerated: a final sequence whose 

116 # declared length overruns the data (the caller's input is 

117 # usually a prefix of a larger whole). Locate that sequence 

118 # from the data itself rather than from the exception, whose 

119 # reported offset is an implementation detail — CPython points 

120 # at the lead byte, and this module also runs interpreted on 

121 # PyPy. The overrunning lead can only be in the final three 

122 # bytes, so the search is O(1). 

123 tail_lead = -1 

124 for k in range(max(0, length - 3), length): 

125 seq_len = _expected_seq_len(data[k]) 

126 if seq_len and k + seq_len > length: 

127 tail_lead = k 

128 break 

129 err_start = pos - pending_len + getattr(exc, "start", 0) 

130 err_start = min(max(err_start, 0), length - 1) 

131 if tail_lead < 0 or err_start < tail_lead: 

132 # The error is a genuine one before the tolerated tail. 

133 return (False, None) 

134 # The old validator stopped at a truncated final sequence 

135 # without examining it, so garbage inside the overrun is 

136 # tolerated too. 

137 tail_start = tail_lead 

138 break 

139 pending_len = len(decoder.getstate()[0]) 

140 pos = chunk_end 

141 else: 

142 # Clean finish: the decoder's buffered bytes are the incomplete 

143 # final sequence, if any. 

144 tail_start = length - pending_len 

145 

146 # The counts above include the tolerated tail (and, on the error path, 

147 # nothing past the erroring chunk — the tolerance condition pins the 

148 # error to the last 3 bytes, so ``counted_end`` covers the tail). Trim 

149 # the tail's contribution: at most 3 bytes. 

150 if tail_start < counted_end: 

151 tail = data[tail_start:counted_end] 

152 multibyte_bytes -= len(tail.translate(None, _ASCII_DELETE)) 

153 

154 # In a validly-decoded prefix every high byte belongs to a complete 

155 # multi-byte sequence, so "no complete sequences" is exactly "no high 

156 # bytes outside the tail". Pure ASCII plus a truncated tail — valid, 

157 # but no evidence; let the later stages handle it. 

158 if multibyte_bytes == 0: 

159 return (True, None) 

160 

161 # Confidence scales with the proportion of multi-byte bytes in the data. 

162 # Even a small amount of valid multi-byte UTF-8 is strong evidence. 

163 mb_ratio = multibyte_bytes / length 

164 confidence_range = _MAX_CONFIDENCE - _BASE_CONFIDENCE 

165 confidence = min( 

166 _MAX_CONFIDENCE, 

167 _BASE_CONFIDENCE + confidence_range * min(mb_ratio * _MB_RATIO_SCALE, 1.0), 

168 ) 

169 return ( 

170 True, 

171 DetectionResult(encoding="utf-8", confidence=confidence, language=None), 

172 )