Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/pipeline/bom.py: 41%

1"""Stage 1a: BOM (Byte Order Mark) detection.""" 

2 

3from __future__ import annotations 

4 

5from chardet._utils import decodes_without_error 

6from chardet.pipeline import DetectionResult 

7 

8# Where two marks share a prefix, the longer must come first: UTF-32 is 

9# checked before UTF-16 because the UTF-32-LE BOM starts with the UTF-16-LE 

10# BOM. The UTF-7 entries share no prefix with anything and sit last. 

11_BOMS: tuple[tuple[bytes, str], ...] = ( 

12 (b"\x00\x00\xfe\xff", "utf-32"), 

13 (b"\xff\xfe\x00\x00", "utf-32"), 

14 (b"\xef\xbb\xbf", "utf-8-sig"), 

15 (b"\xfe\xff", "utf-16"), 

16 (b"\xff\xfe", "utf-16"), 

17 # UTF-7 signatures: U+FEFF encoded in UTF-7 ("+/v8-" and friends). 

18 # The fourth base64 character varies with what follows the BOM, giving 

19 # four prefixes (RFC 2152). All four are ASCII bytes, so without these 

20 # marks a signed UTF-7 file reads as plain ASCII and the signature is 

21 # returned to the caller as literal text. Unlike the other marks these 

22 # bytes occur in ordinary text ("+/v8/src/api.cc" in a diff), so a 

23 # UTF-7 match additionally requires the whole buffer to decode. 

24 (b"+/v8", "utf-7"), 

25 (b"+/v9", "utf-7"), 

26 (b"+/v+", "utf-7"), 

27 (b"+/v/", "utf-7"), 

28) 

29 

30_UTF32_BOMS: frozenset[bytes] = frozenset({b"\x00\x00\xfe\xff", b"\xff\xfe\x00\x00"}) 

31 

32 

33def detect_bom(data: bytes) -> DetectionResult | None: 

34 """Check for a byte order mark at the start of *data*. 

35 

36 :param data: The raw byte data to examine. 

37 :returns: A :class:`DetectionResult` with confidence 1.0, or ``None``. 

38 """ 

39 for bom_bytes, encoding in _BOMS: 

40 if data.startswith(bom_bytes): 

41 # UTF-32 BOMs overlap with UTF-16 BOMs (e.g. FF FE 00 00 starts 

42 # with the UTF-16-LE BOM FF FE). Validate that the payload after 

43 # a UTF-32 BOM is a valid number of UTF-32 code units (multiple of 

44 # 4 bytes). If not, skip to let the shorter UTF-16 BOM match. 

45 if bom_bytes in _UTF32_BOMS: 

46 payload_len = len(data) - len(bom_bytes) 

47 if payload_len % 4 != 0: 

48 continue 

49 # A UTF-7 signature is only believable if the data is UTF-7: 

50 # the prefix alone is ordinary ASCII (see the table comment). 

51 if encoding == "utf-7" and not decodes_without_error(data, "utf-7"): 

52 continue 

53 return DetectionResult(encoding=encoding, confidence=1.0, language=None) 

54 return None