Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/pipeline/bom.py: 35%

1"""Stage 1a: BOM (Byte Order Mark) detection.""" 

2 

3from __future__ import annotations 

4 

5from chardet._utils import decodes_without_error 

6from chardet.pipeline import DetectionResult 

7 

8# Where two marks share a prefix, the longer must come first: UTF-32 is 

9# checked before UTF-16 because the UTF-32-LE BOM starts with the UTF-16-LE 

10# BOM. The UTF-7 entries share no prefix with anything and sit last. 

11_BOMS: tuple[tuple[bytes, str], ...] = ( 

12 (b"\x00\x00\xfe\xff", "utf-32"), 

13 (b"\xff\xfe\x00\x00", "utf-32"), 

14 (b"\xef\xbb\xbf", "utf-8-sig"), 

15 (b"\xfe\xff", "utf-16"), 

16 (b"\xff\xfe", "utf-16"), 

17 # UTF-7 signatures: U+FEFF encoded in UTF-7 ("+/v8-" and friends). 

18 # The fourth base64 character varies with what follows the BOM, giving 

19 # four prefixes (RFC 2152). All four are ASCII bytes, so without these 

20 # marks a signed UTF-7 file reads as plain ASCII and the signature is 

21 # returned to the caller as literal text. These bytes also occur in 

22 # ordinary text ("+/v8/src/api.cc" in a diff), which is one more reason 

23 # every mark is held to the whole-buffer decode below. 

24 (b"+/v8", "utf-7"), 

25 (b"+/v9", "utf-7"), 

26 (b"+/v+", "utf-7"), 

27 (b"+/v/", "utf-7"), 

28) 

29 

30_UTF32_BOMS: frozenset[bytes] = frozenset({b"\x00\x00\xfe\xff", b"\xff\xfe\x00\x00"}) 

31 

32 

33def detect_bom(data: bytes, *, input_truncated: bool = False) -> DetectionResult | None: 

34 """Check for a byte order mark at the start of *data*. 

35 

36 :param data: The raw byte data to examine. 

37 :param input_truncated: Whether the examination window ends before the input. 

38 :returns: A :class:`DetectionResult` with confidence 1.0, or ``None``. 

39 """ 

40 for bom_bytes, encoding in _BOMS: 

41 if data.startswith(bom_bytes): 

42 # UTF-32 BOMs overlap with UTF-16 BOMs (e.g. FF FE 00 00 starts 

43 # with the UTF-16-LE BOM FF FE). Validate that the payload after 

44 # a UTF-32 BOM is a valid number of UTF-32 code units (multiple of 

45 # 4 bytes). If not, skip to let the shorter UTF-16 BOM match, 

46 # unless chardet cut the window mid-unit. The tolerant decode 

47 # below accepts that partial tail but still rejects corruption. 

48 if bom_bytes in _UTF32_BOMS: 

49 payload_len = len(data) - len(bom_bytes) 

50 if payload_len % 4 != 0 and not input_truncated: 

51 continue 

52 # The mark names the encoding, but the answer is only true if the 

53 # body decodes under it: a lone surrogate after a UTF-16 mark, or 

54 # cp1252 bytes after a UTF-8 mark, would otherwise come back at 

55 # 1.0 and raise in the caller's own decode. A mark whose body 

56 # fails is no answer at all, so the other stages take over. 

57 # The usual body behind a UTF-8 mark is plain ASCII, which 

58 # ``isascii`` settles at memory speed; everything else decodes. 

59 body_is_ascii = encoding == "utf-8-sig" and data[len(bom_bytes) :].isascii() 

60 if not body_is_ascii and not decodes_without_error(data, encoding): 

61 # A UTF-32 match admitted only by the relaxed alignment check 

62 # may be UTF-16 starting with NUL. Try its shorter BOM before 

63 # giving up, but preserve rejection for ordinary matches. 

64 if input_truncated and bom_bytes in _UTF32_BOMS and len(data) % 4 != 0: 

65 continue 

66 return None 

67 return DetectionResult(encoding=encoding, confidence=1.0, language=None) 

68 return None