Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/pipeline/binary.py: 40%

1"""Stage 0: Binary content detection.""" 

2 

3from __future__ import annotations 

4 

5from chardet._utils import DEFAULT_MAX_BYTES, count_deleted 

6 

7# Threshold: if more than this fraction of bytes are binary indicators, it's binary 

8_BINARY_THRESHOLD = 0.01 

9 

10# Translation table that maps binary-indicator control bytes (0x00-0x08, 

11# 0x0E-0x1F — excludes \t \n \v \f \r) to None (deleting them) and keeps 

12# everything else. len(data) - len(translated) gives the count in one 

13# C-level pass. 

14_BINARY_DELETE = bytes(range(0x09)) + bytes(range(0x0E, 0x20)) 

15 

16# Minimum fraction of high bytes (>= 0x80) among the *non-space* bytes for 

17# data to plausibly be EBCDIC text. EBCDIC encodes all lowercase letters 

18# above 0x80, so genuine EBCDIC text is dominated by high bytes — but the 

19# EBCDIC space is 0x40, and space-padded fixed-width records (the canonical 

20# mainframe data shape) would dilute a whole-data fraction below any useful 

21# threshold, so padding is excluded from the denominator. 

22_EBCDIC_MIN_HIGH_FRACTION = 0.25 

23 

24# Minimum fraction of EBCDIC word separators (0x40 space or 0x05 HT) in 

25# the data. Text separates words or fields; a binary payload framed with 

26# ENQ/NAK passes the control-byte check yet has no reason to contain 

27# EBCDIC word structure (random bytes put each separator at ~0.4%, and 

28# per-record framing bytes stay well under 2%). HT counts so that 

29# tab-separated record exports with no spaces are still recognized. 

30_EBCDIC_MIN_SPACE_FRACTION = 0.02 

31 

32# The EBCDIC space, horizontal-tab, and newline bytes. 

33_EBCDIC_SPACE = 0x40 

34_EBCDIC_HT = 0x05 

35_EBCDIC_NL = 0x15 

36 

37# High-byte deletion table for the EBCDIC plausibility check. 

38_HIGH_BYTES_DELETE = bytes(range(0x80, 0x100)) 

39 

40 

41def is_binary(data: bytes, max_bytes: int = DEFAULT_MAX_BYTES) -> bool: 

42 """Return ``True`` if *data* appears to be binary (not text) content. 

43 

44 :param data: The raw byte data to examine. 

45 :param max_bytes: Maximum number of bytes to scan. 

46 :returns: ``True`` if the data is classified as binary. 

47 """ 

48 data = data[:max_bytes] 

49 if not data: 

50 return False 

51 

52 binary_count = count_deleted(data, _BINARY_DELETE) 

53 if binary_count / len(data) <= _BINARY_THRESHOLD: 

54 return False 

55 

56 # Above the threshold — but EBCDIC text uses 0x05 (HT) and 0x15 (NL) as 

57 # whitespace, which are binary indicators in ASCII-compatible data. If 

58 # the excess comes entirely from those two bytes and the data looks 

59 # like EBCDIC text, treat it as text. 

60 # 

61 # EBCDIC text uses 0x05 (HT) and 0x15 (NL) as tab and newline, and both 

62 # are in _BINARY_DELETE, so the count excluding them is arithmetic on 

63 # the count including them: no second scan of the data, and the HT 

64 # count is needed for the space fraction below anyway. 

65 ht_count = data.count(_EBCDIC_HT) 

66 hard_count = binary_count - ht_count - data.count(_EBCDIC_NL) 

67 if hard_count / len(data) > _BINARY_THRESHOLD: 

68 return True 

69 space_count = data.count(_EBCDIC_SPACE) + ht_count 

70 non_space = len(data) - space_count 

71 if non_space == 0: 

72 return False 

73 high_count = count_deleted(data, _HIGH_BYTES_DELETE) 

74 if high_count / non_space < _EBCDIC_MIN_HIGH_FRACTION: 

75 return True 

76 return space_count / len(data) < _EBCDIC_MIN_SPACE_FRACTION