Coverage for /pythoncovmergedfiles/medio/medio/src/turbohtml/tools/fuzz/parser_byte_oracles.py: 37%

1"""Fixed byte fixtures expose shared decoding mistakes that parser agreement misses.""" 

2 

3from __future__ import annotations 

4 

5import re 

6from typing import TYPE_CHECKING, Final 

7 

8from turbohtml import IncrementalParser, parse 

9 

10if TYPE_CHECKING: 

11 import random 

12 from collections.abc import Callable 

13 

14 

15def parser_bytes_check( 

16 case: str, 

17 one_shot: Callable[[bytes, str], tuple[str, str | None]] | None = None, 

18 stream: Callable[[bytes, str, int], tuple[str, str | None]] | None = None, 

19) -> str | None: 

20 """Check literal text and labels because two parsers can agree on a wrong decoding.""" 

21 variant, separator, leaf = case.partition(":") 

22 if not separator or variant not in _FIXTURES or re.fullmatch(r"[a-z][a-z0-9]{0,15}", leaf) is None: 

23 raise UnsupportedParserCaseError(case) 

24 label, encoding, prefix, payload, text, framing = _FIXTURES[variant] 

25 data: Final = prefix + f"<p>{leaf}:".encode(framing) + payload + "</p>".encode(framing) 

26 expected: Final = (f"{leaf}:{text}", encoding) 

27 if (one_shot or _parse_bytes)(data, label) != expected: 

28 return "one-shot parser differs from fixed text or label" 

29 for width in (1, 2, 3, 4, 5, len(data)): 

30 if (stream or _stream_bytes)(data, label, width) != expected: 

31 return "streaming parser differs from fixed text or label" 

32 return None 

33 

34 

35def _parse_bytes(data: bytes, label: str) -> tuple[str, str | None]: 

36 document: Final = parse(data, encoding=label) 

37 return document.text, document.encoding 

38 

39 

40def _stream_bytes(data: bytes, label: str, width: int) -> tuple[str, str | None]: 

41 parser: Final = IncrementalParser(encoding=label) 

42 parser.feed(b"") 

43 for start in range(0, len(data), width): 

44 parser.feed(data[start : start + width]) 

45 parser.feed(b"") 

46 document: Final = parser.close() 

47 return document.text, document.encoding 

48 

49 

50def parser_bytes_generate(rng: random.Random) -> str: 

51 """Keep generated prefixes inside the fixed byte fixtures' encoding domain.""" 

52 return f"{rng.choice(tuple(_FIXTURES))}:g{rng.randrange(10000)}" 

53 

54 

55def parser_bytes_seeds() -> list[str]: 

56 """Cover each codec and BOM override before random generation.""" 

57 return [f"{variant}:g{index}" for variant in _FIXTURES for index in range(30)] 

58 

59 

60def parser_bytes_controls() -> dict[str, bool]: 

61 """Detect wrong labels and shared wrong text before trusting chunk agreement.""" 

62 return { 

63 "wrong one-shot text": parser_bytes_check("utf8:g", lambda _data, _label: ("", "UTF-8")) is not None, 

64 "wrong one-shot label": parser_bytes_check("utf8:g", lambda _data, _label: ("g:café 日本😀", None)) is not None, 

65 "wrong streaming text": parser_bytes_check("utf8:g", stream=lambda _data, _label, _width: ("", "UTF-8")) 

66 is not None, 

67 "wrong streaming label": parser_bytes_check( 

68 "utf8:g", stream=lambda _data, _label, _width: ("g:café 日本😀", None) 

69 ) 

70 is not None, 

71 "shared wrong result": parser_bytes_check( 

72 "utf8:g", lambda _data, _label: ("", None), lambda _data, _label, _width: ("", None) 

73 ) 

74 is not None, 

75 } 

76 

77 

78class UnsupportedParserCaseError(ValueError): 

79 """Keep invalid fixture grammar distinct from parser rejection of valid bytes.""" 

80 

81 

82_FIXTURES: Final[dict[str, tuple[str, str, bytes, bytes, str, str]]] = { 

83 "utf8": ("utf-8", "UTF-8", b"", b"caf\xc3\xa9 \xe6\x97\xa5\xe6\x9c\xac\xf0\x9f\x98\x80", "café 日本😀", "ascii"), 

84 "shift-jis": ("shift_jis", "Shift_JIS", b"", b"\x93\xfa\x96{\x8c\xea", "日本語", "ascii"), 

85 "iso-2022-jp": ("iso-2022-jp", "ISO-2022-JP", b"", b"\x1b$BF|K\\8l\x1b(B", "日本語", "ascii"), 

86 "windows1252": ("windows-1252", "windows-1252", b"", b"caf\xe9 \x80", "café €", "ascii"), 

87 "utf8-bom": ("windows-1252", "UTF-8", b"\xef\xbb\xbf", b"\xc3\xa9", "é", "ascii"), 

88 "utf16le-bom": ("windows-1252", "UTF-16LE", b"\xff\xfe", b"\xe9\x00", "é", "utf-16-le"), 

89 "utf16be-bom": ("utf-8", "UTF-16BE", b"\xfe\xff", b"\x00\xe9", "é", "utf-16-be"), 

90} 

91 

92 

93__all__ = [ 

94 "UnsupportedParserCaseError", 

95 "parser_bytes_check", 

96 "parser_bytes_controls", 

97 "parser_bytes_generate", 

98 "parser_bytes_seeds", 

99]