Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/_util.py: 55%

1from __future__ import annotations 

2 

3import re 

4from functools import lru_cache 

5from typing import TYPE_CHECKING 

6 

7if TYPE_CHECKING: 

8 from collections.abc import Iterable 

9 

10 

11def to_unicode( 

12 text: str | bytes, encoding: str | None = None, errors: str = "strict" 

13) -> str: 

14 """Return the unicode representation of a bytes object `text`. If `text` 

15 is already an unicode object, return it as-is.""" 

16 if isinstance(text, str): 

17 return text 

18 if not isinstance(text, (bytes, str)): 

19 raise TypeError( 

20 f"to_unicode must receive bytes or str, got {type(text).__name__}" 

21 ) 

22 if encoding is None: 

23 encoding = "utf-8" 

24 return text.decode(encoding, errors) 

25 

26 

27# An HTML comment, up to where it ends or the text does. As in the HTML 

28# tokenizer, it ends at the first "-->" or "--!>" after its "<!--", or right 

29# away at "<!-->" or "<!--->". Dashes are consumed one at a time only where no 

30# closer starts, so there is a single way to match and nothing to backtrack 

31# into. 

32_COMMENT = r"<!--(?:-?>|[^-]*(?:-(?!-!?>)[^-]*)*(?:--!?>|$))" 

33 

34# The text of a tag after its name, up to where it ends or the text does, read 

35# attribute by attribute as the HTML tokenizer reads it. Unlike _TAG_BODY in 

36# w3lib.html, it consumes "<" as attribute text. Quoted values also consume 

37# ">", and an unfinished value runs to the end of input. 

38_TAG_SPACE = r"[\t\n\f\r ]" 

39_TAG_ATTRIBUTE = ( 

40 rf"[^\t\n\f\r />][^\t\n\f\r />=]*+{_TAG_SPACE}*+" 

41 rf"(?:={_TAG_SPACE}*+(?:\"[^\"]*(?:\"|$)|'[^']*(?:'|$)|[^\t\n\f\r >]*+))?+" 

42) 

43# Possessive repeats keep attribute names and unquoted values from being split 

44# or read again as the scanner advances through a long tag. 

45_TAG_REST = rf"(?:{_TAG_SPACE}|/|{_TAG_ATTRIBUTE})*+(?:>|$)" 

46# Any start or end tag, whole. A scanner that consumes the tags it does not 

47# look for does not read markup written inside them, e.g. the <base> in 

48# <a title="<base href=...>">, which is an attribute value to a browser. 

49_OTHER_TAG = rf"</?[a-zA-Z][^\t\n\f\r />]*+{_TAG_REST}" 

50 

51 

52_ASCII = bytes(range(128)) 

53_ASCII_TEXT = _ASCII.decode() 

54 

55 

56@lru_cache(maxsize=64) 

57def _ascii_compatible(encoding: str | None) -> bool: 

58 """Return whether ASCII bytes decode to the same ASCII characters under 

59 *encoding*. 

60 

61 Unknown encodings are reported as not compatible. 

62 """ 

63 try: 

64 return _ASCII.decode(encoding or "utf-8", "replace") == _ASCII_TEXT 

65 except LookupError: 

66 return False 

67 

68 

69# Bytes that an encoding can drop when decoding, dividing a character 

70# sequence without leaving a trace in the text: ESC ( B redesignates ASCII 

71# while ASCII is already in use, and SO and SI shift between character sets. 

72_DROPPED = (b"\x1b(B", b"\x0e", b"\x0f") 

73 

74 

75@lru_cache(maxsize=64) 

76def _scannable(encoding: str | None) -> bool: 

77 r"""Return whether every character sequence of a document written in 

78 *encoding* is a byte sequence of its undecoded bytes, so that markup that 

79 the bytes do not contain is markup that the document does not contain. 

80 

81 It takes ASCII compatibility, which makes ASCII characters their own 

82 bytes, and an encoding that cannot write those bytes apart, e.g. 

83 ``b"<ba\x1b(Bse"`` decodes to ``"<base"`` under ISO-2022-JP. 

84 """ 

85 if not _ascii_compatible(encoding): 

86 return False 

87 return all( 

88 (b"a" + dropped + b"b").decode(encoding or "utf-8", "replace") != "ab" 

89 for dropped in _DROPPED 

90 ) 

91 

92 

93# One attribute: a name and, optionally, a value, quoted or not. Each quoting 

94# style captures its own group, so a matched value needs no quote stripping. 

95_attr_re = re.compile( 

96 r"""(?P<name>[^\s<>/=]+) # name 

97 (?:\s*=\s*(?:"(?P<double>[^"]*)"|'(?P<single>[^']*)'|(?P<bare>[^\s"'>]*)))?""", 

98 re.VERBOSE, 

99) 

100 

101 

102def iter_tag_attributes(attrs: str) -> Iterable[tuple[str, str]]: 

103 """Yield ``(name, value)`` for every attribute with a value in ``attrs``, 

104 the text of one tag after its name, in order. 

105 

106 ``name`` is lowercased, and ``value`` has its quotes removed, if any. 

107 """ 

108 # finditer() matches one attribute at a time, from where the previous one 

109 # ended, so a crafted tag cannot make it backtrack across attributes. 

110 for attr in _attr_re.finditer(attrs): 

111 # A valueless attribute matches the name group and nothing after it. 

112 if attr.lastindex == 1: 

113 continue 

114 name, double, single, bare = attr.groups() 

115 # Exactly one value group matched; the last "" covers all three being 

116 # empty, which an empty value in any quoting style produces. 

117 yield name.lower(), double or single or bare or ""