Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pip/_vendor/packaging/_tokenizer.py: 39%

1from __future__ import annotations 

2 

3import contextlib 

4import re 

5from dataclasses import dataclass 

6from typing import TYPE_CHECKING, NoReturn 

7 

8from .specifiers import Specifier 

9 

10if TYPE_CHECKING: 

11 from collections.abc import Generator, Mapping 

12 

13 

14@dataclass 

15class Token: 

16 __slots__ = ("name", "position", "text") 

17 

18 name: str 

19 text: str 

20 position: int 

21 

22 

23class ParserSyntaxError(Exception): 

24 """The provided source text could not be parsed correctly.""" 

25 

26 def __init__( 

27 self, 

28 message: str, 

29 *, 

30 source: str, 

31 span: tuple[int, int], 

32 ) -> None: 

33 self.span = span 

34 self.message = message 

35 self.source = source 

36 

37 super().__init__() 

38 

39 def __str__(self) -> str: 

40 marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" 

41 return f"{self.message}\n {self.source}\n {marker}" 

42 

43 

44DEFAULT_RULES: dict[str, re.Pattern[str]] = { 

45 "LEFT_PARENTHESIS": re.compile(r"\("), 

46 "RIGHT_PARENTHESIS": re.compile(r"\)"), 

47 "LEFT_BRACKET": re.compile(r"\["), 

48 "RIGHT_BRACKET": re.compile(r"\]"), 

49 "SEMICOLON": re.compile(r";"), 

50 "COMMA": re.compile(r","), 

51 "QUOTED_STRING": re.compile( 

52 r""" 

53 ( 

54 ('[^']*') 

55 | 

56 ("[^"]*") 

57 ) 

58 """, 

59 re.VERBOSE, 

60 ), 

61 "OP": re.compile(r"(===|==|~=|!=|<=|>=|<|>)"), 

62 "BOOLOP": re.compile(r"\b(or|and)\b"), 

63 "IN": re.compile(r"\bin\b"), 

64 "NOT": re.compile(r"\bnot\b"), 

65 "VARIABLE": re.compile( 

66 r""" 

67 \b( 

68 python_version 

69 |python_full_version 

70 |os[._]name 

71 |sys[._]platform 

72 |platform_(release|system) 

73 |platform[._](version|machine|python_implementation) 

74 |python_implementation 

75 |implementation_(name|version) 

76 |extras? 

77 |dependency_groups 

78 )\b 

79 """, 

80 re.VERBOSE, 

81 ), 

82 "SPECIFIER": re.compile( 

83 Specifier._specifier_regex_str, 

84 re.VERBOSE | re.IGNORECASE, 

85 ), 

86 "AT": re.compile(r"\@"), 

87 "URL": re.compile(r"[^ \t]+"), 

88 "IDENTIFIER": re.compile(r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b"), 

89 "VERSION_PREFIX_TRAIL": re.compile(r"\.\*"), 

90 "VERSION_LOCAL_LABEL_TRAIL": re.compile(r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*"), 

91 "WS": re.compile(r"[ \t]+"), 

92 "END": re.compile(r"\Z"), 

93} 

94 

95 

96class Tokenizer: 

97 """Context-sensitive token parsing. 

98 

99 Provides methods to examine the input stream to check whether the next token 

100 matches. 

101 """ 

102 

103 __slots__ = ("next_token", "position", "rules", "source") 

104 

105 def __init__( 

106 self, 

107 source: str, 

108 *, 

109 rules: Mapping[str, re.Pattern[str]], 

110 ) -> None: 

111 self.source = source 

112 self.rules = rules 

113 self.next_token: Token | None = None 

114 self.position = 0 

115 

116 def consume(self, name: str) -> None: 

117 """Move beyond provided token name, if at current position.""" 

118 if self.check(name): 

119 self.read() 

120 

121 def check(self, name: str, *, peek: bool = False) -> bool: 

122 """Check whether the next token has the provided name. 

123 

124 By default, if the check succeeds, the token *must* be read before 

125 another check. If `peek` is set to `True`, the token is not loaded and 

126 would need to be checked again. 

127 """ 

128 assert self.next_token is None, ( 

129 f"Cannot check for {name!r}, already have {self.next_token!r}" 

130 ) 

131 assert name in self.rules, f"Unknown token name: {name!r}" 

132 

133 expression = self.rules[name] 

134 

135 match = expression.match(self.source, self.position) 

136 if match is None: 

137 return False 

138 if not peek: 

139 self.next_token = Token(name, match[0], self.position) 

140 return True 

141 

142 def expect(self, name: str, *, expected: str) -> Token: 

143 """Expect a certain token name next, failing with a syntax error otherwise. 

144 

145 The token is read and returned. 

146 """ 

147 if not self.check(name): 

148 raise self.raise_syntax_error(f"Expected {expected}") 

149 return self.read() 

150 

151 def read(self) -> Token: 

152 """Consume the next token and return it.""" 

153 token = self.next_token 

154 assert token is not None 

155 

156 self.position += len(token.text) 

157 self.next_token = None 

158 

159 return token 

160 

161 def raise_syntax_error( 

162 self, 

163 message: str, 

164 *, 

165 span_start: int | None = None, 

166 span_end: int | None = None, 

167 ) -> NoReturn: 

168 """Raise ParserSyntaxError at the given position.""" 

169 span = ( 

170 self.position if span_start is None else span_start, 

171 self.position if span_end is None else span_end, 

172 ) 

173 raise ParserSyntaxError( 

174 message, 

175 source=self.source, 

176 span=span, 

177 ) 

178 

179 @contextlib.contextmanager 

180 def enclosing_tokens( 

181 self, open_token: str, close_token: str, *, around: str 

182 ) -> Generator[None, None, None]: 

183 if self.check(open_token): 

184 open_position = self.position 

185 self.read() 

186 else: 

187 open_position = None 

188 

189 yield 

190 

191 if open_position is None: 

192 return 

193 

194 if not self.check(close_token): 

195 self.raise_syntax_error( 

196 f"Expected matching {close_token} for {open_token}, after {around}", 

197 span_start=open_position, 

198 ) 

199 

200 self.read()