Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/_tokenizer.py: 41%

1import contextlib 

2import re 

3from dataclasses import dataclass 

4from typing import Dict, Iterator, NoReturn, Optional, Tuple, Union 

5 

6from .specifiers import Specifier 

7 

8 

9@dataclass 

10class Token: 

11 name: str 

12 text: str 

13 position: int 

14 

15 

16class ParserSyntaxError(Exception): 

17 """The provided source text could not be parsed correctly.""" 

18 

19 def __init__( 

20 self, 

21 message: str, 

22 *, 

23 source: str, 

24 span: Tuple[int, int], 

25 ) -> None: 

26 self.span = span 

27 self.message = message 

28 self.source = source 

29 

30 super().__init__() 

31 

32 def __str__(self) -> str: 

33 marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" 

34 return "\n ".join([self.message, self.source, marker]) 

35 

36 

37DEFAULT_RULES: "Dict[str, Union[str, re.Pattern[str]]]" = { 

38 "LEFT_PARENTHESIS": r"\(", 

39 "RIGHT_PARENTHESIS": r"\)", 

40 "LEFT_BRACKET": r"\[", 

41 "RIGHT_BRACKET": r"\]", 

42 "SEMICOLON": r";", 

43 "COMMA": r",", 

44 "QUOTED_STRING": re.compile( 

45 r""" 

46 ( 

47 ('[^']*') 

48 | 

49 ("[^"]*") 

50 ) 

51 """, 

52 re.VERBOSE, 

53 ), 

54 "OP": r"(===|==|~=|!=|<=|>=|<|>)", 

55 "BOOLOP": r"\b(or|and)\b", 

56 "IN": r"\bin\b", 

57 "NOT": r"\bnot\b", 

58 "VARIABLE": re.compile( 

59 r""" 

60 \b( 

61 python_version 

62 |python_full_version 

63 |os[._]name 

64 |sys[._]platform 

65 |platform_(release|system) 

66 |platform[._](version|machine|python_implementation) 

67 |python_implementation 

68 |implementation_(name|version) 

69 |extra 

70 )\b 

71 """, 

72 re.VERBOSE, 

73 ), 

74 "SPECIFIER": re.compile( 

75 Specifier._operator_regex_str + Specifier._version_regex_str, 

76 re.VERBOSE | re.IGNORECASE, 

77 ), 

78 "AT": r"\@", 

79 "URL": r"[^ \t]+", 

80 "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", 

81 "WS": r"[ \t]+", 

82 "END": r"$", 

83} 

84 

85 

86class Tokenizer: 

87 """Context-sensitive token parsing. 

88 

89 Provides methods to examine the input stream to check whether the next token 

90 matches. 

91 """ 

92 

93 def __init__( 

94 self, 

95 source: str, 

96 *, 

97 rules: "Dict[str, Union[str, re.Pattern[str]]]", 

98 ) -> None: 

99 self.source = source 

100 self.rules: Dict[str, re.Pattern[str]] = { 

101 name: re.compile(pattern) for name, pattern in rules.items() 

102 } 

103 self.next_token: Optional[Token] = None 

104 self.position = 0 

105 

106 def consume(self, name: str) -> None: 

107 """Move beyond provided token name, if at current position.""" 

108 if self.check(name): 

109 self.read() 

110 

111 def check(self, name: str, *, peek: bool = False) -> bool: 

112 """Check whether the next token has the provided name. 

113 

114 By default, if the check succeeds, the token *must* be read before 

115 another check. If `peek` is set to `True`, the token is not loaded and 

116 would need to be checked again. 

117 """ 

118 assert ( 

119 self.next_token is None 

120 ), f"Cannot check for {name!r}, already have {self.next_token!r}" 

121 assert name in self.rules, f"Unknown token name: {name!r}" 

122 

123 expression = self.rules[name] 

124 

125 match = expression.match(self.source, self.position) 

126 if match is None: 

127 return False 

128 if not peek: 

129 self.next_token = Token(name, match[0], self.position) 

130 return True 

131 

132 def expect(self, name: str, *, expected: str) -> Token: 

133 """Expect a certain token name next, failing with a syntax error otherwise. 

134 

135 The token is *not* read. 

136 """ 

137 if not self.check(name): 

138 raise self.raise_syntax_error(f"Expected {expected}") 

139 return self.read() 

140 

141 def read(self) -> Token: 

142 """Consume the next token and return it.""" 

143 token = self.next_token 

144 assert token is not None 

145 

146 self.position += len(token.text) 

147 self.next_token = None 

148 

149 return token 

150 

151 def raise_syntax_error( 

152 self, 

153 message: str, 

154 *, 

155 span_start: Optional[int] = None, 

156 span_end: Optional[int] = None, 

157 ) -> NoReturn: 

158 """Raise ParserSyntaxError at the given position.""" 

159 span = ( 

160 self.position if span_start is None else span_start, 

161 self.position if span_end is None else span_end, 

162 ) 

163 raise ParserSyntaxError( 

164 message, 

165 source=self.source, 

166 span=span, 

167 ) 

168 

169 @contextlib.contextmanager 

170 def enclosing_tokens(self, open_token: str, close_token: str) -> Iterator[bool]: 

171 if self.check(open_token): 

172 open_position = self.position 

173 self.read() 

174 else: 

175 open_position = None 

176 

177 yield open_position is not None 

178 

179 if open_position is None: 

180 return 

181 

182 if not self.check(close_token): 

183 self.raise_syntax_error( 

184 f"Expected closing {close_token}", 

185 span_start=open_position, 

186 ) 

187 

188 self.read()