Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/babel/messages/jslexer.py: 54%

1""" 

2babel.messages.jslexer 

3~~~~~~~~~~~~~~~~~~~~~~ 

4 

5A simple JavaScript 1.5 lexer which is used for the JavaScript 

6extractor. 

7 

8:copyright: (c) 2013-2026 by the Babel Team. 

9:license: BSD, see LICENSE for more details. 

10""" 

11 

12from __future__ import annotations 

13 

14import re 

15from collections.abc import Generator 

16from typing import NamedTuple 

17 

18operators: list[str] = sorted([ 

19 '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=', 

20 '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=', 

21 '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')', 

22 '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':', 

23], key=len, reverse=True) # fmt: skip 

24 

25escapes: dict[str, str] = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v'} 

26 

27name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE) 

28dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE) 

29division_re = re.compile(r'/=?') 

30regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL) 

31line_re = re.compile(r'(\r\n|\n|\r)') 

32line_join_re = re.compile(r'\\' + line_re.pattern) 

33uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') 

34codepoint_escape_re = re.compile(r'\{([a-fA-F0-9]+)\}') 

35hex_escape_re = re.compile(r'[a-fA-F0-9]{1,2}') 

36 

37 

38class Token(NamedTuple): 

39 type: str 

40 value: str 

41 lineno: int 

42 

43 

44_rules: list[tuple[str | None, re.Pattern[str]]] = [ 

45 (None, re.compile(r'\s+', re.UNICODE)), 

46 (None, re.compile(r'<!--[^\r\n]*')), 

47 ('linecomment', re.compile(r'//[^\r\n]*')), 

48 ('multilinecomment', re.compile(r'/\*.*?\*/', re.UNICODE | re.DOTALL)), 

49 ('dotted_name', dotted_name_re), 

50 ('name', name_re), 

51 ('number', re.compile(r'''( 

52 (?:0|[1-9]\d*) 

53 (\.\d+)? 

54 ([eE][-+]?\d+)? | 

55 (0x[a-fA-F0-9]+) 

56 )''', re.VERBOSE)), 

57 ('jsx_tag', re.compile(r'(?:</?[^>\s]+|/>)', re.I)), # May be mangled in `get_rules` 

58 ('operator', re.compile(r'(%s)' % '|'.join(re.escape(op) for op in operators))), 

59 ('template_string', re.compile(r'''`(?:[^`\\]*(?:\\.[^`\\]*)*)`''', re.UNICODE)), 

60 ('string', re.compile(r'''( 

61 '(?:[^'\\]*(?:\\.[^'\\]*)*)' | 

62 "(?:[^"\\]*(?:\\.[^"\\]*)*)" 

63 )''', re.VERBOSE | re.DOTALL)), 

64] # fmt: skip 

65 

66 

67def get_rules( 

68 jsx: bool, 

69 dotted: bool, 

70 template_string: bool, 

71) -> list[tuple[str | None, re.Pattern[str]]]: 

72 """ 

73 Get a tokenization rule list given the passed syntax options. 

74 

75 Internal to this module. 

76 """ 

77 rules = [] 

78 for token_type, rule in _rules: 

79 if not jsx and token_type and 'jsx' in token_type: 

80 continue 

81 if not template_string and token_type == 'template_string': 

82 continue 

83 if token_type == 'dotted_name': 

84 if not dotted: 

85 continue 

86 token_type = 'name' 

87 rules.append((token_type, rule)) 

88 return rules 

89 

90 

91def indicates_division(token: Token) -> bool: 

92 """A helper function that helps the tokenizer to decide if the current 

93 token may be followed by a division operator. 

94 """ 

95 if token.type == 'operator': 

96 return token.value in (')', ']', '}', '++', '--') 

97 return token.type in ('name', 'number', 'string', 'regexp') 

98 

99 

100def unquote_string(string: str) -> str: 

101 """Unquote a string with JavaScript rules. The string has to start with 

102 string delimiters (``'``, ``"`` or the back-tick/grave accent (for template strings).) 

103 """ 

104 assert string and string[0] == string[-1] and string[0] in '"\'`', ( 

105 'string provided is not properly delimited' 

106 ) 

107 string = line_join_re.sub('\\1', string[1:-1]) 

108 result: list[str] = [] 

109 add = result.append 

110 pos = 0 

111 

112 while True: 

113 # scan for the next escape 

114 escape_pos = string.find('\\', pos) 

115 if escape_pos < 0: 

116 break 

117 add(string[pos:escape_pos]) 

118 

119 # check which character is escaped 

120 next_char = string[escape_pos + 1] 

121 if next_char in escapes: 

122 add(escapes[next_char]) 

123 

124 # unicode escapes. trie to consume up to four characters of 

125 # hexadecimal characters and try to interpret them as unicode 

126 # character point. If there is no such character point, put 

127 # all the consumed characters into the string. 

128 elif next_char in 'uU': 

129 if next_char == 'u': 

130 codepoint = codepoint_escape_re.match(string, escape_pos + 2) 

131 if codepoint is not None: 

132 value = int(codepoint.group(1), 16) 

133 if value <= 0x10ffff: 

134 add(chr(value)) 

135 pos = codepoint.end() 

136 continue 

137 escaped = uni_escape_re.match(string, escape_pos + 2) 

138 if escaped is not None: 

139 escaped_value = escaped.group() 

140 if len(escaped_value) == 4: 

141 try: 

142 add(chr(int(escaped_value, 16))) 

143 except ValueError: 

144 pass 

145 else: 

146 pos = escape_pos + 6 

147 continue 

148 add(next_char + escaped_value) 

149 pos = escaped.end() 

150 continue 

151 else: 

152 add(next_char) 

153 

154 # hex escapes. conversion from 2-digits hex to char is infallible 

155 elif next_char in 'xX': 

156 escaped = hex_escape_re.match(string, escape_pos + 2) 

157 if escaped is not None: 

158 escaped_value = escaped.group() 

159 add(chr(int(escaped_value, 16))) 

160 pos = escape_pos + 2 + len(escaped_value) 

161 continue 

162 else: 

163 add(next_char) 

164 

165 # bogus escape. Just remove the backslash. 

166 else: 

167 add(next_char) 

168 pos = escape_pos + 2 

169 

170 if pos < len(string): 

171 add(string[pos:]) 

172 

173 return ''.join(result) 

174 

175 

176def tokenize( 

177 source: str, 

178 jsx: bool = True, 

179 dotted: bool = True, 

180 template_string: bool = True, 

181 lineno: int = 1, 

182) -> Generator[Token, None, None]: 

183 """ 

184 Tokenize JavaScript/JSX source. Returns a generator of tokens. 

185 

186 :param source: The JavaScript source to tokenize. 

187 :param jsx: Enable (limited) JSX parsing. 

188 :param dotted: Read dotted names as single name token. 

189 :param template_string: Support ES6 template strings 

190 :param lineno: starting line number (optional) 

191 """ 

192 may_divide = False 

193 pos = 0 

194 end = len(source) 

195 rules = get_rules(jsx=jsx, dotted=dotted, template_string=template_string) 

196 

197 while pos < end: 

198 # handle regular rules first 

199 for token_type, rule in rules: # noqa: B007 

200 match = rule.match(source, pos) 

201 if match is not None: 

202 break 

203 # if we don't have a match we don't give up yet, but check for 

204 # division operators or regular expression literals, based on 

205 # the status of `may_divide` which is determined by the last 

206 # processed non-whitespace token using `indicates_division`. 

207 else: 

208 if may_divide: 

209 match = division_re.match(source, pos) 

210 token_type = 'operator' 

211 else: 

212 match = regex_re.match(source, pos) 

213 token_type = 'regexp' 

214 if match is None: 

215 # woops. invalid syntax. jump one char ahead and try again. 

216 pos += 1 

217 continue 

218 

219 token_value = match.group() 

220 if token_type is not None: 

221 token = Token(token_type, token_value, lineno) 

222 may_divide = indicates_division(token) 

223 yield token 

224 lineno += len(line_re.findall(token_value)) 

225 pos = match.end()