Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/mako/lexer.py: 89%

1# mako/lexer.py 

2# Copyright 2006-2026 the Mako authors and contributors <see AUTHORS file> 

3# 

4# This module is part of Mako and is released under 

5# the MIT License: http://www.opensource.org/licenses/mit-license.php 

6 

7"""provides the Lexer class for parsing template strings into parse trees.""" 

8 

9import codecs 

10import re 

11 

12from mako import exceptions 

13from mako import parsetree 

14from mako.pygen import adjust_whitespace 

15 

16_regexp_cache = {} 

17 

18 

19class Lexer: 

20 def __init__( 

21 self, text, filename=None, input_encoding=None, preprocessor=None 

22 ): 

23 self.text = text 

24 self.filename = filename 

25 self.template = parsetree.TemplateNode(self.filename) 

26 self.matched_lineno = 1 

27 self.matched_charpos = 0 

28 self.lineno = 1 

29 self.match_position = 0 

30 self.tag = [] 

31 self.control_line = [] 

32 self.ternary_stack = [] 

33 self.encoding = input_encoding 

34 

35 if preprocessor is None: 

36 self.preprocessor = [] 

37 elif not hasattr(preprocessor, "__iter__"): 

38 self.preprocessor = [preprocessor] 

39 else: 

40 self.preprocessor = preprocessor 

41 

42 @property 

43 def exception_kwargs(self): 

44 return { 

45 "source": self.text, 

46 "lineno": self.matched_lineno, 

47 "pos": self.matched_charpos, 

48 "filename": self.filename, 

49 } 

50 

51 def match(self, regexp, flags=None): 

52 """compile the given regexp, cache the reg, and call match_reg().""" 

53 

54 try: 

55 reg = _regexp_cache[(regexp, flags)] 

56 except KeyError: 

57 reg = re.compile(regexp, flags) if flags else re.compile(regexp) 

58 _regexp_cache[(regexp, flags)] = reg 

59 

60 return self.match_reg(reg) 

61 

62 def match_reg(self, reg): 

63 """match the given regular expression object to the current text 

64 position. 

65 

66 if a match occurs, update the current text and line position. 

67 

68 """ 

69 

70 mp = self.match_position 

71 

72 match = reg.match(self.text, self.match_position) 

73 if match: 

74 start, end = match.span() 

75 self.match_position = end + 1 if end == start else end 

76 self.matched_lineno = self.lineno 

77 cp = mp - 1 

78 if cp >= 0 and cp < self.textlength: 

79 cp = self.text[: cp + 1].rfind("\n") 

80 self.matched_charpos = mp - cp 

81 self.lineno += self.text[mp : self.match_position].count("\n") 

82 return match 

83 

84 def parse_until_text(self, watch_nesting, *text): 

85 startpos = self.match_position 

86 startlineno = self.matched_lineno 

87 startcharpos = self.matched_charpos 

88 text_re = r"|".join(text) 

89 brace_level = 0 

90 paren_level = 0 

91 bracket_level = 0 

92 while True: 

93 match = self.match(r"#.*\n") 

94 if match: 

95 continue 

96 match = self.match( 

97 r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S 

98 ) 

99 if match: 

100 continue 

101 match = self.match(r"(%s)" % text_re) 

102 if match and not ( 

103 watch_nesting 

104 and (brace_level > 0 or paren_level > 0 or bracket_level > 0) 

105 ): 

106 return ( 

107 self.text[ 

108 startpos : self.match_position - len(match.group(1)) 

109 ], 

110 match.group(1), 

111 ) 

112 elif not match: 

113 match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) 

114 if match: 

115 brace_level += match.group(1).count("{") 

116 brace_level -= match.group(1).count("}") 

117 paren_level += match.group(1).count("(") 

118 paren_level -= match.group(1).count(")") 

119 bracket_level += match.group(1).count("[") 

120 bracket_level -= match.group(1).count("]") 

121 continue 

122 # the scan consumes the remaining text looking for the 

123 # closing token, so report the position the construct began 

124 # at, rather than the position the scan gave up at 

125 raise exceptions.SyntaxException( 

126 "Expected: %s; unterminated tag or expression beginning" 

127 % ",".join(text), 

128 **{ 

129 **self.exception_kwargs, 

130 "lineno": startlineno, 

131 "pos": startcharpos, 

132 }, 

133 ) 

134 

135 def append_node(self, nodecls, *args, **kwargs): 

136 kwargs.setdefault("source", self.text) 

137 kwargs.setdefault("lineno", self.matched_lineno) 

138 kwargs.setdefault("pos", self.matched_charpos) 

139 kwargs["filename"] = self.filename 

140 node = nodecls(*args, **kwargs) 

141 if len(self.tag): 

142 self.tag[-1].nodes.append(node) 

143 else: 

144 self.template.nodes.append(node) 

145 # build a set of child nodes for the control line 

146 # (used for loop variable detection) 

147 # also build a set of child nodes on ternary control lines 

148 # (used for determining if a pass needs to be auto-inserted 

149 if self.control_line: 

150 control_frame = self.control_line[-1] 

151 control_frame.nodes.append(node) 

152 if ( 

153 not ( 

154 isinstance(node, parsetree.ControlLine) 

155 and control_frame.is_ternary(node.keyword) 

156 ) 

157 and self.ternary_stack 

158 and self.ternary_stack[-1] 

159 ): 

160 self.ternary_stack[-1][-1].nodes.append(node) 

161 if isinstance(node, parsetree.Tag): 

162 if len(self.tag): 

163 node.parent = self.tag[-1] 

164 self.tag.append(node) 

165 elif isinstance(node, parsetree.ControlLine): 

166 if node.isend: 

167 self.control_line.pop() 

168 self.ternary_stack.pop() 

169 elif node.is_primary: 

170 self.control_line.append(node) 

171 self.ternary_stack.append([]) 

172 elif self.control_line and self.control_line[-1].is_ternary( 

173 node.keyword 

174 ): 

175 self.ternary_stack[-1].append(node) 

176 elif self.control_line and not self.control_line[-1].is_ternary( 

177 node.keyword 

178 ): 

179 raise exceptions.SyntaxException( 

180 "Keyword '%s' not a legal ternary for keyword '%s'" 

181 % (node.keyword, self.control_line[-1].keyword), 

182 **self.exception_kwargs, 

183 ) 

184 

185 _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n") 

186 

187 def decode_raw_stream(self, text, decode_raw, known_encoding, filename): 

188 """given string/unicode or bytes/string, determine encoding 

189 from magic encoding comment, return body as unicode 

190 or raw if decode_raw=False 

191 

192 """ 

193 if isinstance(text, str): 

194 m = self._coding_re.match(text) 

195 encoding = m and m.group(1) or known_encoding or "utf-8" 

196 return encoding, text 

197 

198 if text.startswith(codecs.BOM_UTF8): 

199 text = text[len(codecs.BOM_UTF8) :] 

200 parsed_encoding = "utf-8" 

201 m = self._coding_re.match(text.decode("utf-8", "ignore")) 

202 if m is not None and m.group(1) != "utf-8": 

203 raise exceptions.CompileException( 

204 "Found utf-8 BOM in file, with conflicting " 

205 "magic encoding comment of '%s'" % m.group(1), 

206 text.decode("utf-8", "ignore"), 

207 0, 

208 0, 

209 filename, 

210 ) 

211 else: 

212 m = self._coding_re.match(text.decode("utf-8", "ignore")) 

213 parsed_encoding = m.group(1) if m else known_encoding or "utf-8" 

214 if decode_raw: 

215 try: 

216 text = text.decode(parsed_encoding) 

217 except UnicodeDecodeError: 

218 raise exceptions.CompileException( 

219 "Unicode decode operation of encoding '%s' failed" 

220 % parsed_encoding, 

221 text.decode("utf-8", "ignore"), 

222 0, 

223 0, 

224 filename, 

225 ) 

226 

227 return parsed_encoding, text 

228 

229 def parse(self): 

230 self.encoding, self.text = self.decode_raw_stream( 

231 self.text, True, self.encoding, self.filename 

232 ) 

233 

234 for preproc in self.preprocessor: 

235 self.text = preproc(self.text) 

236 

237 # push the match marker past the 

238 # encoding comment. 

239 self.match_reg(self._coding_re) 

240 

241 self.textlength = len(self.text) 

242 

243 while True: 

244 if self.match_position > self.textlength: 

245 break 

246 

247 if self.match_end(): 

248 break 

249 if self.match_expression(): 

250 continue 

251 if self.match_control_line(): 

252 continue 

253 if self.match_comment(): 

254 continue 

255 if self.match_tag_start(): 

256 continue 

257 if self.match_tag_end(): 

258 continue 

259 if self.match_python_block(): 

260 continue 

261 if self.match_percent(): 

262 continue 

263 if self.match_text(): 

264 continue 

265 

266 if self.match_position > self.textlength: 

267 break 

268 # TODO: no coverage here 

269 raise exceptions.MakoException("assertion failed") 

270 

271 if len(self.tag): 

272 raise exceptions.SyntaxException( 

273 "Unclosed tag: <%%%s>" % self.tag[-1].keyword, 

274 **self.exception_kwargs, 

275 ) 

276 if len(self.control_line): 

277 raise exceptions.SyntaxException( 

278 "Unterminated control keyword: '%s'" 

279 % self.control_line[-1].keyword, 

280 self.text, 

281 self.control_line[-1].lineno, 

282 self.control_line[-1].pos, 

283 self.filename, 

284 ) 

285 return self.template 

286 

287 def match_tag_start(self): 

288 reg = r""" 

289 \<% # opening tag 

290 

291 ([\w\.\:]+) # keyword 

292 

293 ((?:\s+\w+|\s*=\s*|"[^"]*?"|'[^']*?'|\s*,\s*)*) # attrname, = \ 

294 # sign, string expression 

295 # comma is for backwards compat 

296 # identified in #366 

297 

298 \s* # more whitespace 

299 

300 (/)?> # closing 

301 

302 """ 

303 

304 match = self.match( 

305 reg, 

306 re.I | re.S | re.X, 

307 ) 

308 

309 if not match: 

310 return False 

311 

312 keyword, attr, isend = match.groups() 

313 self.keyword = keyword 

314 attributes = {} 

315 if attr: 

316 for att in re.findall( 

317 r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr 

318 ): 

319 key, val1, val2 = att 

320 text = val1 or val2 

321 text = text.replace("\r\n", "\n") 

322 attributes[key] = text 

323 self.append_node(parsetree.Tag, keyword, attributes) 

324 if isend: 

325 self.tag.pop() 

326 elif keyword == "text": 

327 match = self.match(r"(.*?)(?=\</%text>)", re.S) 

328 if not match: 

329 raise exceptions.SyntaxException( 

330 "Unclosed tag: <%%%s>" % self.tag[-1].keyword, 

331 **self.exception_kwargs, 

332 ) 

333 self.append_node(parsetree.Text, match.group(1)) 

334 return self.match_tag_end() 

335 return True 

336 

337 def match_tag_end(self): 

338 match = self.match(r"\</%[\t ]*([^\t ]+?)[\t ]*>") 

339 if match: 

340 if not len(self.tag): 

341 raise exceptions.SyntaxException( 

342 "Closing tag without opening tag: </%%%s>" 

343 % match.group(1), 

344 **self.exception_kwargs, 

345 ) 

346 elif self.tag[-1].keyword != match.group(1): 

347 raise exceptions.SyntaxException( 

348 "Closing tag </%%%s> does not match tag: <%%%s>" 

349 % (match.group(1), self.tag[-1].keyword), 

350 **self.exception_kwargs, 

351 ) 

352 self.tag.pop() 

353 return True 

354 else: 

355 return False 

356 

357 def match_end(self): 

358 match = self.match(r"\Z", re.S) 

359 if not match: 

360 return False 

361 

362 string = match.group() 

363 if string: 

364 return string 

365 else: 

366 return True 

367 

368 def match_percent(self): 

369 match = self.match(r"(?<=^)(\s*)%%(%*)", re.M) 

370 if match: 

371 self.append_node( 

372 parsetree.Text, match.group(1) + "%" + match.group(2) 

373 ) 

374 return True 

375 else: 

376 return False 

377 

378 def match_text(self): 

379 match = self.match( 

380 r""" 

381 (.*?) # anything, followed by: 

382 ( 

383 (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based 

384 # comment, preceded by a 

385 # consumed newline and whitespace 

386 | 

387 (?=\${) # an expression 

388 | 

389 (?=</?%) # a substitution or block or call start or end 

390 # - don't consume 

391 | 

392 (\\\r?\n) # an escaped newline - throw away 

393 | 

394 \Z # end of string 

395 )""", 

396 re.X | re.S, 

397 ) 

398 

399 if match: 

400 text = match.group(1) 

401 if text: 

402 self.append_node(parsetree.Text, text) 

403 return True 

404 else: 

405 return False 

406 

407 def match_python_block(self): 

408 match = self.match(r"<%(!)?") 

409 if match: 

410 line, pos = self.matched_lineno, self.matched_charpos 

411 text, end = self.parse_until_text(False, r"%>") 

412 # the trailing newline helps 

413 # compiler.parse() not complain about indentation 

414 text = adjust_whitespace(text) + "\n" 

415 self.append_node( 

416 parsetree.Code, 

417 text, 

418 match.group(1) == "!", 

419 lineno=line, 

420 pos=pos, 

421 ) 

422 return True 

423 else: 

424 return False 

425 

426 def match_expression(self): 

427 match = self.match(r"\${") 

428 if not match: 

429 return False 

430 

431 line, pos = self.matched_lineno, self.matched_charpos 

432 text, end = self.parse_until_text(True, r"\|", r"}") 

433 if end == "|": 

434 escapes, end = self.parse_until_text(True, r"}") 

435 else: 

436 escapes = "" 

437 text = text.replace("\r\n", "\n") 

438 self.append_node( 

439 parsetree.Expression, 

440 text, 

441 escapes.strip(), 

442 lineno=line, 

443 pos=pos, 

444 ) 

445 return True 

446 

447 def match_control_line(self): 

448 match = self.match( 

449 r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\\r?\n)|[^\r\n])*)" 

450 r"(?:\r?\n|\Z)", 

451 re.M, 

452 ) 

453 if not match: 

454 return False 

455 

456 operator = match.group(1) 

457 text = match.group(2) 

458 if operator == "%": 

459 m2 = re.match(r"(end)?(\w+)\s*(.*)", text) 

460 if not m2: 

461 raise exceptions.SyntaxException( 

462 "Invalid control line: '%s'" % text, 

463 **self.exception_kwargs, 

464 ) 

465 isend, keyword = m2.group(1, 2) 

466 isend = isend is not None 

467 

468 if isend: 

469 if not len(self.control_line): 

470 raise exceptions.SyntaxException( 

471 "No starting keyword '%s' for '%s'" % (keyword, text), 

472 **self.exception_kwargs, 

473 ) 

474 elif self.control_line[-1].keyword != keyword: 

475 raise exceptions.SyntaxException( 

476 "Keyword '%s' doesn't match keyword '%s'" 

477 % (text, self.control_line[-1].keyword), 

478 **self.exception_kwargs, 

479 ) 

480 self.append_node(parsetree.ControlLine, keyword, isend, text) 

481 else: 

482 self.append_node(parsetree.Comment, text) 

483 return True 

484 

485 def match_comment(self): 

486 """matches the multiline version of a comment""" 

487 match = self.match(r"<%doc>(.*?)</%doc>", re.S) 

488 if match: 

489 self.append_node(parsetree.Comment, match.group(1)) 

490 return True 

491 else: 

492 return False