Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/mistune/block_parser.py: 99%

1from bisect import bisect_left 

2import re 

3from typing import Optional, List, Tuple, Match, Pattern, Set 

4import string 

5from .util import ( 

6 unikey, 

7 escape_url, 

8 expand_tab, 

9 expand_leading_tab, 

10) 

11from .core import Parser, BlockState 

12from .helpers import ( 

13 LINK_LABEL, 

14 HTML_TAGNAME, 

15 HTML_ATTRIBUTES, 

16 BLOCK_TAGS, 

17 PRE_TAGS, 

18 unescape_char, 

19 parse_link_href, 

20 parse_link_title, 

21) 

22from .list_parser import parse_list, LIST_PATTERN 

23 

24DEFAULT_MAX_NESTED_LEVEL = 20 

25 

26_INDENT_CODE_TRIM = re.compile(r"^ {1,4}", flags=re.M) 

27_ATX_HEADING_TRIM = re.compile(r"(\s+|^)#+\s*$") 

28_BLOCK_QUOTE_TRIM = re.compile(r"^ ?", flags=re.M) 

29 

30_BLANK_TO_LINE = re.compile(r"[ \t]*\n") 

31 

32_BLOCK_TAGS_PATTERN = "(" + "|".join(BLOCK_TAGS) + "|" + "|".join(PRE_TAGS) + ")" 

33_OPEN_TAG_END = re.compile(HTML_ATTRIBUTES + r"[ \t]*>[ \t]*(?:\n|$)") 

34_CLOSE_TAG_END = re.compile(r"[ \t]*>[ \t]*(?:\n|$)") 

35_BLOCK_QUOTE_LINE = re.compile(r"^ {0,3}>([^\n]*(?:\n|$))") 

36 

37 

38class BlockParser(Parser[BlockState]): 

39 state_cls = BlockState 

40 

41 BLANK_LINE = re.compile(r"(^[ \t\v\f]*\n)+", re.M) 

42 

43 RAW_HTML = ( 

44 r"^ {0,3}(" 

45 r"</?" + HTML_TAGNAME + r"|" 

46 r"<!--|" # comment 

47 r"<\?|" # script 

48 r"<![A-Z]|" 

49 r"<!\[CDATA\[)" 

50 ) 

51 

52 BLOCK_HTML = ( 

53 r"^ {0,3}(?:" 

54 r"(?:</?" + _BLOCK_TAGS_PATTERN + r"(?:[ \t]+|\n|$))" 

55 r"|<!--" # comment 

56 r"|<\?" # script 

57 r"|<![A-Z]" 

58 r"|<!\[CDATA\[)" 

59 ) 

60 

61 SPECIFICATION = { 

62 "blank_line": r"(^[ \t\v\f]*\n)+", 

63 "atx_heading": r"^ {0,3}(?P<atx_1>#{1,6})(?!#+)(?P<atx_2>[ \t]*|[ \t]+.*?)$", 

64 "setex_heading": r"^ {0,3}(?P<setext_1>=|-){1,}[ \t]*$", 

65 "fenced_code": ( 

66 r"^(?P<fenced_1> {0,3})(?P<fenced_2>`{3,}|~{3,})" 

67 r"[ \t]*(?P<fenced_3>.*?)$" 

68 ), 

69 "indent_code": ( 

70 r"^(?: {4}| *\t)[^\n]+(?:\n+|$)" 

71 r"((?:(?: {4}| *\t)[^\n]+(?:\n+|$))|\s)*" 

72 ), 

73 "thematic_break": r"^ {0,3}((?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})$", 

74 "ref_link": r"^ {0,3}\[(?P<reflink_1>" + LINK_LABEL + r")\]:", 

75 "block_quote": r"^ {0,3}>(?P<quote_1>.*?)$", 

76 "list": LIST_PATTERN, 

77 "block_html": BLOCK_HTML, 

78 "raw_html": RAW_HTML, 

79 } 

80 

81 DEFAULT_RULES = ( 

82 "fenced_code", 

83 "indent_code", 

84 "atx_heading", 

85 "setex_heading", 

86 "thematic_break", 

87 "block_quote", 

88 "list", 

89 "ref_link", 

90 "raw_html", 

91 "blank_line", 

92 ) 

93 

94 def __init__( 

95 self, 

96 block_quote_rules: Optional[List[str]] = None, 

97 list_rules: Optional[List[str]] = None, 

98 max_nested_level: int = DEFAULT_MAX_NESTED_LEVEL, 

99 ): 

100 super(BlockParser, self).__init__() 

101 

102 if block_quote_rules is None: 

103 block_quote_rules = list(self.DEFAULT_RULES) 

104 

105 if list_rules is None: 

106 list_rules = list(self.DEFAULT_RULES) 

107 

108 self.block_quote_rules = block_quote_rules 

109 self.list_rules = list_rules 

110 self.max_nested_level = max_nested_level 

111 # register default parse methods 

112 self._methods = {name: getattr(self, "parse_" + name) for name in self.SPECIFICATION} 

113 

114 def parse_blank_line(self, m: Match[str], state: BlockState) -> int: 

115 """Parse token for blank lines.""" 

116 state.append_token({"type": "blank_line"}) 

117 return m.end() 

118 

119 def parse_thematic_break(self, m: Match[str], state: BlockState) -> int: 

120 """Parse token for thematic break, e.g. ``<hr>`` tag in HTML.""" 

121 state.append_token({"type": "thematic_break"}) 

122 # $ does not count '\n' 

123 return m.end() + 1 

124 

125 def parse_indent_code(self, m: Match[str], state: BlockState) -> int: 

126 """Parse token for code block which is indented by 4 spaces.""" 

127 # it is a part of the paragraph 

128 end_pos = state.append_paragraph() 

129 if end_pos: 

130 return end_pos 

131 

132 code = m.group(0) 

133 end_pos = _trim_partial_next_line_indent(code, m.end()) 

134 if end_pos != m.end(): 

135 code = state.get_text(end_pos) 

136 code = expand_leading_tab(code) 

137 code = _INDENT_CODE_TRIM.sub("", code) 

138 code = code.strip("\n") 

139 state.append_token({"type": "block_code", "raw": code, "style": "indent"}) 

140 return end_pos 

141 

142 def parse_fenced_code(self, m: Match[str], state: BlockState) -> Optional[int]: 

143 """Parse token for fenced code block. A fenced code block is started with 

144 3 or more backtick(`) or tilde(~). 

145 

146 An example of a fenced code block: 

147 

148 .. code-block:: markdown 

149 

150 ```python 

151 def markdown(text): 

152 return mistune.html(text) 

153 ``` 

154 """ 

155 spaces = m.group("fenced_1") 

156 marker = m.group("fenced_2") 

157 info = m.group("fenced_3") 

158 

159 c = marker[0] 

160 if info and c == "`": 

161 # CommonMark Example 145 

162 # Info strings for backtick code blocks cannot contain backticks 

163 if info.find(c) != -1: 

164 return None 

165 

166 _end = re.compile(r"^ {0,3}" + c + "{" + str(len(marker)) + r",}[ \t]*(?:\n|$)", re.M) 

167 cursor_start = m.end() + 1 

168 

169 m2 = _end.search(state.src, cursor_start) 

170 if m2: 

171 code = state.src[cursor_start : m2.start()] 

172 end_pos = m2.end() 

173 else: 

174 code = state.src[cursor_start:] 

175 end_pos = state.cursor_max 

176 

177 if spaces and code: 

178 _trim_pattern = re.compile("^ {0," + str(len(spaces)) + "}", re.M) 

179 code = _trim_pattern.sub("", code) 

180 

181 token = {"type": "block_code", "raw": code, "style": "fenced", "marker": marker} 

182 if info: 

183 info = unescape_char(info) 

184 token["attrs"] = {"info": info.strip()} 

185 

186 state.append_token(token) 

187 return end_pos 

188 

189 def parse_atx_heading(self, m: Match[str], state: BlockState) -> int: 

190 """Parse token for ATX heading. An ATX heading is started with 1 to 6 

191 symbol of ``#``.""" 

192 level = len(m.group("atx_1")) 

193 text = m.group("atx_2").strip(string.whitespace) 

194 # remove last # 

195 if text: 

196 text = _ATX_HEADING_TRIM.sub("", text) 

197 

198 token = {"type": "heading", "text": text, "attrs": {"level": level}, "style": "atx"} 

199 state.append_token(token) 

200 return m.end() + 1 

201 

202 def parse_setex_heading(self, m: Match[str], state: BlockState) -> Optional[int]: 

203 """Parse token for setex style heading. A setex heading syntax looks like: 

204 

205 .. code-block:: markdown 

206 

207 H1 title 

208 ======== 

209 """ 

210 if state.cursor in state.lazy_line_starts: 

211 return None 

212 

213 last_token = state.last_token() 

214 if last_token and last_token["type"] == "paragraph": 

215 level = 1 if m.group("setext_1") == "=" else 2 

216 last_token["type"] = "heading" 

217 last_token["style"] = "setext" 

218 last_token["attrs"] = {"level": level} 

219 return m.end() + 1 

220 

221 sc = self.compile_sc(["thematic_break", "list"]) 

222 m2 = sc.match(state.src, state.cursor) 

223 if m2: 

224 return self.parse_method(m2, state) 

225 return None 

226 

227 def parse_ref_link(self, m: Match[str], state: BlockState) -> Optional[int]: 

228 """Parse link references and save the link information into ``state.env``. 

229 

230 Here is an example of a link reference: 

231 

232 .. code-block:: markdown 

233 

234 a [link][example] 

235 

236 [example]: https://example.com "Optional title" 

237 

238 This method will save the link reference into ``state.env`` as:: 

239 

240 state.env['ref_links']['example'] = { 

241 'url': 'https://example.com', 

242 'title': "Optional title", 

243 } 

244 """ 

245 end_pos = state.append_paragraph() 

246 if end_pos: 

247 return end_pos 

248 

249 label = m.group("reflink_1") 

250 key = unikey(label) 

251 if not key: 

252 return None 

253 

254 href, href_pos = parse_link_href(state.src, m.end(), block=True) 

255 if href is None: 

256 return None 

257 

258 assert href_pos is not None 

259 

260 blank_pos = _find_next_blank_line(state, href_pos, self.BLANK_LINE) 

261 if blank_pos is None: 

262 max_pos = state.cursor_max 

263 else: 

264 max_pos = blank_pos 

265 

266 title, title_pos = parse_link_title(state.src, href_pos, max_pos) 

267 if title_pos: 

268 m2 = _BLANK_TO_LINE.match(state.src, title_pos) 

269 if m2: 

270 title_pos = m2.end() 

271 else: 

272 title_pos = None 

273 title = None 

274 

275 if title_pos is None: 

276 m3 = _BLANK_TO_LINE.match(state.src, href_pos) 

277 if m3: 

278 href_pos = m3.end() 

279 else: 

280 href_pos = None 

281 href = None 

282 

283 end_pos = title_pos or href_pos 

284 if not end_pos: 

285 return None 

286 

287 if key not in state.env["ref_links"]: 

288 assert href is not None 

289 href = unescape_char(href) 

290 data = {"url": escape_url(href), "label": label} 

291 if title: 

292 data["title"] = title 

293 state.env["ref_links"][key] = data 

294 return end_pos 

295 

296 def extract_block_quote(self, m: Match[str], state: BlockState) -> Tuple[str, Optional[int], Set[int]]: 

297 """Extract text and cursor end position of a block quote.""" 

298 

299 text = _parse_block_quote_line(state.get_line(state.cursor)) 

300 assert text is not None 

301 lazy_line_starts: Set[int] = set() 

302 

303 sc = self.compile_sc(["blank_line", "indent_code", "fenced_code"]) 

304 require_marker = bool(sc.match(text)) 

305 

306 state.cursor += len(state.get_line(state.cursor)) 

307 

308 end_pos: Optional[int] = None 

309 if require_marker: 

310 while state.cursor < state.cursor_max: 

311 quote = _parse_block_quote_line(state.get_line(state.cursor)) 

312 if quote is None: 

313 break 

314 text += quote 

315 state.cursor += len(state.get_line(state.cursor)) 

316 else: 

317 prev_blank_line = False 

318 break_sc = self.compile_sc( 

319 [ 

320 "blank_line", 

321 "thematic_break", 

322 "fenced_code", 

323 "list", 

324 "block_html", 

325 ] 

326 ) 

327 while state.cursor < state.cursor_max: 

328 quote = _parse_block_quote_line(state.get_line(state.cursor)) 

329 if quote is not None: 

330 text += quote 

331 state.cursor += len(state.get_line(state.cursor)) 

332 if not quote.strip(): 

333 prev_blank_line = True 

334 else: 

335 prev_blank_line = False 

336 continue 

337 

338 if prev_blank_line: 

339 # CommonMark Example 249 

340 # because of laziness, a blank line is needed between 

341 # a block quote and a following paragraph 

342 break 

343 

344 m4 = break_sc.match(state.src, state.cursor) 

345 if m4: 

346 end_pos = self.parse_method(m4, state) 

347 if end_pos: 

348 break 

349 

350 # lazy continuation line 

351 line = state.get_line(state.cursor) 

352 lazy_line_starts.add(len(text)) 

353 text += expand_leading_tab(line, 3) 

354 state.cursor += len(line) 

355 

356 # according to CommonMark Example 6, the second tab should be 

357 # treated as 4 spaces 

358 return expand_tab(text), end_pos, lazy_line_starts 

359 

360 def parse_block_quote(self, m: Match[str], state: BlockState) -> int: 

361 """Parse token for block quote. Here is an example of the syntax: 

362 

363 .. code-block:: markdown 

364 

365 > a block quote starts 

366 > with right arrows 

367 """ 

368 text, end_pos, lazy_line_starts = self.extract_block_quote(m, state) 

369 # scan children state 

370 child = state.child_state(text, lazy_line_starts=lazy_line_starts) 

371 if state.depth() >= self.max_nested_level - 1: 

372 # At the nesting limit, stop descending into any further container 

373 # blocks. Trimming only "block_quote" still allowed block quotes and 

374 # lists to recurse into each other without bound (RecursionError). 

375 rules = [rule for rule in self.block_quote_rules if rule not in ("block_quote", "list")] 

376 else: 

377 rules = self.block_quote_rules 

378 

379 self.parse(child, rules) 

380 token = {"type": "block_quote", "children": child.tokens} 

381 if end_pos: 

382 state.prepend_token(token) 

383 return end_pos 

384 state.append_token(token) 

385 return state.cursor 

386 

387 def parse_list(self, m: Match[str], state: BlockState) -> int: 

388 """Parse tokens for ordered and unordered list.""" 

389 return parse_list(self, m, state) 

390 

391 def parse_block_html(self, m: Match[str], state: BlockState) -> Optional[int]: 

392 return self.parse_raw_html(m, state) 

393 

394 def parse_raw_html(self, m: Match[str], state: BlockState) -> Optional[int]: 

395 marker = m.group(0).strip() 

396 

397 # rule 2 

398 if marker == "<!--": 

399 return _parse_html_to_end(state, "-->", m.end()) 

400 

401 # rule 3 

402 if marker == "<?": 

403 return _parse_html_to_end(state, "?>", m.end()) 

404 

405 # rule 5 

406 if marker == "<![CDATA[": 

407 return _parse_html_to_end(state, "]]>", m.end()) 

408 

409 # rule 4 

410 if marker.startswith("<!"): 

411 return _parse_html_to_end(state, ">", m.end()) 

412 

413 close_tag = None 

414 open_tag = None 

415 if marker.startswith("</"): 

416 close_tag = marker[2:].lower() 

417 # rule 6 

418 if close_tag in BLOCK_TAGS: 

419 return _parse_html_to_newline(state, self.BLANK_LINE) 

420 else: 

421 open_tag = marker[1:].lower() 

422 # rule 1 

423 if open_tag in PRE_TAGS: 

424 end_tag = "</" + open_tag + ">" 

425 return _parse_html_to_end(state, end_tag, m.end()) 

426 # rule 6 

427 if open_tag in BLOCK_TAGS: 

428 return _parse_html_to_newline(state, self.BLANK_LINE) 

429 

430 # Blocks of type 7 may not interrupt a paragraph. 

431 end_pos = state.append_paragraph() 

432 if end_pos: 

433 return end_pos 

434 

435 # rule 7 

436 start_pos = m.end() 

437 end_pos = state.find_line_end() 

438 if (open_tag and _OPEN_TAG_END.match(state.src, start_pos, end_pos)) or ( 

439 close_tag and _CLOSE_TAG_END.match(state.src, start_pos, end_pos) 

440 ): 

441 return _parse_html_to_newline(state, self.BLANK_LINE) 

442 

443 return None 

444 

445 def parse(self, state: BlockState, rules: Optional[List[str]] = None) -> None: 

446 sc = self.compile_sc(rules) 

447 

448 while state.cursor < state.cursor_max: 

449 m = sc.match(state.src, state.cursor) 

450 if not m and self._parse_plain_paragraph(state, sc): 

451 continue 

452 

453 if not m: 

454 m = sc.search(state.src, state.cursor) 

455 if not m: 

456 break 

457 

458 end_pos = m.start() 

459 if end_pos > state.cursor: 

460 text = state.get_text(end_pos) 

461 state.add_paragraph(text) 

462 state.cursor = end_pos 

463 

464 end_pos2 = self.parse_method(m, state) 

465 if end_pos2: 

466 state.cursor = end_pos2 

467 else: 

468 end_pos3 = state.find_line_end() 

469 text = state.get_text(end_pos3) 

470 state.add_paragraph(text) 

471 state.cursor = end_pos3 

472 

473 if state.cursor < state.cursor_max: 

474 text = state.src[state.cursor :] 

475 state.add_paragraph(text) 

476 state.cursor = state.cursor_max 

477 

478 def _parse_plain_paragraph(self, state: BlockState, sc: Pattern[str]) -> bool: 

479 if not _is_plain_paragraph_start(state.src, state.cursor): 

480 return False 

481 

482 pos = state.cursor 

483 while pos < state.cursor_max: 

484 if pos > state.cursor and sc.match(state.src, pos): 

485 break 

486 

487 line = state.get_line(pos) 

488 if not line.strip(): 

489 break 

490 

491 pos += len(line) 

492 

493 if pos <= state.cursor: 

494 return False 

495 

496 state.add_paragraph(state.get_text(pos)) 

497 state.cursor = pos 

498 return True 

499 

500 

501def _parse_html_to_end(state: BlockState, end_marker: str, start_pos: int) -> int: 

502 marker_pos = state.src.find(end_marker, start_pos) 

503 if marker_pos == -1: 

504 text = state.src[state.cursor :] 

505 end_pos = state.cursor_max 

506 else: 

507 text = state.get_text(marker_pos) 

508 state.cursor = marker_pos 

509 end_pos = state.find_line_end() 

510 text += state.get_text(end_pos) 

511 

512 state.append_token({"type": "block_html", "raw": text}) 

513 return end_pos 

514 

515 

516def _parse_html_to_newline(state: BlockState, newline: Pattern[str]) -> int: 

517 m = newline.search(state.src, state.cursor) 

518 if m: 

519 end_pos = m.start() 

520 text = state.get_text(end_pos) 

521 else: 

522 text = state.src[state.cursor :] 

523 end_pos = state.cursor_max 

524 

525 state.append_token({"type": "block_html", "raw": text}) 

526 return end_pos 

527 

528 

529def _parse_block_quote_line(line: str) -> Optional[str]: 

530 m = _BLOCK_QUOTE_LINE.match(line) 

531 if not m: 

532 return None 

533 text = expand_leading_tab(m.group(1), 3) 

534 return _BLOCK_QUOTE_TRIM.sub("", text) 

535 

536 

537def _find_next_blank_line(state: BlockState, pos: int, pattern: Pattern[str]) -> Optional[int]: 

538 cache = state.env.get("__blank_line_starts__") 

539 if cache is None or cache[0] is not state.src: 

540 cache = (state.src, [m.start() for m in pattern.finditer(state.src)]) 

541 state.env["__blank_line_starts__"] = cache 

542 

543 positions = cache[1] 

544 index = bisect_left(positions, pos) 

545 if index < len(positions): 

546 return positions[index] 

547 return None 

548 

549 

550def _trim_partial_next_line_indent(text: str, end_pos: int) -> int: 

551 line_start = text.rfind("\n") + 1 

552 if line_start == 0: 

553 return end_pos 

554 

555 suffix = text[line_start:] 

556 if suffix and suffix.strip(" \t") == "" and len(suffix.expandtabs(4)) < 4: 

557 return end_pos - len(suffix) 

558 return end_pos 

559 

560 

561def _is_plain_paragraph_start(src: str, pos: int) -> bool: 

562 if pos >= len(src): 

563 return False 

564 c = src[pos] 

565 return not c.isspace() and not c.isdigit() and c not in string.punctuation