Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/mistune/block_parser.py: 95%

1import re 

2import string 

3from bisect import bisect_left 

4from typing import List, Match, Optional, Pattern, Set, Tuple 

5 

6from .core import DEFAULT_MAX_NESTED_LEVEL, BlockState, Parser 

7from .helpers import ( 

8 BLOCK_TAGS, 

9 HTML_ATTRIBUTES, 

10 HTML_TAGNAME, 

11 LINK_LABEL, 

12 PRE_TAGS, 

13 parse_link_href, 

14 parse_link_title, 

15 unescape_char, 

16) 

17from .list_parser import LIST_PATTERN, parse_list 

18from .util import escape_url, expand_leading_tab, expand_tab, unikey 

19 

20_INDENT_CODE_TRIM = re.compile(r"^ {1,4}", flags=re.M) 

21_ATX_HEADING_TRIM = re.compile(r"(\s+|^)#+\s*$") 

22_BLOCK_QUOTE_TRIM = re.compile(r"^ ?", flags=re.M) 

23 

24_BLANK_TO_LINE = re.compile(r"[ \t]*\n") 

25 

26_BLOCK_TAGS_PATTERN = "(" + "|".join(BLOCK_TAGS) + "|" + "|".join(PRE_TAGS) + ")" 

27_OPEN_TAG_END = re.compile(HTML_ATTRIBUTES + r"[ \t]*>[ \t]*(?:\n|$)") 

28_CLOSE_TAG_END = re.compile(r"[ \t]*>[ \t]*(?:\n|$)") 

29_BLOCK_QUOTE_LINE = re.compile(r"^ {0,3}>([^\n]*(?:\n|$))") 

30 

31 

32class BlockParser(Parser[BlockState]): 

33 state_cls = BlockState 

34 

35 BLANK_LINE = re.compile(r"(^[ \t\v\f]*\n)+", re.M) 

36 

37 RAW_HTML = ( 

38 r"^ {0,3}(" 

39 r"</?" + HTML_TAGNAME + r"|" 

40 r"<!--|" # comment 

41 r"<\?|" # script 

42 r"<![A-Z]|" 

43 r"<!\[CDATA\[)" 

44 ) 

45 

46 BLOCK_HTML = ( 

47 r"^ {0,3}(?:" 

48 r"(?:</?" + _BLOCK_TAGS_PATTERN + r"(?:[ \t]+|\n|$))" 

49 r"|<!--" # comment 

50 r"|<\?" # script 

51 r"|<![A-Z]" 

52 r"|<!\[CDATA\[)" 

53 ) 

54 

55 SPECIFICATION = { 

56 "blank_line": r"(^[ \t\v\f]*\n)+", 

57 "atx_heading": r"^ {0,3}(?P<atx_1>#{1,6})(?!#+)(?P<atx_2>[ \t]*|[ \t]+.*?)$", 

58 "setex_heading": r"^ {0,3}(?P<setext_1>=|-){1,}[ \t]*$", 

59 "fenced_code": ( 

60 r"^(?P<fenced_1> {0,3})(?P<fenced_2>`{3,}|~{3,})" 

61 r"[ \t]*(?P<fenced_3>.*?)$" 

62 ), 

63 "indent_code": ( 

64 r"^(?: {4}| *\t)[^\n]+(?:\n+|$)" 

65 r"((?:(?: {4}| *\t)[^\n]+(?:\n+|$))|\s)*" 

66 ), 

67 "thematic_break": r"^ {0,3}((?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})$", 

68 "ref_link": r"^ {0,3}\[(?P<reflink_1>" + LINK_LABEL + r")\]:", 

69 "block_quote": r"^ {0,3}>(?P<quote_1>.*?)$", 

70 "list": LIST_PATTERN, 

71 "block_html": BLOCK_HTML, 

72 "raw_html": RAW_HTML, 

73 } 

74 

75 DEFAULT_RULES = ( 

76 "fenced_code", 

77 "indent_code", 

78 "atx_heading", 

79 "setex_heading", 

80 "thematic_break", 

81 "block_quote", 

82 "list", 

83 "ref_link", 

84 "raw_html", 

85 "blank_line", 

86 ) 

87 

88 def __init__( 

89 self, 

90 block_quote_rules: Optional[List[str]] = None, 

91 list_rules: Optional[List[str]] = None, 

92 max_nested_level: int = DEFAULT_MAX_NESTED_LEVEL, 

93 ): 

94 super(BlockParser, self).__init__() 

95 

96 if block_quote_rules is None: 

97 block_quote_rules = list(self.DEFAULT_RULES) 

98 

99 if list_rules is None: 

100 list_rules = list(self.DEFAULT_RULES) 

101 

102 if max_nested_level < 1: 

103 raise ValueError("max_nested_level must be positive") 

104 self.block_quote_rules = block_quote_rules 

105 self.list_rules = list_rules 

106 self.max_nested_level = max_nested_level 

107 # register default parse methods 

108 self._methods = {name: getattr(self, "parse_" + name) for name in self.SPECIFICATION} 

109 

110 def parse_blank_line(self, m: Match[str], state: BlockState) -> int: 

111 """Parse token for blank lines.""" 

112 if state.cursor == 0: 

113 state.append_token({"type": "blank_line", "newlines": m.group(0).count("\n")}) 

114 else: 

115 state.append_token({"type": "blank_line"}) 

116 return m.end() 

117 

118 def parse_thematic_break(self, m: Match[str], state: BlockState) -> int: 

119 """Parse token for thematic break, e.g. ``<hr>`` tag in HTML.""" 

120 state.append_token({"type": "thematic_break"}) 

121 # $ does not count '\n' 

122 return m.end() + 1 

123 

124 def parse_indent_code(self, m: Match[str], state: BlockState) -> int: 

125 """Parse token for code block which is indented by 4 spaces.""" 

126 # it is a part of the paragraph 

127 end_pos = state.append_paragraph() 

128 if end_pos: 

129 return end_pos 

130 

131 code = m.group(0) 

132 end_pos = _trim_partial_next_line_indent(code, m.end()) 

133 if end_pos != m.end(): 

134 code = state.get_text(end_pos) 

135 code = expand_leading_tab(code) 

136 code = _INDENT_CODE_TRIM.sub("", code) 

137 code = code.strip("\n") 

138 state.append_token({"type": "block_code", "raw": code, "style": "indent"}) 

139 return end_pos 

140 

141 def parse_fenced_code(self, m: Match[str], state: BlockState) -> Optional[int]: 

142 """Parse token for fenced code block. A fenced code block is started with 

143 3 or more backtick(`) or tilde(~). 

144 

145 An example of a fenced code block: 

146 

147 .. code-block:: markdown 

148 

149 ```python 

150 def markdown(text): 

151 return mistune.html(text) 

152 ``` 

153 """ 

154 spaces = m.group("fenced_1") 

155 marker = m.group("fenced_2") 

156 info = m.group("fenced_3") 

157 

158 c = marker[0] 

159 if info and c == "`": 

160 # CommonMark Example 145 

161 # Info strings for backtick code blocks cannot contain backticks 

162 if info.find(c) != -1: 

163 return None 

164 

165 _end = re.compile(r"^ {0,3}" + c + "{" + str(len(marker)) + r",}[ \t]*(?:\n|$)", re.M) 

166 cursor_start = m.end() + 1 

167 

168 m2 = _end.search(state.src, cursor_start) 

169 if m2: 

170 code = state.src[cursor_start : m2.start()] 

171 end_pos = m2.end() 

172 else: 

173 code = state.src[cursor_start:] 

174 end_pos = state.cursor_max 

175 

176 if spaces and code: 

177 _trim_pattern = re.compile("^ {0," + str(len(spaces)) + "}", re.M) 

178 code = _trim_pattern.sub("", code) 

179 

180 token = {"type": "block_code", "raw": code, "style": "fenced", "marker": marker} 

181 if info: 

182 info = unescape_char(info) 

183 token["attrs"] = {"info": info.strip()} 

184 

185 state.append_token(token) 

186 return end_pos 

187 

188 def parse_atx_heading(self, m: Match[str], state: BlockState) -> int: 

189 """Parse token for ATX heading. An ATX heading is started with 1 to 6 

190 symbol of ``#``.""" 

191 level = len(m.group("atx_1")) 

192 text = m.group("atx_2").strip(string.whitespace) 

193 # remove last # 

194 if text: 

195 text = _ATX_HEADING_TRIM.sub("", text) 

196 

197 token = {"type": "heading", "text": text, "attrs": {"level": level}, "style": "atx"} 

198 state.append_token(token) 

199 return m.end() + 1 

200 

201 def parse_setex_heading(self, m: Match[str], state: BlockState) -> Optional[int]: 

202 """Parse token for setex style heading. A setex heading syntax looks like: 

203 

204 .. code-block:: markdown 

205 

206 H1 title 

207 ======== 

208 """ 

209 if state.cursor in state.lazy_line_starts: 

210 return None 

211 

212 last_token = state.last_token() 

213 if last_token and last_token["type"] == "paragraph": 

214 level = 1 if m.group("setext_1") == "=" else 2 

215 last_token["type"] = "heading" 

216 last_token["style"] = "setext" 

217 last_token["attrs"] = {"level": level} 

218 return m.end() + 1 

219 

220 sc = self.compile_sc(["thematic_break", "list"]) 

221 m2 = sc.match(state.src, state.cursor) 

222 if m2: 

223 return self.parse_method(m2, state) 

224 return None 

225 

226 def parse_ref_link(self, m: Match[str], state: BlockState) -> Optional[int]: 

227 """Parse link references and save the link information into ``state.env``. 

228 

229 Here is an example of a link reference: 

230 

231 .. code-block:: markdown 

232 

233 a [link][example] 

234 

235 [example]: https://example.com "Optional title" 

236 

237 This method will save the link reference into ``state.env`` as:: 

238 

239 state.env['ref_links']['example'] = { 

240 'url': 'https://example.com', 

241 'title': "Optional title", 

242 } 

243 """ 

244 end_pos = state.append_paragraph() 

245 if end_pos: 

246 return end_pos 

247 

248 label = m.group("reflink_1") 

249 key = unikey(label) 

250 if not key: 

251 return None 

252 

253 href, href_pos = parse_link_href(state.src, m.end(), block=True, max_nested_level=self.max_nested_level) 

254 if href is None: 

255 return None 

256 

257 assert href_pos is not None 

258 

259 blank_pos = _find_next_blank_line(state, href_pos, self.BLANK_LINE) 

260 if blank_pos is None: 

261 max_pos = state.cursor_max 

262 else: 

263 max_pos = blank_pos 

264 

265 title, title_pos = parse_link_title(state.src, href_pos, max_pos) 

266 if title_pos: 

267 m2 = _BLANK_TO_LINE.match(state.src, title_pos) 

268 if m2: 

269 title_pos = m2.end() 

270 else: 

271 title_pos = None 

272 title = None 

273 

274 if title_pos is None: 

275 m3 = _BLANK_TO_LINE.match(state.src, href_pos) 

276 if m3: 

277 href_pos = m3.end() 

278 else: 

279 href_pos = None 

280 href = None 

281 

282 end_pos = title_pos or href_pos 

283 if not end_pos: 

284 return None 

285 

286 if key not in state.env["ref_links"]: 

287 assert href is not None 

288 href = unescape_char(href) 

289 data = {"url": escape_url(href), "label": label} 

290 if title: 

291 data["title"] = title 

292 state.env["ref_links"][key] = data 

293 return end_pos 

294 

295 def extract_block_quote(self, m: Match[str], state: BlockState) -> Tuple[str, Optional[int], Set[int]]: 

296 """Extract text and cursor end position of a block quote.""" 

297 

298 text = _parse_block_quote_line(state.get_line(state.cursor)) 

299 assert text is not None 

300 lazy_line_starts: Set[int] = set() 

301 

302 sc = self.compile_sc(["blank_line", "indent_code", "fenced_code"]) 

303 require_marker = bool(sc.match(text)) 

304 

305 state.cursor += len(state.get_line(state.cursor)) 

306 

307 end_pos: Optional[int] = None 

308 if require_marker: 

309 while state.cursor < state.cursor_max: 

310 quote = _parse_block_quote_line(state.get_line(state.cursor)) 

311 if quote is None: 

312 break 

313 text += quote 

314 state.cursor += len(state.get_line(state.cursor)) 

315 else: 

316 prev_blank_line = False 

317 break_sc = self.compile_sc( 

318 [ 

319 "blank_line", 

320 "thematic_break", 

321 "fenced_code", 

322 "list", 

323 "block_html", 

324 ] 

325 ) 

326 while state.cursor < state.cursor_max: 

327 quote = _parse_block_quote_line(state.get_line(state.cursor)) 

328 if quote is not None: 

329 text += quote 

330 state.cursor += len(state.get_line(state.cursor)) 

331 if not quote.strip(): 

332 prev_blank_line = True 

333 else: 

334 prev_blank_line = False 

335 continue 

336 

337 if prev_blank_line: 

338 # CommonMark Example 249 

339 # because of laziness, a blank line is needed between 

340 # a block quote and a following paragraph 

341 break 

342 

343 m4 = break_sc.match(state.src, state.cursor) 

344 if m4: 

345 end_pos = self.parse_method(m4, state) 

346 if end_pos: 

347 break 

348 

349 # lazy continuation line 

350 line = state.get_line(state.cursor) 

351 lazy_line_starts.add(len(text)) 

352 text += expand_leading_tab(line, 3) 

353 state.cursor += len(line) 

354 

355 # according to CommonMark Example 6, the second tab should be 

356 # treated as 4 spaces 

357 return expand_tab(text), end_pos, lazy_line_starts 

358 

359 def parse_block_quote(self, m: Match[str], state: BlockState) -> int: 

360 """Parse token for block quote. Here is an example of the syntax: 

361 

362 .. code-block:: markdown 

363 

364 > a block quote starts 

365 > with right arrows 

366 """ 

367 text, end_pos, lazy_line_starts = self.extract_block_quote(m, state) 

368 # scan children state 

369 child = state.child_state(text, lazy_line_starts=lazy_line_starts) 

370 if state.depth() >= self.max_nested_level - 1: 

371 # At the nesting limit, stop descending into any further container 

372 # blocks. Trimming only "block_quote" still allowed block quotes and 

373 # lists to recurse into each other without bound (RecursionError). 

374 rules = [rule for rule in self.block_quote_rules if rule not in ("block_quote", "list")] 

375 else: 

376 rules = self.block_quote_rules 

377 

378 self.parse(child, rules) 

379 token = {"type": "block_quote", "children": child.tokens} 

380 if end_pos: 

381 state.prepend_token(token) 

382 return end_pos 

383 state.append_token(token) 

384 return state.cursor 

385 

386 def parse_list(self, m: Match[str], state: BlockState) -> int: 

387 """Parse tokens for ordered and unordered list.""" 

388 return parse_list(self, m, state) 

389 

390 def parse_block_html(self, m: Match[str], state: BlockState) -> Optional[int]: 

391 return self.parse_raw_html(m, state) 

392 

393 def parse_raw_html(self, m: Match[str], state: BlockState) -> Optional[int]: 

394 marker = m.group(0).strip() 

395 

396 # rule 2 

397 if marker == "<!--": 

398 return _parse_html_to_end(state, "-->", m.end()) 

399 

400 # rule 3 

401 if marker == "<?": 

402 return _parse_html_to_end(state, "?>", m.end()) 

403 

404 # rule 5 

405 if marker == "<![CDATA[": 

406 return _parse_html_to_end(state, "]]>", m.end()) 

407 

408 # rule 4 

409 if marker.startswith("<!"): 

410 return _parse_html_to_end(state, ">", m.end()) 

411 

412 close_tag = None 

413 open_tag = None 

414 if marker.startswith("</"): 

415 close_tag = marker[2:].lower() 

416 # rule 6 

417 if close_tag in BLOCK_TAGS: 

418 return _parse_html_to_newline(state, self.BLANK_LINE) 

419 else: 

420 open_tag = marker[1:].lower() 

421 # rule 1 

422 if open_tag in PRE_TAGS: 

423 end_tag = "</" + open_tag + ">" 

424 return _parse_html_to_end(state, end_tag, m.end()) 

425 # rule 6 

426 if open_tag in BLOCK_TAGS: 

427 return _parse_html_to_newline(state, self.BLANK_LINE) 

428 

429 # Blocks of type 7 may not interrupt a paragraph. 

430 end_pos = state.append_paragraph() 

431 if end_pos: 

432 return end_pos 

433 

434 # rule 7 

435 start_pos = m.end() 

436 end_pos = state.find_line_end() 

437 if (open_tag and _OPEN_TAG_END.match(state.src, start_pos, end_pos)) or ( 

438 close_tag and _CLOSE_TAG_END.match(state.src, start_pos, end_pos) 

439 ): 

440 return _parse_html_to_newline(state, self.BLANK_LINE) 

441 

442 return None 

443 

444 def parse(self, state: BlockState, rules: Optional[List[str]] = None) -> None: 

445 if rules is None: 

446 rules = self.rules 

447 sc = self.compile_sc(rules) 

448 

449 while state.cursor < state.cursor_max: 

450 if "indent_code" in rules: 

451 end_pos = self._parse_indent_code_fast(state) 

452 if end_pos is not None: 

453 state.cursor = end_pos 

454 continue 

455 

456 m = sc.match(state.src, state.cursor) 

457 if not m and self._parse_plain_paragraph(state, sc): 

458 continue 

459 

460 if not m: 

461 m = sc.search(state.src, state.cursor) 

462 if not m: 

463 break 

464 

465 end_pos = m.start() 

466 if end_pos > state.cursor: 

467 text = state.get_text(end_pos) 

468 state.add_paragraph(text) 

469 state.cursor = end_pos 

470 

471 end_pos2 = self.parse_method(m, state) 

472 if end_pos2: 

473 state.cursor = end_pos2 

474 else: 

475 end_pos3 = state.find_line_end() 

476 text = state.get_text(end_pos3) 

477 state.add_paragraph(text) 

478 state.cursor = end_pos3 

479 

480 if state.cursor < state.cursor_max: 

481 text = state.src[state.cursor :] 

482 state.add_paragraph(text) 

483 state.cursor = state.cursor_max 

484 

485 def _parse_indent_code_fast(self, state: BlockState) -> Optional[int]: 

486 line_end = state.find_line_end() 

487 line = state.get_text(line_end) 

488 if not _is_indent_code_line(line): 

489 return None 

490 

491 if state.last_token() and state.last_token()["type"] == "paragraph": 

492 return state.append_paragraph() 

493 

494 end_pos = _find_indent_code_end(state.src, line_end, state.cursor_max) 

495 code = state.get_text(end_pos) 

496 code = expand_leading_tab(code) 

497 code = _INDENT_CODE_TRIM.sub("", code) 

498 code = code.strip("\n") 

499 state.append_token({"type": "block_code", "raw": code, "style": "indent"}) 

500 return end_pos 

501 

502 def _parse_plain_paragraph(self, state: BlockState, sc: Pattern[str]) -> bool: 

503 if not _is_plain_paragraph_start(state.src, state.cursor): 

504 return False 

505 

506 pos = state.cursor 

507 while pos < state.cursor_max: 

508 if pos > state.cursor and sc.match(state.src, pos): 

509 break 

510 

511 line = state.get_line(pos) 

512 if not line.strip(): 

513 break 

514 

515 pos += len(line) 

516 

517 if pos <= state.cursor: 

518 return False 

519 

520 state.add_paragraph(state.get_text(pos)) 

521 state.cursor = pos 

522 return True 

523 

524 

525def _parse_html_to_end(state: BlockState, end_marker: str, start_pos: int) -> int: 

526 marker_pos = state.src.find(end_marker, start_pos) 

527 if marker_pos == -1: 

528 text = state.src[state.cursor :] 

529 end_pos = state.cursor_max 

530 else: 

531 text = state.get_text(marker_pos) 

532 state.cursor = marker_pos 

533 end_pos = state.find_line_end() 

534 text += state.get_text(end_pos) 

535 

536 state.append_token({"type": "block_html", "raw": text}) 

537 return end_pos 

538 

539 

540def _parse_html_to_newline(state: BlockState, newline: Pattern[str]) -> int: 

541 m = newline.search(state.src, state.cursor) 

542 if m: 

543 end_pos = m.start() 

544 text = state.get_text(end_pos) 

545 else: 

546 text = state.src[state.cursor :] 

547 end_pos = state.cursor_max 

548 

549 state.append_token({"type": "block_html", "raw": text}) 

550 return end_pos 

551 

552 

553def _parse_block_quote_line(line: str) -> Optional[str]: 

554 m = _BLOCK_QUOTE_LINE.match(line) 

555 if not m: 

556 return None 

557 text = expand_leading_tab(m.group(1), 3) 

558 return _BLOCK_QUOTE_TRIM.sub("", text) 

559 

560 

561def _find_next_blank_line(state: BlockState, pos: int, pattern: Pattern[str]) -> Optional[int]: 

562 cache = state.env.get("__blank_line_starts__") 

563 if cache is None or cache[0] is not state.src: 

564 cache = (state.src, [m.start() for m in pattern.finditer(state.src)]) 

565 state.env["__blank_line_starts__"] = cache 

566 

567 positions = cache[1] 

568 index = bisect_left(positions, pos) 

569 if index < len(positions): 

570 return positions[index] 

571 return None 

572 

573 

574def _trim_partial_next_line_indent(text: str, end_pos: int) -> int: 

575 line_start = text.rfind("\n") + 1 

576 if line_start == 0: 

577 return end_pos 

578 

579 suffix = text[line_start:] 

580 if suffix and suffix.strip(" \t") == "" and len(suffix.expandtabs(4)) < 4: 

581 return end_pos - len(suffix) 

582 return end_pos 

583 

584 

585def _is_indent_code_line(line: str) -> bool: 

586 if line.endswith("\n"): 

587 line = line[:-1] 

588 

589 if len(line) > 4 and line.startswith(" "): 

590 return True 

591 

592 index = 0 

593 while index < len(line) and line[index] == " ": 

594 index += 1 

595 return index + 1 < len(line) and line[index] == "\t" 

596 

597 

598def _find_indent_code_end(src: str, start: int, end: int) -> int: 

599 pos = start 

600 while pos < end: 

601 newline = src.find("\n", pos, end) 

602 line_end = end if newline < 0 else newline + 1 

603 line = src[pos:line_end] 

604 if _is_indent_code_line(line) or not line.strip(): 

605 pos = line_end 

606 continue 

607 break 

608 return pos 

609 

610 

611def _is_plain_paragraph_start(src: str, pos: int) -> bool: 

612 if pos >= len(src): 

613 return False 

614 c = src[pos] 

615 return not c.isspace() and not c.isdigit() and c not in string.punctuation