Coverage for /pythoncovmergedfiles/medio/medio/src/turbohtml/tools/fuzz/markdown_structure_generators.py: 43%

1"""Reader wrappers must consume budgets even when rendered HTML has fewer nodes.""" 

2 

3from __future__ import annotations 

4 

5import argparse 

6import random 

7import re 

8from dataclasses import dataclass, field 

9from pathlib import Path 

10from typing import TYPE_CHECKING, Final 

11 

12from fuzz.structure_generators import ( 

13 BudgetError, 

14 Generated, 

15 GenerationBudget, 

16 Grammar, 

17 Identifier, 

18 Production, 

19 ProductionFloorError, 

20 Reference, 

21 compile_grammar, 

22 generate, 

23 generation_sweep, 

24 write_corpus, 

25) 

26from markdown_it import MarkdownIt 

27from markdown_it.tree import SyntaxTreeNode 

28from typing_extensions import override 

29 

30from turbohtml import Element, Namespace, Node, Text, parse_fragment 

31 

32if TYPE_CHECKING: 

33 from collections.abc import Callable, Iterable, Iterator, Sequence 

34 

35MarkdownRecord = tuple[str, str, str, tuple[tuple[str, str], ...], int] 

36_Meaning = tuple[str, tuple[tuple[str, str], ...], tuple["_Meaning", ...]] 

37 

38 

39def main(argv: Sequence[str] | None = None) -> int: 

40 """Write source bytes so corpus consumers need no mode-string interpretation.""" 

41 parser: Final = argparse.ArgumentParser() 

42 parser.add_argument("--output", type=Path, required=True) 

43 parser.add_argument("--budget", type=int, default=64) 

44 parser.add_argument("--steps", type=int, default=256) 

45 parser.add_argument("--count", type=int, default=0) 

46 parser.add_argument("--seed", type=int, default=0) 

47 parser.add_argument("--html", action="store_true") 

48 args: Final = parser.parse_args(argv) 

49 if args.count < 0: 

50 parser.error("count must be nonnegative") 

51 try: 

52 cases: Final = ( 

53 tuple( 

54 markdown_generate(random.Random(args.seed + index), args.budget, steps=args.steps, html=args.html) 

55 for index in range(args.count) 

56 ) 

57 if args.count 

58 else generation_sweep(markdown_grammar(html=args.html), budget=GenerationBudget(args.budget, args.steps)) 

59 ) 

60 except (BudgetError, ProductionFloorError) as error: 

61 parser.error(str(error)) 

62 write_corpus(cases, args.output) 

63 return 0 

64 

65 

66def markdown_generate(rng: random.Random, budget: int = 64, *, steps: int = 256, html: bool = False) -> Generated: 

67 """Reader wrappers and HTML elements require different node costs.""" 

68 return generate(markdown_grammar(html=html), rng, GenerationBudget(budget, steps)) 

69 

70 

71def markdown_grammar(*, html: bool = False) -> Grammar: 

72 """Provenance and finite floors let corpus consumers audit coverage.""" 

73 return _HTML_GRAMMAR if html else _RAW_GRAMMAR 

74 

75 

76def markdown_snapshot(source: str) -> tuple[MarkdownRecord, ...]: 

77 """Inline source spelling duplicates the semantic leaf text.""" 

78 nodes: Final = tuple(SyntaxTreeNode(_READER.parse(source)).walk()) 

79 return tuple( 

80 ( 

81 node.type, 

82 "" if node.is_root else node.tag, 

83 "" if node.is_root or node.type == "inline" else node.content, 

84 () if node.is_root else tuple(sorted((name, str(value)) for name, value in node.attrs.items())), 

85 -1 if node.parent is None else nodes.index(node.parent), 

86 ) 

87 for node in nodes 

88 ) 

89 

90 

91def markdown_source_check(source: str, render: Callable[[Node], str] = Node.to_markdown) -> str | None: 

92 """Check the first conversion because repeated passes can stabilize a loss.""" 

93 return markdown_html_check(_READER.render(source), render) 

94 

95 

96def markdown_html_check(markup: str, render: Callable[[Node], str] = Node.to_markdown) -> str | None: 

97 """Compare source meaning because a stable conversion can lose HTML semantics.""" 

98 expected: Final = markdown_html_meaning(markup) 

99 converted: Final = _READER.render(render(parse_fragment(markup))) 

100 if markdown_html_meaning(converted) == expected: 

101 return None 

102 if any(char in markup and char not in converted for char in "\x0b\x1c\x1d\x1e\x1f\x85"): 

103 msg = "Markdown reader discards control whitespace at paragraph edges" 

104 raise MarkdownProfileError(msg) 

105 return "Markdown conversion changes meaning" 

106 

107 

108def markdown_html_meaning(markup: str) -> tuple[_Meaning, ...]: 

109 """Collapse reader HTML to the structure and text the Markdown oracle compares.""" 

110 # read the tree to_markdown converts, so a parser that lags the WHATWG tree builder cannot pose as a lost meaning 

111 root: Final = parse_fragment(markup) 

112 if any(sum(1 for _ in element.ancestors) > 128 for element in root.iter_elements()): 

113 msg = "HTML nesting exceeds the meaning model" 

114 raise MarkdownProfileError(msg) 

115 return _flow(_children(_tree(root), preserve=False)) 

116 

117 

118def _tree(node: Element) -> _TreeNode: 

119 element: Final = _TreeNode( 

120 node.tag if node.namespace is Namespace.HTML else f"{{{node.namespace.value}}}{node.tag}", 

121 {name: node.attr(name) or "" for name in node.attrs}, 

122 ) 

123 for child in node.children: 

124 if isinstance(child, Text): 

125 if element.children: 

126 element.children[-1].tail += child.text 

127 else: 

128 element.text += child.text 

129 elif isinstance(child, Element): 

130 element.children.append(_tree(child)) 

131 return element 

132 

133 

134def _flow(records: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

135 output: Final[list[_Meaning]] = [] 

136 inline: Final[list[_Meaning]] = [] 

137 for record in (*_spread(records), _BOUNDARY): 

138 if record[0] in _FLOW_INLINE: 

139 inline.append(record) 

140 continue 

141 # a paragraph needs a non-blank line (CommonMark 4.8), so a run of breaks alone is none, as an empty <p> is 

142 if content := _paragraph(inline): 

143 output.append(("p", (), content)) 

144 inline.clear() 

145 if record != _BOUNDARY: 

146 output.append(record) 

147 return tuple(output) 

148 

149 

150def _paragraph(records: list[_Meaning]) -> tuple[_Meaning, ...]: 

151 # spreading emphasis can leave a run that starts or ends at a block, where its edge spaces collapse away 

152 return _block_end(_spacing(_merge(records), preserve=False, inline=False)) 

153 

154 

155def _spread(records: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

156 # a delimiter run pairs only within one block's inline content (CommonMark 6.2), so in a flow of blocks emphasis 

157 # around a block reaches each block's own inline runs 

158 output: Final[list[_Meaning]] = [] 

159 for record in records: 

160 if record[0] not in _EMPHASIS or not _holds_block(record): 

161 output.append(record) 

162 continue 

163 run: list[_Meaning] = [] 

164 for child in _spread(record[2]): 

165 if child[0] in _FLOW_INLINE: 

166 run.append(child) 

167 continue 

168 output.extend(_emphasis(record[0], tuple(run))) 

169 run = [] 

170 output.append(_emphasize_block(record[0], child)) 

171 output.extend(_emphasis(record[0], tuple(run))) 

172 return tuple(output) 

173 

174 

175def _holds_block(record: _Meaning) -> bool: 

176 return any(child[0] not in _FLOW_INLINE or _holds_block(child) for child in record[2]) 

177 

178 

179def _emphasize_block(tag: str, record: _Meaning) -> _Meaning: 

180 if record[0] in {"p", "td", *_HEADINGS}: 

181 return (record[0], record[1], _emphasis(tag, record[2])) 

182 if record[0] in {"blockquote", "li", "ul", "ol", "table", "tr"}: 

183 return (record[0], record[1], tuple(_emphasize_block(tag, child) for child in record[2])) 

184 return record # code holds literal text (CommonMark 4.5), and a rule or a boundary holds none 

185 

186 

187def _block_end(records: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

188 # a hard break that ends a block draws no line in a browser and does nothing in CommonMark (6.7), and a link with 

189 # no content draws nothing, so a break before one ends the block too 

190 tail = len(records) 

191 while tail and (records[tail - 1][0] == "br" or _draws_nothing(records[tail - 1])): 

192 tail -= 1 

193 kept: Final = tuple(record for record in records[tail:] if record[0] != "br") 

194 if len(kept) == len(records) - tail or not tail or records[tail - 1][0] != "#text": 

195 return (*records[:tail], *kept) 

196 text: Final = records[tail - 1][1][0][1].rstrip(" \t\n\f\r") 

197 return (*records[: tail - 1], *((("#text", (("value", text),), ()),) if text else ()), *kept) 

198 

199 

200def _draws_nothing(record: _Meaning) -> bool: 

201 return record[0] in {"a", *_EMPHASIS} and all(map(_draws_nothing, record[2])) 

202 

203 

204def _children(element: _TreeNode, *, preserve: bool) -> tuple[_Meaning, ...]: 

205 records: Final[list[_Meaning]] = list(_text(element.text, preserve=preserve)) 

206 for child in element: 

207 records.extend(_element(child, element.tag, preserve=preserve)) 

208 records.extend(_text(child.tail, preserve=preserve)) 

209 spaced: Final = _spacing(_merge(records), preserve=preserve, inline=element.tag in _INLINE_TEXT) 

210 return spaced if preserve or element.tag in _INLINE_TEXT else _block_end(spaced) 

211 

212 

213def _merge(records: Iterable[_Meaning]) -> tuple[_Meaning, ...]: 

214 merged: Final[list[_Meaning]] = [] 

215 for record in records: 

216 if record[0] == "#text" and merged and merged[-1][0] == "#text": 

217 merged[-1] = ("#text", (("value", merged[-1][1][0][1] + record[1][0][1]),), ()) 

218 else: 

219 merged.append(record) 

220 return tuple(merged) 

221 

222 

223def _spacing(records: tuple[_Meaning, ...], *, preserve: bool, inline: bool) -> tuple[_Meaning, ...]: 

224 output: Final[list[_Meaning]] = [] 

225 for index, record in enumerate(records): 

226 if record[0] != "#text" or preserve: 

227 output.append(record) 

228 continue 

229 text: str = re.sub(r"[ \t\n\f\r]+", " ", record[1][0][1]) 

230 previous: Final = records[index - 1][0] if index else None 

231 if (previous is None and not inline) or ( 

232 previous is not None and (previous not in _FLOW_INLINE or previous == "br") 

233 ): 

234 text = text.lstrip(" \t\n\f\r") 

235 following: Final = records[index + 1][0] if index + 1 < len(records) else None 

236 # a line drops the collapsible spaces at its end (CSS Text 3, 4.1.2), so a space before a break draws nothing 

237 if (following is None and not inline) or ( 

238 following is not None and (following not in _FLOW_INLINE or following == "br") 

239 ): 

240 text = text.rstrip(" \t\n\f\r") 

241 if text: 

242 output.append(("#text", (("value", text),), ())) 

243 return tuple(output) 

244 

245 

246def _element(element: _TreeNode, parent: str, *, preserve: bool) -> tuple[_Meaning, ...]: 

247 if element.tag in _HIDDEN or element.tag.endswith(("}script", "}style")): 

248 return () 

249 _check_profile(element, parent) 

250 children: Final = _children(element, preserve=preserve or element.tag == "pre") 

251 if element.tag == "div": 

252 return (_BOUNDARY, *children, _BOUNDARY) # a block ends the paragraph around it even when it holds no text 

253 if ( 

254 element.tag in {"span", "thead", "tbody"} 

255 # an <a> without href is a placeholder (WHATWG 4.5.1) and Markdown has no link without a destination 

256 or (element.tag == "a" and "href" not in element.attrib) 

257 ): 

258 return children 

259 return _record({"b": "strong", "i": "em", "del": "s", "th": "td"}.get(element.tag, element.tag), element, children) 

260 

261 

262def _check_profile(element: _TreeNode, parent: str) -> None: 

263 if element.tag not in _ALLOWED: 

264 msg = f"HTML element has no declared Markdown meaning: {element.tag}" 

265 raise MarkdownProfileError(msg) 

266 if _breaks_content_model(element, parent): 

267 msg = f"HTML content model violated at {element.tag}" 

268 raise MarkdownProfileError(msg) 

269 

270 

271def _breaks_content_model(element: _TreeNode, parent: str) -> bool: 

272 if element.tag == "li": 

273 return parent not in {"ul", "ol"} 

274 if element.tag == "a": 

275 # an a holds no a descendant (WHATWG 4.5.1), and link text holds no link (CommonMark 6.3) 

276 return any(descendant.tag == "a" for descendant in _descendants(element)) 

277 return (allowed := _CONTENT_MODEL.get(element.tag)) is not None and ( 

278 any(child.tag not in allowed for child in element) 

279 or any(text.strip(" \t\n\f\r") for text in (element.text, *(child.tail for child in element))) 

280 ) 

281 

282 

283def _descendants(element: _TreeNode) -> Iterator[_TreeNode]: 

284 for child in element: 

285 yield child 

286 yield from _descendants(child) 

287 

288 

289def _record(tag: str, element: _TreeNode, children: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

290 if tag in {"pre", "code"}: 

291 # code holds literal text only (CommonMark 4.5, 6.1), so markup inside it means nothing and its text is read 

292 # off the tree 

293 return ((tag, (), text),) if (text := _code_text(element, block=tag == "pre")) or tag == "pre" else () 

294 if tag in _CONTENT_REQUIRED and not children: 

295 return (_BOUNDARY,) if tag in _BLOCKS else () 

296 if tag in _EMPHASIS: 

297 return _emphasis(tag, children) 

298 if tag in {"blockquote", "li"}: 

299 # quote and item content is a flow of blocks; a tight item's single paragraph renders bare (CommonMark 5.1-5.3), 

300 # so items compare with every inline run wrapped 

301 return ((tag, (), _flow(children)),) 

302 if tag == "a": 

303 # link text holds inline content only (CommonMark 6.3), so its blocks flatten onto its line 

304 return _link(_attributes(tag, element.attrib), _line(children, inline=True, breaks=True)) 

305 if tag in _HEADINGS or ( 

306 tag == "td" and any(child[0] not in _FLOW_INLINE or _holds_block(child) for child in children) 

307 ): 

308 # a heading is one line of inline content (CommonMark 4.2), so its blocks and breaks flatten onto it as spaces; 

309 # a table cell holds inline content too (GFM 4.10), but a raw <br> keeps its break there 

310 children = _line(children, inline=False, breaks=tag == "td") 

311 return ((tag, _attributes(tag, element.attrib), children),) 

312 

313 

314def _line(records: tuple[_Meaning, ...], *, inline: bool, breaks: bool) -> tuple[_Meaning, ...]: 

315 # blocks flatten onto one line as spaces, and so do hard breaks where the line cannot hold one; emphasis and links 

316 # keep their place on the line with their own content flattened 

317 space: Final[_Meaning] = ("#text", (("value", " "),), ()) 

318 flat: Final[list[_Meaning]] = [] 

319 for record in records: 

320 if record == _BOUNDARY or (record[0] == "br" and not breaks): 

321 flat.append(space) 

322 elif record[0] in _EMPHASIS: 

323 flat.extend(_emphasis(record[0], _line(record[2], inline=True, breaks=breaks))) 

324 elif record[0] == "a": 

325 flat.extend(_link(record[1], _line(record[2], inline=True, breaks=breaks))) 

326 elif record[0] in _FLOW_INLINE: 

327 flat.append(record) 

328 elif record[0] == "pre": 

329 # the inline form of a code block is a code span (CommonMark 6.1), which reads its line endings as spaces 

330 text = re.sub(r"[ \t\n\f\r]+", " ", record[2][0][1][0][1]) if record[2] else "" 

331 flat.extend((space, *((("code", (), (("#text", (("value", text),), ()),)),) if text else ()), space)) 

332 else: 

333 flat.extend((space, *_line(record[2], inline=False, breaks=breaks), space)) 

334 return _spacing(_merge(flat), preserve=False, inline=inline) 

335 

336 

337def _emphasis(tag: str, children: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

338 if all(record[0] == "#text" and not record[1][0][1].strip(" \t\n\f\r") for record in children): 

339 return children # a delimiter run cannot wrap whitespace alone (CommonMark 6.2) 

340 before, inner, after = _edges(children) 

341 return (*before, *(((tag, (), inner),) if inner else ()), *after) 

342 

343 

344def _link(attributes: tuple[tuple[str, str], ...], children: tuple[_Meaning, ...]) -> tuple[_Meaning, ...]: 

345 before, inner, after = _edges(children) 

346 return (*before, ("a", attributes, inner), *after) 

347 

348 

349def _edges(children: tuple[_Meaning, ...]) -> tuple[list[_Meaning], tuple[_Meaning, ...], list[_Meaning]]: 

350 # emphasis and link text do not style a hard break, and spaces at their edge collapse with the ones outside (CSS 

351 # Text 3, 4.1.1), so both render the same outside; a delimiter run touching whitespace cannot open or close 

352 # (CommonMark 6.2), so they sit outside 

353 inner: Final[list[_Meaning]] = list(children) 

354 before: Final[list[_Meaning]] = [] 

355 after: Final[list[_Meaning]] = [] 

356 _peel(inner, before, 0, breaks=True) 

357 _peel(inner, after, -1, breaks=True) 

358 return before, tuple(inner), after 

359 

360 

361def _peel(inner: list[_Meaning], outside: list[_Meaning], edge: int, *, breaks: bool) -> None: 

362 while inner and ( 

363 (breaks and inner[edge][0] == "br") 

364 or (inner[edge][0] == "#text" and inner[edge][1][0][1][edge:][:1] in _SPACES) 

365 ): 

366 text: str = inner[edge][1][0][1] if inner[edge][0] == "#text" else "" 

367 kept: str = text.lstrip(" \t\n\f\r") if edge == 0 else text.rstrip(" \t\n\f\r") 

368 if not kept: 

369 outside.insert(0 if edge else len(outside), inner.pop(edge)) 

370 continue 

371 space: _Meaning = ("#text", (("value", text[: len(text) - len(kept)] if edge == 0 else text[len(kept) :]),), ()) 

372 outside.insert(0 if edge else len(outside), space) 

373 inner[edge] = ("#text", (("value", kept),), ()) 

374 

375 

376def _code_text(element: _TreeNode, *, block: bool) -> tuple[_Meaning, ...]: 

377 # a block keeps its line breaks and drops the newline fenced content always ends in, which a browser does not 

378 # draw; a span reads its line breaks as spaces (CommonMark 6.1) and collapses spaces as the normal flow does 

379 # (CSS Text 3, 4.1.1) 

380 text: Final = _literal(element).removesuffix("\n") if block else re.sub(r"[ \t\n\f\r]+", " ", _literal(element)) 

381 return (("#text", (("value", text),), ()),) if text else () 

382 

383 

384def _literal(element: _TreeNode) -> str: 

385 # a block starts and ends a line even when it holds no text, so its edge breaks a line that already holds text 

386 text = "" 

387 boundary = False 

388 for part in _literal_parts(element): 

389 if part is None: 

390 boundary = True 

391 elif part: 

392 text += "\n" if boundary and text and not text.endswith("\n") else "" 

393 text += part 

394 boundary = False 

395 return text 

396 

397 

398def _literal_parts(element: _TreeNode) -> Iterator[str | None]: 

399 yield element.text 

400 for child in element: 

401 if child.tag == "br": 

402 yield "\n" 

403 elif child.tag not in _HIDDEN: 

404 edge: tuple[None, ...] = () if child.tag in _LITERAL_INLINE else (None,) 

405 yield from edge 

406 yield from _literal_parts(child) 

407 yield from edge 

408 yield child.tail 

409 

410 

411def _attributes(tag: str, attributes: dict[str, str]) -> tuple[tuple[str, str], ...]: 

412 # an empty title gives no advisory information (WHATWG 3.2.6.1), the same as none 

413 # URL navigation and Markdown destinations both discard tabs and line breaks. 

414 semantic: Final = { 

415 name: re.sub(r"[\t\n\r]", "", value) if name in {"href", "src"} else value 

416 for name, value in attributes.items() 

417 if name in _SEMANTIC_ATTRIBUTES.get(tag, ()) and (value or name != "title") 

418 } 

419 # image syntax always writes both attributes (CommonMark 6.4) 

420 return tuple(sorted((({"src": "", "alt": ""} if tag == "img" else {}) | semantic).items())) 

421 

422 

423def _text(text: str, *, preserve: bool) -> tuple[_Meaning, ...]: 

424 if not text: 

425 return () 

426 if not preserve: 

427 text = re.sub(r"[ \t\n\f\r]+", " ", text) 

428 return (("#text", (("value", text),), ()),) 

429 

430 

431def markdown_source_seeds(*, html: bool = False) -> list[str]: 

432 """Force named productions to make corpus coverage reproducible.""" 

433 return [ 

434 case.data.decode("utf-8") 

435 for case in generation_sweep(markdown_grammar(html=html), budget=GenerationBudget(64, 256)) 

436 ] 

437 

438 

439def markdown_controls(*, html: bool = False) -> dict[str, bool]: 

440 """Stable losses distinguish first-conversion and repeated-pass checks.""" 

441 check: Final = markdown_html_check if html else markdown_source_check 

442 sources: Final = ( 

443 ("<p><strong>x</strong></p>", '<p><a href="/target">x</a></p>', "<p>x</p>") 

444 if html 

445 else ("**x**", "[x](/target)", "x") 

446 ) 

447 return { 

448 "lost emphasis": check(sources[0], lambda node: node.text) is not None, 

449 "changed destination": check(sources[1], lambda node: node.to_markdown().replace("/target", "/changed")) 

450 is not None, 

451 "empty conversion": check(sources[2], lambda _node: "") is not None, 

452 } 

453 

454 

455@dataclass(slots=True) 

456class _TreeNode: 

457 """Mirror the element/text/tail shape the meaning walk reads, built from turbohtml's own tree.""" 

458 

459 tag: str 

460 attrib: dict[str, str] 

461 text: str = "" 

462 tail: str = "" 

463 children: list[_TreeNode] = field(default_factory=list) 

464 

465 def __iter__(self) -> Iterator[_TreeNode]: 

466 return iter(self.children) 

467 

468 

469class MarkdownProfileError(ValueError): 

470 """Keep unsupported HTML semantics outside the Markdown conversion profile.""" 

471 

472 

473class _FaithfulMarkdown(MarkdownIt): 

474 """Keep link spelling and policy independent of the reader's URL normalization.""" 

475 

476 @override 

477 def normalizeLink(self, url: str) -> str: 

478 return url 

479 

480 @override 

481 def validateLink(self, url: str) -> bool: 

482 return True 

483 

484 

485def _raw_grammar() -> Grammar: 

486 productions: Final = [ 

487 Production("markdown:root", "root", (Reference("definition", 0), Reference("blocks", 1)), 1, _COMMONMARK), 

488 Production( 

489 "markdown:definition", 

490 "definition", 

491 (b"[", Identifier("reference", definition=True), b"]: /target\n\n"), 

492 0, 

493 _COMMONMARK, 

494 ), 

495 Production("markdown:single", "blocks", (Reference("block", 0),), 0, _COMMONMARK), 

496 Production( 

497 "markdown:siblings", 

498 "blocks", 

499 (Reference("block", 0), b"\n\n---\n\n", Reference("blocks", 0)), 

500 1, 

501 _COMMONMARK, 

502 ), 

503 Production( 

504 "markdown:reference", 

505 "block", 

506 (b"[", Identifier("reference", definition=False), b"]\n"), 

507 4, 

508 _COMMONMARK, 

509 3, 

510 ), 

511 ] 

512 productions.extend( 

513 Production( 

514 f"markdown:{name}", 

515 "block", 

516 (source.encode("utf-8"),), 

517 nodes, 

518 _GFM if name in {"table", "strike"} else _COMMONMARK, 

519 height, 

520 ) 

521 for name, source, nodes, height in _RAW 

522 ) 

523 return compile_grammar(productions, "root") 

524 

525 

526def _html_grammar() -> Grammar: 

527 productions: Final = [ 

528 Production( 

529 "markdown-html:root", 

530 "root", 

531 ( 

532 b'<div id="', 

533 Identifier("target", definition=True), 

534 b'">', 

535 Reference("blocks", 2), 

536 b'<p><a href="#', 

537 Identifier("target", definition=False), 

538 b'">target</a></p></div>', 

539 ), 

540 5, 

541 _COMMONMARK, 

542 4, 

543 ), 

544 Production("markdown-html:single", "blocks", (Reference("block", 0),), 0, _COMMONMARK), 

545 Production( 

546 "markdown-html:siblings", 

547 "blocks", 

548 (Reference("block", 0), b"<hr>", Reference("blocks", 0)), 

549 1, 

550 _COMMONMARK, 

551 ), 

552 ] 

553 productions.extend( 

554 Production( 

555 f"markdown-html:{name}", 

556 "block", 

557 (source.encode("utf-8"),), 

558 nodes, 

559 _GFM if name in {"table", "s", "del"} else _COMMONMARK, 

560 height, 

561 ) 

562 for name, source, nodes, height in _HTML 

563 ) 

564 return compile_grammar(productions, "root") 

565 

566 

567_COMMONMARK: Final = "https://spec.commonmark.org/0.31.2/" 

568_GFM: Final = "https://github.github.com/gfm/" 

569MARKDOWN_READER_SOURCE: Final = ( 

570 "https://github.com/executablebooks/markdown-it-py/tree/36c5f547144df2d01970a5792d68c71a3380b227" 

571) 

572MARKDOWN_RULES: Final = ( 

573 "table", 

574 "code", 

575 "fence", 

576 "blockquote", 

577 "hr", 

578 "list", 

579 "reference", 

580 "html_block", 

581 "heading", 

582 "lheading", 

583 "paragraph", 

584 "text", 

585 "linkify-disabled", 

586 "newline", 

587 "escape", 

588 "backticks", 

589 "strikethrough", 

590 "emphasis", 

591 "link", 

592 "image", 

593 "autolink", 

594 "html_inline", 

595 "entity", 

596) 

597_RAW: Final = ( 

598 ("heading1", "# x\n", 3, 2), 

599 ("heading2", "## x\n", 3, 2), 

600 ("heading3", "### x\n", 3, 2), 

601 ("heading4", "#### x\n", 3, 2), 

602 ("heading5", "##### x\n", 3, 2), 

603 ("heading6", "###### x\n", 3, 2), 

604 ("setext1", "x\n===\n", 3, 2), 

605 ("setext2", "x\n---\n", 3, 2), 

606 ("thematic", "---\n", 1, 0), 

607 ("indented", " x\n", 1, 0), 

608 ("fence-backtick", "```\nx\n```\n", 1, 0), 

609 ("fence-tilde", "~~~\nx\n~~~\n", 1, 0), 

610 ("html-comment", "<!-- x -->\n", 1, 0), 

611 ("html-pi", "<?x?>\n", 1, 0), 

612 ("html-declaration", "<!DOCTYPE html>\n", 1, 0), 

613 ("html-cdata", "<![CDATA[x]]>\n", 1, 0), 

614 ("html-script", "<script>x</script>\n", 1, 0), 

615 ("html-block", "<p>x</p>\n", 1, 0), 

616 ("html-span", "<span>x</span>\n", 5, 2), 

617 ("html-tag", '<img src="/image" alt="x">\n', 1, 0), 

618 ("quote", "> x\n", 4, 3), 

619 ("bullet", "- x\n", 5, 4), 

620 ("ordered", "1. x\n", 5, 4), 

621 ("loose", "- x\n\n- y\n", 9, 4), 

622 ("nested", "- x\n - y\n", 10, 6), 

623 ("table", "| a |\n|---|\n| b |\n", 11, 5), 

624 ("paragraph", "x\n", 3, 2), 

625 ("softbreak", "x\ny\n", 5, 2), 

626 ("hardbreak", "x \ny\n", 5, 2), 

627 ("em-star", "*x*\n", 4, 3), 

628 ("em-underscore", "_x_\n", 4, 3),