Coverage for /pythoncovmergedfiles/medio/medio/src/turbohtml/tools/fuzz/round_trip_oracles.py: 46%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Round-trip and cross-entry-point oracles: the ``--mode round-trip`` lane of ``tools/fuzz/fuzz.py``.
4A printer or minifier bug usually returns wrong text without crashing, so each oracle checks a property every output
5must keep. A printed language parses back to the tree it came from and prints again unchanged (HTML, XML, CSS, JS,
6style declarations, Markdown through markdown-it-py). A minifier keeps what the input means: JS keeps its free
7identifiers and static strings, read by acorn and eslint-scope; CSS keeps every element's computed style and every
8selector's match set. Entry points that answer one question agree, and source spans stay ordered and shift exactly
9with a prefix.
11Every oracle carries negative controls that feed it a deliberately broken printer, minifier or entry point and must
12fire, and a vacuity floor on the share of in-scope cases it compared, so an oracle that stops detecting or stops
13comparing fails the run (DESIGN-v2 §4d). A finding lands in ``--crash-dir`` as ``crash-<sha256>`` with a JSON sidecar;
14the log names only the oracle and the hash, because CI logs on a public repository are public.
15"""
17from __future__ import annotations
19import argparse
20import hashlib
21import json
22import math
23import os
24import random
25import re
26import shutil
27import subprocess
28import sys
29import time
30import unicodedata
31from collections import Counter
32from dataclasses import dataclass, field
33from functools import cache, partial
34from itertools import pairwise, starmap
35from pathlib import Path
36from typing import TYPE_CHECKING, Final, Literal
37from urllib.parse import urlsplit
39import tinycss2
40from fuzz.css_custom_oracles import (
41 UnsupportedCssCustomCaseError,
42 css_custom_check,
43 css_custom_controls,
44 css_custom_generate,
45 css_custom_seeds,
46)
47from fuzz.css_structure_generators import css_generate, css_source_generate, css_source_seeds
48from fuzz.dom_lifecycle import (
49 UnsupportedDomLifecycleError,
50 dom_lifecycle_check,
51 dom_lifecycle_controls,
52 dom_lifecycle_generate,
53 dom_lifecycle_seeds,
54)
55from fuzz.dom_program import (
56 UnsupportedDomProgramError,
57 dom_program_check,
58 dom_program_controls,
59 dom_program_generate,
60 dom_program_seeds,
61)
62from fuzz.html_foreign_oracles import (
63 UnsupportedHtmlForeignCaseError,
64 html_foreign_check,
65 html_foreign_controls,
66 html_foreign_generate,
67 html_foreign_seeds,
68)
69from fuzz.html_grammar_oracles import (
70 UnsupportedHtmlSiblingCaseError,
71 html_sibling_check,
72 html_sibling_controls,
73 html_sibling_generate,
74 html_sibling_seeds,
75)
76from fuzz.html_list_oracles import (
77 UnsupportedHtmlListCaseError,
78 html_list_check,
79 html_list_controls,
80 html_list_generate,
81 html_list_seeds,
82)
83from fuzz.html_structure_generators import html_generate
84from fuzz.html_table_oracles import (
85 UnsupportedHtmlTableCaseError,
86 html_table_check,
87 html_table_controls,
88 html_table_generate,
89 html_table_seeds,
90)
91from fuzz.idna_nfc_oracles import (
92 UnsupportedIdnaNfcCaseError,
93 idna_nfc_check,
94 idna_nfc_controls,
95 idna_nfc_generate,
96 idna_nfc_seeds,
97)
98from fuzz.iterator_oracles import (
99 UnsupportedIteratorCaseError,
100 iterator_sequence_check,
101 iterator_sequence_controls,
102 iterator_sequence_generate,
103 iterator_sequence_seeds,
104)
105from fuzz.markdown_structure_generators import (
106 MarkdownProfileError,
107 markdown_controls,
108 markdown_generate,
109 markdown_html_check,
110 markdown_html_meaning,
111 markdown_source_check,
112 markdown_source_seeds,
113)
114from fuzz.observer_oracles import (
115 UnsupportedObserverCaseError,
116 observer_sequence_check,
117 observer_sequence_controls,
118 observer_sequence_generate,
119 observer_sequence_seeds,
120)
121from fuzz.parser_byte_oracles import (
122 UnsupportedParserCaseError,
123 parser_bytes_check,
124 parser_bytes_controls,
125 parser_bytes_generate,
126 parser_bytes_seeds,
127)
128from fuzz.reduce import minimize
129from fuzz.xml_grammar_oracles import (
130 UnsupportedXmlLiteralCaseError,
131 xml_literal_check,
132 xml_literal_controls,
133 xml_literal_generate,
134 xml_literal_seeds,
135)
136from fuzz.xml_island_oracles import (
137 UnsupportedXmlIslandCaseError,
138 xml_island_check,
139 xml_island_controls,
140 xml_island_generate,
141 xml_island_seeds,
142)
143from fuzz.xml_structure_generators import (
144 xml_document_check,
145 xml_source_controls,
146 xml_source_generate,
147 xml_source_seeds,
148)
149from markdown_it import MarkdownIt
150from tinycss2.ast import AtRule, ParseError, QualifiedRule
151from typing_extensions import override
153from turbohtml import (
154 CData,
155 Comment,
156 Doctype,
157 Document,
158 DocumentFragment,
159 Element,
160 Html,
161 HTMLParseError,
162 IncrementalParser,
163 Namespace,
164 Node,
165 ProcessingInstruction,
166 SelectorSyntaxError,
167 SourceLocation,
168 SourceSpan,
169 Text,
170 XPath,
171 parse,
172 parse_fragment,
173 parse_xml,
174)
175from turbohtml.clean import JSMinify, minify_css, minify_js
176from turbohtml.convert import ExpressionError, css_to_xpath
177from turbohtml.cssom import StyleDeclaration, StyleSheet, computed_style
178from turbohtml.detect import EncodingDetector, EncodingMatch, detect
179from turbohtml.extract import clean_url, normalize_url
180from turbohtml.query import Matcher
181from turbohtml.query import compile as compile_selector
183if TYPE_CHECKING:
184 from collections.abc import Callable, Iterable, Sequence
185 from urllib.parse import SplitResult
187__all__ = [
188 "MAX_INPUT",
189 "ORACLES",
190 "UNBOUNDED",
191 "Floor",
192 "JsNames",
193 "Oracle",
194 "OutOfScopeError",
195 "clean_url_check",
196 "css_semantics_check",
197 "encoding_decode_check",
198 "encoding_stream_check",
199 "fixpoint_check",
200 "html_check",
201 "idna_host_check",
202 "js_names_check",
203 "main",
204 "markdown_check",
205 "normalize_url_check",
206 "resolve_links_check",
207 "selector_entry_check",
208 "span_check",
209 "style_check",
210 "url_reparse_check",
211 "xml_check",
212 "xpath_entry_check",
213]
215_ROOT: Final[Path] = Path(__file__).resolve().parents[2]
216_NODE_DIR: Final[Path] = _ROOT / "tools" / "bench" / "node"
217_MINIMIZE_BUDGET: Final = 600
218# TH_MAX_TREE_DEPTH (src/turbohtml/_c/dom/tree.h): past 512 open elements a start tag is inserted but not pushed, so
219# further tags become siblings and reparsing the flattened output nests them differently
220_MAX_TREE_DEPTH: Final = 512
221# the CSS minifier caps nesting at 100 levels and drops a deeper block (docs/changelog.rst, 1.13.0)
222_CSS_MAX_NESTING: Final = 100
223# tree.h raises this RecursionError for a walk past its depth budget; the one documented depth error
224_DEPTH_ERROR: Final = re.compile(r"does not support trees nested \d+ levels or deeper")
225# Script text where "<!--" opens and "<script" plus a tag end follows before any "-->" breaks the script content
226# restrictions (https://html.spec.whatwg.org/multipage/scripting.html#restrictions-for-contents-of-script-elements):
227# reparsing its serialization lands in the double-escaped state, which swallows the closing "</script>". The standard
228# warns that such parser-made trees need not survive serialize and reparse
229# (https://html.spec.whatwg.org/multipage/parsing.html#serialising-html-fragments).
230_SCRIPT_DOUBLE_ESCAPE: Final = re.compile(r"<!--(?:(?!-->).)*?<script[\t\n\f />]", re.IGNORECASE | re.DOTALL)
231# rust-cssparser dropped its idempotence assert because dtoa takes two passes to converge (9999995e-45 -> 10e-39 ->
232# 1e-38), so a numeric rewrite may settle on the second pass:
233# https://github.com/servo/rust-cssparser/blob/106021c7bc55ed2cac83f590c0d356fa530b7061/fuzz/fuzz_targets/cssparser.rs
234_NUMBER: Final = re.compile(r"(?<![\w$#\\.])[+-]?(?:\d[\d_]*\.?[\d_]*|\.\d[\d_]*)(?:[eE][+-]?\d+)?n?")
235_JS_OFF: Final = JSMinify(mangle=False, fold=False)
236_MAX_RENESTS: Final = 5
237# cap one case's input: generators stay under a few KiB, but a mutated .dat seed can grow, and to_markdown/to_text
238# amplify nested input about 100-240x (tox-dev/turbohtml#1028), so an input this long could print hundreds of MiB
239MAX_INPUT: Final = 64 * 1024
240# parse() allocates without bound on a selectedcontent nested in its own option (findings/01); the generators never
241# emit the tag, so skipping it keeps a machine-safe run while the defect is tracked on its own
242UNBOUNDED: Final = re.compile(r"selectedcontent", re.IGNORECASE)
243_HEADINGS: Final = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
244_SELF_NESTING: Final[dict[str, frozenset[str]]] = {
245 **{tag: frozenset({tag}) for tag in ("a", "nobr", "p", "button", "select", "form", "table", "option", "li")},
246 **dict.fromkeys(("dd", "dt"), frozenset({"dd", "dt"})),
247 **dict.fromkeys(_HEADINGS, _HEADINGS),
248 **dict.fromkeys(("rb", "rt", "rp", "rtc"), frozenset({"rb", "rt", "rp", "rtc"})),
249}
250_MARKDOWN_BLOCKS: Final = frozenset({
251 "address", "article", "aside", "blockquote", "dd", "details", "div", "dl", "dt", "fieldset", "figure", "footer",
252 "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section",
253 "table", "ul",
254}) # fmt: skip
255# the elements whose Markdown form holds only inline content: phrasing elements and GFM table cells
256_MARKDOWN_HOSTS: Final = frozenset({
257 "a", "abbr", "b", "bdi", "bdo", "cite", "code", "del", "dfn", "em", "i", "ins", "kbd", "mark", "q", "s", "samp",
258 "small", "span", "strike", "strong", "sub", "sup", "td", "th", "u", "var",
259}) # fmt: skip
262class _FaithfulMarkdown(MarkdownIt):
263 """markdown-it percent-encodes link destinations and drops ``javascript:`` links, reader policy this undoes."""
265 @override
266 def normalizeLink(self, url: str) -> str:
267 return url
269 @override
270 def validateLink(self, url: str) -> bool:
271 return True
274_MARKDOWN: Final = _FaithfulMarkdown("gfm-like", {"linkify": False})
275# markdown-it-py 4.2.0 strips a paragraph with str.strip (rules_block/paragraph.py), so a paragraph of only U+00A0
276# reads as an empty one, where CommonMark strips only spaces and tabs; an empty paragraph shows nothing either way
277_EMPTY_PARAGRAPH: Final = re.compile(r"<p></p>\n")
280class OutOfScopeError(Exception):
281 """The case is outside the oracle's scope; it counts against the oracle's vacuity floor."""
284@dataclass(frozen=True)
285class Floor:
286 """
287 The least an oracle must compare for a run to mean anything.
289 Modeled on meriyah's ``MINIMUM_COMPARED_RATIO``: ``ratio`` bounds compared cases over compared plus skipped ones,
290 ``count`` bounds the compared cases themselves.
291 """
293 count: int
294 ratio: float
297@dataclass(frozen=True)
298class Oracle:
299 """
300 One wrong-output oracle.
302 ``check`` returns None when the case holds, a shape-only description when it breaks, and raises
303 :class:`OutOfScopeError` when the case is out of scope. ``controls`` maps each negative control to whether it fired.
304 A case that is a JSON object names the string ``fields`` to shrink one at a time; any other case shrinks whole.
305 """
307 check: Callable[[str], str | None]
308 generate: Callable[[random.Random], str]
309 seeds: Callable[[], list[str]]
310 controls: Callable[[], dict[str, bool]]
311 floor: Floor
312 fields: tuple[str, ...] = ()
313 syntax: Literal["html", "css", "js"] | None = None
316def main(argv: Sequence[str] | None = None) -> int:
317 """Return 0 when every oracle holds, 1 on a finding, 2 when a control stayed silent or a run compared too little."""
318 parser = argparse.ArgumentParser(description=__doc__)
319 parser.add_argument("--minutes", type=float, default=1.0, help="generation budget after the seed pass")
320 parser.add_argument(
321 "--rng-seed",
322 type=int,
323 default=int(os.environ.get("FUZZ_RNG_SEED", "0")),
324 help="seed of the generated cases (default: $FUZZ_RNG_SEED, else 0)",
325 )
326 parser.add_argument("--crash-dir", type=Path, default=_ROOT / ".fuzz-crashes", help="where findings land")
327 parser.add_argument("--oracle", action="append", choices=sorted(ORACLES), help="run only these (repeatable)")
328 args = parser.parse_args(argv)
330 names = args.oracle or sorted(ORACLES)
331 if blind := [
332 f"{name}/{control}" for name in names for control, fired in ORACLES[name].controls().items() if not fired
333 ]:
334 print(f"BLIND ORACLE: the negative control did not fire for {', '.join(blind)}", file=sys.stderr)
335 return 2
336 args.crash_dir.mkdir(parents=True, exist_ok=True)
337 run = _Run(args.crash_dir)
338 for name in names:
339 for seed in ORACLES[name].seeds():
340 run.case(name, seed)
341 rng = random.Random(args.rng_seed)
342 deadline = time.monotonic() + args.minutes * 60
343 while time.monotonic() < deadline:
344 name = rng.choice(names)
345 run.case(name, ORACLES[name].generate(rng))
346 for name in names:
347 print(f"{name}: {dict(run.stats[name])}")
348 findings = run.report(args.rng_seed)
349 if vacuous := [name for name in names if not _meets(ORACLES[name].floor, run.stats[name])]:
350 print(f"VACUOUS RUN: {', '.join(vacuous)} compared below the floor", file=sys.stderr)
351 return 2
352 print(f"{findings} finding(s) written to {args.crash_dir}")
353 return int(findings > 0)
356def _meets(floor: Floor, stats: Counter[str]) -> bool:
357 compared = stats["compared"]
358 return compared >= floor.count and compared >= floor.ratio * (compared + stats["skipped"])
361@dataclass
362class _Found:
363 oracle: str
364 detail: str
365 text: str
366 hits: int = 1
369@dataclass
370class _Run:
371 crash_dir: Path
372 stats: dict[str, Counter[str]] = field(default_factory=lambda: {name: Counter() for name in ORACLES})
373 found: dict[tuple[str, str], _Found] = field(default_factory=dict)
375 def case(self, name: str, text: str) -> None:
376 if len(text) > MAX_INPUT or UNBOUNDED.search(text):
377 self.stats[name]["skipped"] += 1
378 return
379 # an ASan abort kills the process, so the input it died on must already be on disk
380 (self.crash_dir / "current_input").write_text(text, encoding="utf-8", errors="surrogatepass")
381 try:
382 detail = _guarded(ORACLES[name].check, text)
383 except OutOfScopeError:
384 self.stats[name]["skipped"] += 1
385 return
386 self.stats[name]["compared"] += 1
387 if detail is not None:
388 if (key := (name, detail)) in self.found:
389 self.found[key].hits += 1
390 else:
391 self.found[key] = _Found(name, detail, text)
393 def report(self, rng_seed: int) -> int:
394 for found in self.found.values():
395 oracle = ORACLES[found.oracle]
396 text = _minimize(found.text, partial(_reproduces, oracle.check, found.detail), oracle.fields, oracle.syntax)
397 data = text.encode("utf-8", "surrogatepass")
398 digest = hashlib.sha256(data).hexdigest()
399 (self.crash_dir / f"crash-{digest}").write_bytes(data)
400 sidecar = {"oracle": found.oracle, "detail": found.detail, "hits": found.hits, "rng_seed": rng_seed}
401 (self.crash_dir / f"crash-{digest}.json").write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8")
402 # only the oracle and a hash reach the log: the input may be an unreported bug (DESIGN-v2 decision D5)
403 print(f"FINDING {found.oracle} sha256={digest}", file=sys.stderr, flush=True)
404 return len(self.found)
407def _guarded(check: Callable[[str], str | None], text: str) -> str | None:
408 """Run one check, turning the documented depth error into a skip and any other escape into a finding."""
409 try:
410 return check(text)
411 except RecursionError as error:
412 if _DEPTH_ERROR.search(str(error)):
413 raise OutOfScopeError from error
414 return "raised RecursionError"
415 except OutOfScopeError:
416 raise
417 except Exception as error: # ruff: ignore[blind-except] # an unexpected exception type from turbohtml is itself the finding
418 return f"raised {type(error).__name__}"
421def _reproduces(check: Callable[[str], str | None], detail: str, text: str) -> bool:
422 try:
423 return _guarded(check, text) == detail
424 except OutOfScopeError:
425 return False
428def _minimize(
429 text: str,
430 reproduces: Callable[[str], bool],
431 fields: Sequence[str],
432 syntax: Literal["html", "css", "js"] | None,
433) -> str:
434 remaining = _MINIMIZE_BUDGET
436 def check(candidate: str) -> bool:
437 nonlocal remaining
438 remaining -= 1
439 return reproduces(candidate)
441 def shrink(source: str, predicate: Callable[[str], bool], language: Literal["html", "css", "js"] | None) -> str:
442 if language is not None:
443 source = minimize(source, predicate, language, remaining)
444 return _ddmin(source, predicate, remaining)
446 if not fields:
447 return shrink(text, check, syntax)
448 payload = json.loads(text)
449 for name in fields:
450 payload[name] = shrink(
451 payload[name],
452 lambda value, name=name: check(json.dumps({**payload, name: value})),
453 "html" if name == "html" else syntax,
454 )
455 return json.dumps(payload)
458def _ddmin(text: str, reproduces: Callable[[str], bool], budget: int) -> str:
459 """Shrink ``text`` while ``reproduces`` holds, with Zeller and Hildebrandt's ddmin under a call budget."""
460 calls = 0
461 granularity = 2
462 while len(text) >= 2 and calls < budget:
463 chunk = max(len(text) // granularity, 1)
464 for start in range(0, len(text), chunk):
465 if calls == budget:
466 return text
467 calls += 1
468 if reproduces(candidate := text[:start] + text[start + chunk :]):
469 text = candidate
470 granularity = max(granularity - 1, 2)
471 break
472 else:
473 if chunk == 1:
474 break
475 granularity = min(granularity * 2, len(text))
476 return text
479def fixpoint_check(text: str, printer: Callable[[str], str], *, numeric: bool) -> str | None:
480 """
481 Require ``printer`` to reach a fixpoint: printing its own output changes nothing.
483 With ``numeric``, a second pass that only rewrites numbers may settle on the third (``f(f(f(x))) == f(f(x))``), the
484 rust-cssparser exception; anything else that moves on the second pass is a finding. A first pass that rejects the
485 input skips it; a later pass that rejects the printer's own output is a finding.
486 """
487 try:
488 once = printer(text)
489 except ValueError as error:
490 raise OutOfScopeError from error
491 try:
492 twice = printer(once)
493 except ValueError:
494 return "rejects its own output"
495 if twice == once:
496 return None
497 if numeric and _NUMBER.sub("0", twice) == _NUMBER.sub("0", once) and printer(twice) == twice:
498 return None
499 return f"not a fixpoint: {_divergence(once, twice)}"
502def _html_foreign_check(case: str) -> str | None:
503 try:
504 return html_foreign_check(case)
505 except UnsupportedHtmlForeignCaseError as error:
506 raise OutOfScopeError from error
509def _html_list_check(case: str) -> str | None:
510 try:
511 return html_list_check(case)
512 except UnsupportedHtmlListCaseError as error:
513 raise OutOfScopeError from error
516def _html_table_check(case: str) -> str | None:
517 try:
518 return html_table_check(case)
519 except UnsupportedHtmlTableCaseError as error:
520 raise OutOfScopeError from error
523def _html_sibling_check(case: str) -> str | None:
524 try:
525 return html_sibling_check(case)
526 except UnsupportedHtmlSiblingCaseError as error:
527 raise OutOfScopeError from error
530def _css_custom_check(case: str) -> str | None:
531 try:
532 return css_custom_check(case)
533 except UnsupportedCssCustomCaseError as error:
534 raise OutOfScopeError from error
537def _observer_sequence_check(case: str) -> str | None:
538 try:
539 return observer_sequence_check(case)
540 except UnsupportedObserverCaseError as error:
541 raise OutOfScopeError from error
544def _dom_lifecycle_check(case: str) -> str | None:
545 try:
546 return dom_lifecycle_check(case)
547 except UnsupportedDomLifecycleError as error:
548 raise OutOfScopeError from error
551def _dom_program_check(case: str) -> str | None:
552 try:
553 return dom_program_check(case)
554 except UnsupportedDomProgramError as error:
555 raise OutOfScopeError from error
558def _iterator_sequence_check(case: str) -> str | None:
559 try:
560 return iterator_sequence_check(case)
561 except UnsupportedIteratorCaseError as error:
562 raise OutOfScopeError from error
565_IDNA_LABELS: Final[dict[str, tuple[str, str]]] = {
566 "umlaut": ("münchen", "münchen"),
567 "sharp-s": ("faß", "faß"),
568 "acute-decomposed": ("a\u0301", "á"),
569 "acute-precomposed": ("á", "á"),
570 "umlaut-decomposed": ("u\u0308", "ü"),
571 "japanese": ("日本語", "日本語"),
572 "greek": ("δοκιμή", "δοκιμή"),
573 "ignored": ("ex\u00adample", "example"),
574 "fullwidth": ("\uff26\uff2f\uff2f", "foo"),
575 "mapped-dot": ("日本語。\uff2a\uff30", "日本語.jp"),
576}
579def idna_host_check(case: str, normalize: Callable[[str], str] = normalize_url) -> str | None:
580 """Pin supported Unicode 16 host mappings through the public URL normalizer."""
581 variant, separator, affix = case.partition("\n")
582 if (mapping := _IDNA_LABELS.get(variant)) is None or not separator or not re.fullmatch(r"[a-z0-9]{0,8}", affix):
583 raise OutOfScopeError(case)
584 source, mapped = mapping
585 mapped = affix + mapped
586 expected: Final = (
587 "https://"
588 + ".".join(
589 label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
590 for label in mapped.split(".")
591 )
592 + ".example/"
593 )
594 try:
595 once = normalize(f"https://{affix}{source}.example/")
596 except ValueError:
597 return "rejects known-valid Unicode host"
598 if (problem := _idna_output_problem(once)) is not None:
599 return problem
600 if once != expected:
601 return "IDNA host differs from expected mapping"
602 try:
603 twice = normalize(once)
604 except ValueError:
605 return "rejects canonical IDNA output"
606 return None if twice == once else "IDNA output is not a fixpoint"
609def _idna_output_problem(once: str) -> str | None:
610 if not once.isascii():
611 return "IDNA output is not ASCII"
612 for label in once.removeprefix("https://").removesuffix("/").split("."):
613 if not label.startswith("xn--"):
614 continue
615 payload = label[4:]
616 try:
617 decoded = payload.encode("ascii").decode("punycode")
618 except UnicodeError:
619 return "IDNA output has undecodable Punycode"
620 if not unicodedata.is_normalized("NFC", decoded):
621 return "IDNA decoded label is not NFC"
622 if decoded.encode("punycode").decode("ascii") != payload:
623 return "IDNA Punycode is not canonical"
624 return None
627def _idna_nfc_check(case: str) -> str | None:
628 try:
629 return idna_nfc_check(case)
630 except UnsupportedIdnaNfcCaseError as error:
631 raise OutOfScopeError(str(error)) from error
634def normalize_url_check(text: str, normalize: Callable[[str], str] = normalize_url) -> str | None:
635 """URL normalization promises a stable representation after one successful pass."""
636 return fixpoint_check(text, normalize, numeric=False)
639def url_reparse_check(text: str, normalize: Callable[[str], str] = normalize_url) -> str | None:
640 """Preserve delimiter presence when checking serialization through a reference split."""
641 try:
642 normalized: Final = normalize(text)
643 parts: Final = urlsplit(normalized)
644 except ValueError as error:
645 raise OutOfScopeError(str(error)) from error
646 recomposed: Final = _url_recompose(parts, normalized)
647 if recomposed != normalized:
648 return f"split/recompose changes normalized URL: {_divergence(normalized, recomposed)}"
649 try:
650 reparsed: Final = normalize(recomposed)
651 except ValueError:
652 return "rejects recomposed output"
653 return None if reparsed == normalized else f"reparse changes normalized URL: {_divergence(normalized, reparsed)}"
656def _url_recompose(parts: SplitResult, text: str) -> str:
657 body: Final = text[len(parts.scheme) + 1 :] if parts.scheme else text
658 return "".join((
659 f"{parts.scheme}:" if parts.scheme else "",
660 f"//{parts.netloc}" if body.startswith("//") else "",
661 parts.path,
662 f"?{parts.query}" if "?" in body.partition("#")[0] else "",
663 f"#{parts.fragment}" if "#" in body else "",
664 ))
667def clean_url_check(text: str, clean: Callable[[str], str | None] = clean_url) -> str | None:
668 """Skip rejected inputs; flag rejection of a cleaned URL."""
669 try:
670 once = clean(text)
671 except ValueError as error:
672 raise OutOfScopeError from error
673 if once is None:
674 raise OutOfScopeError(text)
675 try:
676 twice = clean(once)
677 except ValueError:
678 return "rejects its own output"
679 if twice is None:
680 return "rejects its own output"
681 return None if twice == once else f"not a fixpoint: {_divergence(once, twice)}"
684_ENCODING_FORMATS: Final[dict[str, tuple[str, bytes, str, str]]] = {
685 "utf-8-bom": ("utf-8", b"\xef\xbb\xbf", "UTF-8-SIG", ""),
686 "utf-16le-bom": ("utf-16-le", b"\xff\xfe", "UTF-16LE", ""),
687 "utf-16be-bom": ("utf-16-be", b"\xfe\xff", "UTF-16BE", ""),
688 "utf-16-meta-remap": ("utf-8", b"", "UTF-8", "<meta charset=utf-16>"),
689 "utf-16le-meta-remap": ("utf-8", b"", "UTF-8", "<meta charset=utf-16le>"),
690 "utf-16be-meta-remap": ("utf-8", b"", "UTF-8", "<meta charset=utf-16be>"),
691 "utf-8-meta": ("utf-8", b"", "UTF-8", "<meta charset=utf-8>"),
692 "windows-1252-meta": ("cp1252", b"", "windows-1252", "<meta charset=windows-1252>"),
693 "latin1-meta-alias": ("cp1252", b"", "windows-1252", "<meta charset=latin1>"),
694 "iso-8859-1-meta-alias": ("cp1252", b"", "windows-1252", "<meta charset=iso-8859-1>"),
695 "us-ascii-meta-alias": ("cp1252", b"", "windows-1252", "<meta charset=us-ascii>"),
696 "utf-8-bom-conflicting-meta": ("utf-8", b"\xef\xbb\xbf", "UTF-8-SIG", "<meta charset=windows-1252>"),
697 "utf-16le-bom-conflicting-meta": ("utf-16-le", b"\xff\xfe", "UTF-16LE", "<meta charset=utf-8>"),
698 "utf-16be-bom-conflicting-meta": ("utf-16-be", b"\xfe\xff", "UTF-16BE", "<meta charset=utf-8>"),
699 "utf-8-pragma-first": ("utf-8", b"", "UTF-8", '<meta http-equiv=Content-Type content="text/html; charset=utf-8">'),
700 "utf-8-content-first": ("utf-8", b"", "UTF-8", '<meta content="text/html; charset=utf-8" http-equiv=Content-Type>'),
701 "windows-1252-pragma-first": (
702 "cp1252",
703 b"",
704 "windows-1252",
705 '<meta http-equiv=Content-Type content="text/html; charset=windows-1252">',
706 ),
707 "windows-1252-content-first": (
708 "cp1252",
709 b"",
710 "windows-1252",
711 '<meta content="text/html; charset=windows-1252" http-equiv=Content-Type>',
712 ),
713}
716def encoding_stream_check(
717 case: str,
718 one_shot: Callable[[bytes], EncodingMatch] = detect,
719 stream: Callable[[bytes, int], EncodingMatch] | None = None,
720) -> str | None:
721 """Pin expected detection independently of the chunk-boundary comparison."""
722 expected: Final = _encoding_case(case)
723 if one_shot(expected.data) != expected.match:
724 return "one-shot detection differs from expected match"
725 for width in range(1, 6):
726 if (stream or _encoding_stream)(expected.data, width) != expected.match:
727 return "chunked detection differs from expected match"
728 return None
731def encoding_decode_check(
732 case: str,
733 detect_bytes: Callable[[bytes], EncodingMatch] = detect,
734 decode: Callable[[bytes, str], str] = bytes.decode,
735 parse_text: Callable[[bytes], str | None] | None = None,
736) -> str | None:
737 """Require the codec and byte parser to preserve fixture text."""
738 expected: Final = _encoding_case(case)
739 if expected.match.codec is None:
740 raise OutOfScopeError(case)
741 if detect_bytes(expected.data) != expected.match:
742 return "detection differs from expected match"
743 if decode(expected.data, expected.match.codec) != expected.decoded:
744 return "detected codec differs from expected text"
745 if (parse_text or _encoding_text)(expected.data) != expected.text:
746 return "byte parse differs from expected text"
747 return None
750def _encoding_case(case: str) -> _EncodingCase:
751 variant, separator, text = case.partition("\n")
752 if not separator or len(text) > 256 or (text and not text.isprintable()) or any(char in text for char in "<>&"):
753 raise OutOfScopeError(case)
754 if variant == "empty" and not text:
755 return _EncodingCase(b"", EncodingMatch(None, 0.0, None), "", "")
756 if (formatting := _ENCODING_FORMATS.get(variant)) is None:
757 raise OutOfScopeError(case)
758 codec, prefix, label, meta = formatting
759 markup: Final = f"{meta}<p>{text}</p>"
760 try:
761 data: Final = prefix + markup.encode(codec)
762 except UnicodeEncodeError as error:
763 raise OutOfScopeError from error
764 decoded: Final = ("\ufeff" if label in {"UTF-16LE", "UTF-16BE"} else "") + markup
765 return _EncodingCase(data, EncodingMatch(label, 1.0, None, bool(prefix), f"whatwg-{label.lower()}"), decoded, text)
768@dataclass(frozen=True)
769class _EncodingCase:
770 data: bytes
771 match: EncodingMatch
772 decoded: str
773 text: str
776def _encoding_stream(data: bytes, width: int) -> EncodingMatch:
777 detector: Final = EncodingDetector()
778 detector.feed(b"")
779 for start in range(0, len(data), width):
780 detector.feed(data[start : start + width])
781 detector.feed(b"")
782 return detector.close()
785def _encoding_text(data: bytes) -> str:
786 return parse(data).text
789def _divergence(left: str, right: str) -> str:
790 """Name the character classes at the first difference, so findings that differ only in payload share a key."""
791 pairs = enumerate(zip(left, right, strict=False))
792 index = next((position for position, (one, two) in pairs if one != two), min(len(left), len(right)))
793 return f"{_char_class(left[index : index + 1])} vs {_char_class(right[index : index + 1])}"
796def _char_class(char: str) -> str:
797 if not char:
798 return "end"
799 if char.isalpha():
800 return "letter"
801 if char.isdigit():
802 return "digit"
803 if char.isspace():
804 return "space"
805 return repr(char) if char in "<>&\"'=/;:{}()[],.!*#`\\|_-+~$@%^?" else "other"
808def html_check(markup: str, serialize: Callable[[Node], str] = Node.serialize) -> str | None:
809 """
810 Require HTML serialization to keep a tree's content and reach a fixpoint, for a document and a ``<div>`` fragment.
812 The standard warns that parser-made trees need not survive serialize and reparse
813 (https://html.spec.whatwg.org/multipage/parsing.html#serialising-html-fragments): nested ``<a>``, a ``<table>`` in
814 a quirks-mode ``<p>`` and the like re-nest. So the first reparse may move elements but must keep every text run,
815 comment and element with its attributes, and the second serialization must equal the first byte for byte. A tree
816 flattened past the depth cap, script text that enters the double-escaped state, and a ``plaintext`` element (its
817 serialized end tag is text to the PLAINTEXT state, which never exits) skip. A carriage return compares as a line
818 feed: serialization emits it raw and input preprocessing turns it into one, and text holding one skips, since the
819 line feed it becomes can then be the one a ``pre`` drops after its start tag. Each reparse may re-nest further, so a
820 fixpoint must come within five.
821 """
822 for label, tree, reparse in (
823 ("document", parse(markup), parse),
824 ("fragment", parse_fragment(markup), parse_fragment),
825 ):
826 if _reparse_may_differ(tree):
827 raise OutOfScopeError
828 content = _content(tree)
829 printed = _newlines(_print(tree, serialize))
830 for _ in range(_MAX_RENESTS):
831 if (again := _print(reparsed := reparse(printed), serialize)) == printed:
832 break
833 if _content(reparsed) != content:
834 return f"{label} reparse changed content"
835 printed = again
836 else:
837 return f"{label} not a fixpoint after {_MAX_RENESTS} reparses"
838 return None
841def _print(tree: Node, serialize: Callable[[Node], str]) -> str:
842 return serialize(tree) if isinstance(tree, Document) else "".join(serialize(child) for child in tree.children)
845def _reparse_may_differ(root: Node) -> bool:
846 return (
847 _past_depth_cap(root)
848 or any(_SCRIPT_DOUBLE_ESCAPE.search(script.text) for script in root.iter_elements("script"))
849 or any(element.namespace is Namespace.HTML for element in root.iter_elements("plaintext"))
850 or any(isinstance(node, Text) and "\r" in node.data for node in root.descendants)
851 or any(_nests_in_itself(element) for element in root.iter_elements())
852 )
855def _nests_in_itself(element: Element) -> bool:
856 # a start tag of these names closes, re-nests or drops an open element of its group (WHATWG 13.2.6.4.7 "in
857 # body": the adoption agency for a, nobr; "close a p element"; li, dd, dt, option, headings, button, select,
858 # form's element pointer, table, ruby), so a parser-made tree nesting one inside its group cannot be re-read
859 group = _SELF_NESTING.get(element.tag)
860 return (
861 group is not None
862 and element.namespace is Namespace.HTML
863 and any(getattr(ancestor, "tag", None) in group for ancestor in element.ancestors)
864 )
867def _content(root: Node) -> tuple[str, list[str], set[tuple[str, str, tuple[tuple[str, str], ...]]]]:
868 """
869 Reduce a tree to what re-nesting keeps: its text in order, its comments in order, its distinct elements.
871 A bare ``p`` or ``br`` does not count: re-nesting can strand a ``</p>`` or meet a ``</br>``, and the tree builder
872 answers each with a new empty element (WHATWG 13.2.6.4.7, "in body").
873 """
874 nodes = list(root.descendants)
875 elements = {
876 (
877 element.namespace.value,
878 element.tag,
879 tuple(sorted((name, _attr_text(value)) for name, value in element.attrs.items())),
880 )
881 for element in root.iter_elements()
882 }
883 return (
884 _newlines("".join(node.data for node in nodes if isinstance(node, Text))),
885 [node.data for node in nodes if isinstance(node, Comment)],
886 elements - {("html", "p", ()), ("html", "br", ())},
887 )
890def _newlines(text: str) -> str:
891 return text.replace("\r\n", "\n").replace("\r", "\n")
894def _past_depth_cap(root: Node) -> bool:
895 # the root node plus 512 ancestors is the deepest placement the tree builder allows, reached only on a full stack
896 return any(sum(1 for _ in element.ancestors) > _MAX_TREE_DEPTH for element in root.iter_elements())
899def _xml(node: Node) -> str:
900 return node.serialize(Html(xml=True))
903def xml_check(markup: str, serialize: Callable[[Node], str] = _xml) -> str | None:
904 """
905 Require ``serialize(Html(xml=True))`` output to re-read with ``parse_xml`` into the tree it came from.
907 The string must also print unchanged. Content the ``Html.xml`` docstring says the serializer drops or spaces (C0
908 controls, surrogates, U+FFFE/U+FFFF, ``--`` or a trailing ``-`` in a comment, non-XML attribute names) waives only
909 the tree comparison.
910 """
911 document = parse(markup)
912 if sum(isinstance(node, Element) for node in document.children) != 1 or any(
913 isinstance(node, Text) for node in document.children
914 ):
915 raise OutOfScopeError
916 if _xml_has_unbound_prefix(document):
917 raise OutOfScopeError
918 try:
919 reread = parse_xml(once := serialize(document))
920 except (ValueError, HTMLParseError):
921 return "parse_xml rejects the XML serialization"
922 if (twice := serialize(reread)) != once:
923 return f"xml not a fixpoint: {_divergence(once, twice)}"
924 if _xml_lossy(document):
925 return None
926 expected, got = [*_xml_dump(document), "#end"], [*_xml_dump(reread), "#end"]
927 if expected == got:
928 return None
929 return "xml re-read tree differs: " + next(
930 f"{want[:1]} vs {have[:1]}" for want, have in zip(expected, got, strict=False) if want != have
931 )
934def _xml_has_unbound_prefix(document: Document) -> bool:
935 for element in document.iter_elements():
936 for name in (element.tag, *element.attrs):
937 if ":" not in name or name.startswith((":", "xmlns:")) or name.endswith(":") or name.count(":") > 1:
938 continue
939 prefix = name.split(":", 1)[0]
940 if prefix == "xml" or (prefix == "xlink" and element.namespace is not Namespace.HTML):
941 continue
942 if not any(
943 isinstance(ancestor, Element) and ancestor.attrs.get(f"xmlns:{prefix}")
944 for ancestor in (element, *element.ancestors)
945 ):
946 return True
947 return False
950# XML 1.0 Char (https://www.w3.org/TR/xml/#NT-Char) and Name (https://www.w3.org/TR/xml/#NT-Name) productions
951_XML_CHAR: Final = re.compile(r"[^\t\n\r\x20-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]")
952_XML_PUBID: Final = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \r\n-'()+,./:=?;!*#@$_%")
953_NAME_START: Final = (
954 r":A-Z_a-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef"
955 r"\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd\U00010000-\U000effff"
956)
957_XML_NAME: Final = re.compile(rf"[{_NAME_START}][{_NAME_START}\-.0-9\xb7\u0300-\u036f\u203f-\u2040]*")
960def _xml_lossy(document: Document) -> bool:
961 for node in document.descendants:
962 if isinstance(node, Doctype) and _xml_doctype_lossy(node):
963 return True
964 if isinstance(node, (Text, CData, ProcessingInstruction)) and _XML_CHAR.search(node.data):
965 return True
966 if isinstance(node, Comment) and (_XML_CHAR.search(node.data) or "--" in node.data or node.data.endswith("-")):
967 return True
968 if isinstance(node, Element) and (_XML_NAME.fullmatch(node.tag) is None or ":" in node.tag):
969 return True
970 if isinstance(node, Element) and any(
971 _XML_NAME.fullmatch(name) is None
972 or name.startswith(":")
973 or name.endswith(":")
974 or name.count(":") > 1
975 or (":" in name and not name.startswith("xmlns:"))
976 or _XML_CHAR.search(_attr_text(value))
977 for name, value in node.attrs.items()
978 ):
979 return True
980 return False
983def _xml_doctype_lossy(node: Doctype) -> bool:
984 if _XML_NAME.fullmatch(node.name) is None:
985 return True
986 if node.public_id is not None and (
987 node.system_id is None or any(character not in _XML_PUBID for character in node.public_id)
988 ):
989 return True
990 return node.system_id is not None and _XML_CHAR.search(node.system_id) is not None
993def _attr_text(value: str | list[str] | None) -> str:
994 return " ".join(value) if isinstance(value, list) else value or ""
997def _xml_dump(root: Node) -> list[str]:
998 """
999 Flatten a tree to comparable lines: element names with their non-namespace attributes, merged text, comments.
1001 ``parse_xml`` reports every element in the HTML namespace and keeps ``xmlns`` declarations as attributes, and a
1002 template's content fragment holds what XML reads as the template's children, so those three are normalized away.
1003 """
1004 out: list[str] = []
1005 text = ""
1006 stack: list[Node | str] = list(reversed(root.children))
1007 while stack:
1008 if isinstance(node := stack.pop(), (Text, CData)):
1009 text += node.data
1010 continue
1011 if text:
1012 out.append(f'"{text}')
1013 text = ""
1014 if isinstance(node, str):
1015 out.append(node)
1016 elif isinstance(node, (Element, DocumentFragment)):
1017 if isinstance(node, Element):
1018 out.append(f"<{node.tag} {_xml_attrs(node)}")
1019 stack.append(f"</{node.tag}")
1020 stack.extend(reversed(node.children))
1021 else:
1022 out.append(_xml_leaf(node))
1023 if text:
1024 out.append(f'"{text}')
1025 return out
1028def _xml_attrs(element: Element) -> list[tuple[str, str]]:
1029 return sorted(
1030 (name, _attr_text(value))
1031 for name, value in element.attrs.items()
1032 if name != "xmlns" and not name.startswith("xmlns:")
1033 )
1036def _xml_leaf(node: Node) -> str:
1037 if isinstance(node, Comment):
1038 return f"!{node.data}"
1039 if isinstance(node, ProcessingInstruction):
1040 return f"?{node.target} {node.data}"
1041 if isinstance(node, Doctype):
1042 # DOM Parsing 3.2.1.3 writes an external ID only when non-empty, so empty and absent read the same
1043 return f"D{node.name} {node.public_id or None} {node.system_id or None}"
1044 return type(node).__name__
1047def resolve_links_check(text: str, resolve: Callable[[Node, str], None] = Node.resolve_links) -> str | None:
1048 """Check fixed RFC targets because stable wrong resolutions satisfy a fixed-point check."""
1049 case, separator, leaf = text.partition(":")
1050 if not separator or case not in _RESOLUTION_CASES or re.fullmatch(r"[a-z][a-z0-9]{0,15}", leaf) is None:
1051 raise OutOfScopeError(text)
1052 reference, target = _RESOLUTION_CASES[case]
1053 document = parse_fragment('<a href="">link</a>')
1054 document.select("a")[0].attrs["href"] = reference.format(leaf=leaf)
1055 resolve(document, _RESOLUTION_BASE)
1056 expected: Final = target.format(leaf=leaf)
1057 if document.links()[0].url != expected:
1058 return "resolution differs from RFC target"
1059 resolve(document, _RESOLUTION_BASE)
1060 return None if document.links()[0].url == expected else "resolved link changes on repeat"
1063def style_check(text: str, read: Callable[[str], StyleDeclaration] = StyleDeclaration.parse) -> str | None:
1064 """Require ``StyleDeclaration.text`` to re-read to the same declarations and print unchanged."""
1065 first = read(text)
1066 second = read(once := first.text)
1067 if _winning(first) != _winning(second):
1068 return "declarations differ after re-read"
1069 if (twice := second.text) != once:
1070 return f"not a fixpoint: {_divergence(once, twice)}"
1071 return None
1074def _winning(declaration: StyleDeclaration) -> list[tuple[str, str | None, bool]]:
1075 return [(name, declaration.get(name), declaration.important(name)) for name in declaration.properties()]
1078def markdown_check(markup: str, render: Callable[[Node], str] = Node.to_markdown) -> str | None:
1079 """
1080 Require ``to_markdown`` output to mean, to markdown-it-py's GFM reader, what its own re-rendering means.
1082 ``m1 = to_markdown(html)`` is read into ``h1``, and ``m2 = to_markdown(h1)`` must read into ``h1`` again: the
1083 Markdown tree is the one compared, so spelling choices (which characters get escaped, how many blank lines) do not
1084 count, while emphasis delimiters that merge, a table that reads as a paragraph, or a character left unescaped that
1085 starts a construct do. CommonMark has no syntax for a block inside an inline or inside a GFM table cell, so a tree
1086 that nests one (a ``<pre>`` in a ``<del>``, as the parser builds from misnested tags) has no faithful Markdown form
1087 and skips.
1088 """
1089 document = parse(markup)
1090 if any(
1091 element.tag in _MARKDOWN_BLOCKS
1092 and any(getattr(ancestor, "tag", "") in _MARKDOWN_HOSTS for ancestor in element.ancestors)
1093 for element in document.iter_elements()
1094 ):
1095 raise OutOfScopeError
1096 if any(
1097 element.tag == "br"
1098 and isinstance(element.previous_sibling, Element)
1099 and element.previous_sibling.tag == "br"
1100 and any(getattr(ancestor, "tag", "") in _MARKDOWN_HOSTS for ancestor in element.ancestors)
1101 for element in document.iter_elements()
1102 ):
1103 raise OutOfScopeError
1104 if any(
1105 element.tag == "br"
1106 and any(getattr(ancestor, "tag", "") in {"td", "th"} for ancestor in element.ancestors)
1107 and any(
1108 getattr(ancestor, "tag", "") in {"em", "strong", "del", "s", "strike"} for ancestor in element.ancestors
1109 )
1110 for element in document.iter_elements()
1111 ):
1112 raise OutOfScopeError
1113 first = _MARKDOWN.render(render(document))
1114 if (second := _MARKDOWN.render(render(parse(first)))) != first and _markdown_html(second) != _markdown_html(first):
1115 try:
1116 if markdown_html_meaning(second) != markdown_html_meaning(first):
1117 return f"not a fixpoint: {_divergence(first, second)}"
1118 except MarkdownProfileError as error:
1119 raise OutOfScopeError from error
1120 return None
1123def _markdown_html(markup: str) -> str:
1124 markup = _EMPTY_PARAGRAPH.sub("", markup)
1125 markup = re.sub(r"<li>\n?<p>(.*?)</p>\n?</li>", r"<li>\1</li>", markup, flags=re.DOTALL)
1126 markup = markup.replace("<del>", "<s>").replace("</del>", "</s>")
1127 markup = re.sub(r"<br\s*/?>[ \t]*\n?", "<br>", markup)
1128 markup = markup.replace("<li>\n<br>", "<li><br>")
1129 markup = re.sub(r"((?:<(?:strong|em|s)>)+)<br>", r"<br>\1", markup)
1130 markup = re.sub(r"<br>((?:</(?:strong|em|s)>)+)", r"\1<br>", markup)
1131 return re.sub(r"<(strong|em|s)></\1>", "", markup)
1134class JsNames:
1135 """
1136 The free identifiers and static strings of a JavaScript source, read by acorn and eslint-scope in one Node process.
1138 ``tools/bench/node/js_names_runner.js`` defines both sets. Kept apart from turbohtml's own JS front end on purpose:
1139 a reader that shared the minifier's lexer would share its bugs.
1140 """
1142 def __init__(self, runner_dir: Path = _NODE_DIR) -> None:
1143 """Start nothing yet; the Node process runs ``js_names_runner.js`` from ``runner_dir`` on the first read."""
1144 self._runner_dir = runner_dir
1145 self._process: subprocess.Popen[str] | None = None
1147 def read(self, source: str) -> tuple[frozenset[str], frozenset[str]] | None:
1148 """
1149 Return ``(free identifiers, static strings)``, or None when acorn rejects ``source``.
1151 :raises OutOfScopeError: eslint-scope could not scope the tree acorn built.
1152 """
1153 process = self._start()
1154 assert process.stdin is not None # ruff: ignore[assert] # Popen with stdin=PIPE always sets it
1155 assert process.stdout is not None # ruff: ignore[assert] # Popen with stdout=PIPE always sets it
1156 process.stdin.write(json.dumps(source) + "\n")
1157 process.stdin.flush()
1158 if (answer := json.loads(process.stdout.readline())) is None:
1159 return None
1160 if "error" in answer:
1161 raise OutOfScopeError(answer["error"])
1162 return frozenset(answer["free"]), frozenset(answer["strings"])
1164 def _start(self) -> subprocess.Popen[str]:
1165 if self._process is None:
1166 if (node := shutil.which("node")) is None or not (
1167 self._runner_dir / "node_modules" / "eslint-scope"
1168 ).is_dir():
1169 msg = f"node and `npm ci` in {self._runner_dir} are required by the js-names oracle"
1170 raise FileNotFoundError(msg)
1171 self._process = subprocess.Popen(
1172 [node, str(self._runner_dir / "js_names_runner.js")],
1173 stdin=subprocess.PIPE,
1174 stdout=subprocess.PIPE,
1175 text=True,
1176 encoding="utf-8",
1177 errors="surrogatepass",
1178 cwd=self._runner_dir,
1179 # an ASan-preloaded run must not hand its runtime to node
1180 env={
1181 key: value
1182 for key, value in os.environ.items()
1183 if key not in {"LD_PRELOAD", "DYLD_INSERT_LIBRARIES"}
1184 },
1185 )
1186 return self._process
1189_OPTION_SETS: Final = tuple(JSMinify(mangle=mangle, fold=fold) for fold in (False, True) for mangle in (False, True))
1192def js_names_check(source: str, names: JsNames, minify: Callable[[str, JSMinify], str] = minify_js) -> str | None:
1193 """
1194 Require ``minify_js`` to invent and lose no free identifier and no static string.
1196 With ``mangle`` and ``fold`` off both sets must match exactly. ``fold`` deletes dead code and concatenates strings
1197 and ``mangle`` drops unused declarations, so with either on no free identifier may appear that the input lacked.
1198 Every output must parse as JavaScript.
1199 """
1200 if (before := names.read(source)) is None:
1201 raise OutOfScopeError
1202 compared = False
1203 for options in _OPTION_SETS:
1204 try:
1205 output = minify(source, options)
1206 except ValueError:
1207 continue
1208 compared = True
1209 label = f"mangle={options.mangle} fold={options.fold}"
1210 if (after := names.read(output)) is None:
1211 return f"{label}: output is not JavaScript"
1212 if options.fold or options.mangle:
1213 if after[0] - before[0]:
1214 return f"{label}: free identifier invented"
1215 elif after[0] != before[0]:
1216 return f"{label}: free identifier {'invented' if after[0] - before[0] else 'lost'}"
1217 elif after[1] != before[1]:
1218 return f"{label}: static string {'invented' if after[1] - before[1] else 'lost'}"
1219 if not compared:
1220 raise OutOfScopeError
1221 return None
1224def css_semantics_check(
1225 markup: str, minify: Callable[[str], str] = minify_css, *, stylesheet: str | None = None
1226) -> str | None:
1227 """
1228 Require ``minify_css`` to keep every element's computed style and every selector's match set.
1230 cssnano's fuzzCheck model: the document's ``<style>`` sheet is minified in place and ``computed_style`` of each
1231 element compared. ``computed_style`` returns values as written and keeps invalid ones, so each pair is read with
1232 the CSS value grammar first: two valid values must be equal values, and a value the grammar rejects must not become
1233 one it accepts. A value outside what this reader covers makes no claim. Each rule's selector is also minified on its
1234 own and must select the same elements.
1235 """
1236 document = parse(markup)
1237 style = _css_style(document, stylesheet)
1238 sheet = style.text
1239 _require_supported_css_shorthands(sheet)
1240 elements = list(document.iter_elements())
1241 before = [computed_style(element) for element in elements]
1242 style.text = minify(sheet)
1243 after = [computed_style(element) for element in elements]
1244 for want, got in zip(before, after, strict=True):
1245 for name in want.properties():
1246 if (verdict := _value_verdict(name, want[name], got[name])) is not None:
1247 return f"computed {_VALUE_KIND.get(name, 'keyword')}: {verdict}"
1248 for rule in StyleSheet(sheet).rules:
1249 try:
1250 expected = document.select(rule.selector_text)
1251 except SelectorSyntaxError:
1252 continue
1253 minified = StyleSheet(minify(f"{rule.selector_text}{{color:red}}")).rules
1254 if len(minified) != 1:
1255 return "selector rule dropped"
1256 try:
1257 if document.select(minified[0].selector_text) != expected:
1258 return "selector match set differs"
1259 except SelectorSyntaxError:
1260 return "minified selector does not parse"
1261 return None
1264def _require_supported_css_shorthands(sheet: str) -> None:
1265 for rule in StyleSheet(sheet).rules:
1266 for name in ("margin", "padding", "border-width"):
1267 if (value := rule.style.get(name)) is not None and any(
1268 _read_value("border-top-width" if name == "border-width" else f"{name}-top", part) is _INVALID
1269 for part in value.split()
1270 ):
1271 raise OutOfScopeError
1274def _css_style(document: Document, stylesheet: str | None) -> Element:
1275 if (style := document.select_one("style")) is None:
1276 raise OutOfScopeError
1277 if stylesheet is not None:
1278 style.text = stylesheet
1279 return style
1282def _value_verdict(name: str, before: str, after: str) -> str | None:
1283 if before == after:
1284 return None
1285 want, got = _read_value(name, before), _read_value(name, after)
1286 if want is _INVALID:
1287 return None if got is None or got is _INVALID else "invalid value became valid"
1288 if want is None or got is None:
1289 return None
1290 if got is _INVALID:
1291 return "valid value became invalid"
1292 return None if want == got else "value changed"
1295_INVALID: Final = ("invalid",)
1296_SIDES: Final = ("top", "right", "bottom", "left")
1297_VALUE_KIND: Final[dict[str, str]] = {
1298 **dict.fromkeys(
1299 ("color", "background-color", "outline-color", *(f"border-{side}-color" for side in _SIDES)), "color"
1300 ),
1301 **dict.fromkeys(
1302 (
1303 *_SIDES,
1304 *(f"{box}-{side}" for box in ("margin", "padding") for side in _SIDES),
1305 *(f"border-{side}-width" for side in _SIDES),
1306 *(f"{bound}-{axis}" for bound in ("min", "max") for axis in ("width", "height")),
1307 "width",
1308 "height",
1309 "text-indent",
1310 "letter-spacing",
1311 "word-spacing",
1312 "font-size",
1313 "outline-width",
1314 ),
1315 "length",
1316 ),
1317 "line-height": "line-height",
1318 "z-index": "integer",
1319 "opacity": "alpha",
1320 "font-weight": "weight",
1321}
1322_LENGTH_UNITS: Final = frozenset({
1323 "px", "em", "rem", "ex", "ch", "vw", "vh", "vmin", "vmax", "cm", "mm", "q", "in", "pt", "pc", "%",
1324}) # fmt: skip
1325_NUMERIC: Final = re.compile(r"([+-]?(?:\d+\.\d+|\d+|\.\d+)(?:[eE][+-]?\d+)?)([a-zA-Z]+|%)?")
1326# CSS Syntax 3 4.3.12: a "." starts a fraction only before a digit, so "1." is a number and a stray "." delim
1327_TRAILING_DOT: Final = re.compile(r"[+-]?\d+\.(?:[a-zA-Z]+|%)?")
1328_NAMED_COLORS: Final = {
1329 "red": (255, 0, 0, 1.0),
1330 "white": (255, 255, 255, 1.0),
1331 "black": (0, 0, 0, 1.0),
1332 "navy": (0, 0, 128, 1.0),
1333 "transparent": (0, 0, 0, 0.0),
1334}
1337def _read_value(name: str, value: str) -> object:
1338 """
1339 Read one computed value with its property's grammar, for the properties the generator writes.
1341 Returns a comparable value, ``_INVALID`` when the grammar rejects it, or None when it falls outside what this reader
1342 covers. Sources: CSS Values 4 (lengths, a unitless zero, integers), CSS Color 4 (hex, ``rgb()`` legacy and modern
1343 syntax, named colors, opacity percentages), CSS Fonts 4 (``bold`` and ``normal`` are 700 and 400).
1344 """
1345 text = value.strip()
1346 kind = _VALUE_KIND.get(name)
1347 if _TRAILING_DOT.fullmatch(text):
1348 return _INVALID
1349 if kind == "color":
1350 return _read_color(text.lower())
1351 if (numeric := _NUMERIC.fullmatch(text)) is not None:
1352 return _read_number(kind, numeric[1], (numeric[2] or "").lower())
1353 keyword = text.lower()
1354 if re.fullmatch(r"-?[a-z][a-z-]*", keyword) is None:
1355 return None
1356 return {"bold": 700.0, "normal": 400.0}.get(keyword, keyword) if kind == "weight" else keyword
1359def _read_number(kind: str | None, digits: str, unit: str) -> object:
1360 number = float(digits)
1361 readers: dict[str | None, Callable[[], object]] = {
1362 "integer": lambda: int(number) if not unit and re.fullmatch(r"[+-]?\d+", digits) else _INVALID,
1363 "alpha": lambda: {"": number, "%": number / 100}.get(unit, _INVALID),
1364 "weight": lambda: number if not unit and 1 <= number <= 1000 else _INVALID,
1365 "length": lambda: _read_length(number, unit),
1366 "line-height": lambda: ("number", number) if not unit else _read_length(number, unit),
1367 }
1368 return readers.get(kind, lambda: None)()
1371def _read_length(number: float, unit: str) -> object:
1372 if unit and unit not in _LENGTH_UNITS:
1373 return _INVALID
1374 if number == 0 and unit != "%":
1375 return ("length", 0.0, "")
1376 return ("length", number, unit) if unit else _INVALID
1379def _read_color(text: str) -> object:
1380 if text in _NAMED_COLORS:
1381 return _NAMED_COLORS[text]
1382 if text.startswith("#"):
1383 digits = text[1:]
1384 if re.fullmatch(r"[0-9a-f]+", digits) is None or len(digits) not in {3, 4, 6, 8}:
1385 return _INVALID
1386 full = "".join(digit * 2 for digit in digits) if len(digits) < 6 else digits
1387 channels = [int(full[index : index + 2], 16) for index in range(0, len(full), 2)]
1388 return (*channels[:3], round(channels[3] / 255, 3) if len(channels) == 4 else 1.0)
1389 if (function := re.fullmatch(r"rgba?\((.*)\)", text, re.DOTALL)) is None or re.search(r"[a-z(]", function[1]):
1390 return None # other color functions, calc(), var() and none need a full value parser
1391 return _read_rgb(function[1])
1394def _read_rgb(arguments: str) -> object:
1395 """Read ``rgb()`` arguments: three comma- or space-separated channels and an alpha, after ``/`` in the latter."""
1396 parts = [part.strip() for part in arguments.split(",")]
1397 legacy = len(parts) > 1
1398 if not legacy:
1399 modern = re.fullmatch(r"(\S+)\s+(\S+)\s+(\S+)(?:\s*/\s*(\S+))?", parts[0])
1400 parts = [] if modern is None else [part for part in modern.groups() if part is not None]
1401 elif any(re.search(r"\s", part) for part in parts):
1402 parts = []
1403 numbers = [match for part in parts if (match := _NUMERIC.fullmatch(part)) is not None and match[2] in {None, "%"}]
1404 if len(parts) not in {3, 4} or len(numbers) != len(parts):
1405 return _INVALID
1406 if legacy and len({match[2] for match in numbers[:3]}) != 1:
1407 return _INVALID # the comma syntax takes three numbers or three percentages, never a mix
1408 channels = [round(min(max(float(match[1]) * (2.55 if match[2] else 1), 0), 255)) for match in numbers[:3]]
1409 alpha = 1.0 if len(numbers) == 3 else float(numbers[3][1]) / (100 if numbers[3][2] else 1)
1410 return (*channels, round(min(max(alpha, 0.0), 1.0), 3))
1413# several spellings of one value each, so the generator writes values the minifier rewrites
1414_VALUE_SPELLINGS: Final[dict[str, tuple[str, ...]]] = {
1415 "red": ("red", "#f00", "#ff0000", "#F00", "#FF0000", "rgb(255,0,0)", "rgb(255 0 0)", "#ff0000ff", "#f00f"),
1416 "white": ("white", "#fff", "#ffffff", "#FFF", "rgb(255,255,255)", "#ffffffff"),
1417 "black": ("black", "#000", "#000000", "rgb(0,0,0)", "#000000ff"),
1418 "navy": ("navy", "#000080", "rgb(0,0,128)"),
1419 "transparent": ("transparent", "#0000", "#00000000", "rgba(0,0,0,0)"),
1420 "0": ("0", "0px", "0em", "0.0px", "-0px", "+0px", "00px", ".0px", "0.0", "-0", "+0"),
1421 "1.5px": ("1.5px", "1.50px", "+1.5px", "01.5px", "15e-1px", "0.15e1px"),
1422 "10px": ("10px", "1e1px", "10.0px", "+10px", "010px"),
1423 "1": ("1", "+1", "01"),
1424 "0.5": ("0.5", ".5", "0.50", "+.5", "5e-1"),
1425 "700": ("700", "bold"),
1426 "400": ("400", "normal"),
1427}
1430def xpath_entry_check(
1431 case: str,
1432 iterate: Callable[[Node, str], Sequence[object]] = lambda node, expression: list(node.xpath_iter(expression)),
1433 first: Callable[[Node, str], object] = lambda node, expression: node.xpath_one(expression),
1434) -> str | None:
1435 """
1436 Require ``xpath``, ``xpath_iter``, ``xpath_one`` and a compiled ``XPath`` to agree on one expression.
1438 pugixml evaluates one query through all its result APIs; here ``list(xpath_iter(e)) == xpath(e)`` (a scalar result
1439 is its single item), ``xpath_one(e)`` is the first item or None, ``XPath(e)(node) == xpath(e)``, and every entry
1440 point raises the same exception type when one does.
1441 """
1442 payload = json.loads(case)
1443 document = parse(payload["html"])
1444 expression = payload["xpath"]
1445 for context in [document, *list(document.iter_elements())[:6]]:
1446 result = _outcome(lambda context=context: context.xpath(expression))
1447 compiled = _outcome(lambda context=context: XPath(expression)(context))
1448 iterated = _outcome(lambda context=context: iterate(context, expression))
1449 one = _outcome(lambda context=context: first(context, expression))
1450 if isinstance(result, _Raised):
1451 if {type(outcome) for outcome in (compiled, iterated, one)} != {_Raised} or len({
1452 outcome.kind for outcome in (result, compiled, iterated, one) if isinstance(outcome, _Raised)
1453 }) != 1:
1454 return "entry points disagree on raising"
1455 continue
1456 items = result if isinstance(result, list) else [result]
1457 if not _same(compiled, result):
1458 return "XPath()() != xpath()"
1459 if not _same(iterated, items):
1460 return "list(xpath_iter()) != xpath()"
1461 if not _same(one, items[0] if items else None):
1462 return "xpath_one() != first item"
1463 return None
1466@dataclass(frozen=True)
1467class _Raised:
1468 kind: str
1471def _outcome(call: Callable[[], object]) -> object:
1472 try:
1473 return call()
1474 except (ValueError, TypeError) as error:
1475 return _Raised(type(error).__name__)
1478def _same(left: object, right: object) -> bool:
1479 """Compare XPath results: nodes by identity (``Node.__eq__``), NaN equal to itself, lists item by item."""
1480 if isinstance(left, list) and isinstance(right, list):
1481 return len(left) == len(right) and all(starmap(_same, zip(left, right, strict=False)))
1482 if isinstance(left, float) and isinstance(right, float) and math.isnan(left) and math.isnan(right):
1483 return True
1484 return type(left) is type(right) and left == right
1487def selector_entry_check(case: str, translate: Callable[[str], str] = css_to_xpath) -> str | None:
1488 """
1489 Require the CSS entry points to agree on one selector over one document.
1491 ``select(s) == xpath(css_to_xpath(s))`` from the document; a compiled ``Matcher`` selects, filters and matches what
1492 ``Node.select`` and ``Node.matches`` do; a scoped select returns exactly the matching descendants, in order; and
1493 ``closest`` finds the nearest matching ancestor-or-self.
1494 """
1495 payload = json.loads(case)
1496 document = parse(payload["html"])
1497 selector = payload["css"]
1498 try:
1499 selected = document.select(selector)
1500 matcher = compile_selector(selector)
1501 except SelectorSyntaxError as error:
1502 raise OutOfScopeError from error
1503 if (problem := _matcher_problem(document, selector, matcher, selected)) is not None:
1504 return problem
1505 try:
1506 translated = translate(selector)
1507 except ExpressionError:
1508 return None
1509 try:
1510 xpath_selected = document.xpath(translated)
1511 except ValueError as error:
1512 if str(error) == "xpath: an expression nested too deeply":
1513 raise OutOfScopeError from error
1514 raise
1515 return None if xpath_selected == selected else "xpath(css_to_xpath()) != select()"
1518def _matcher_problem(document: Document, selector: str, matcher: Matcher, selected: list[Element]) -> str | None:
1519 elements = list(document.iter_elements())
1520 # a node can come back as a fresh wrapper (free-threaded builds), so match by node equality, not id()
1521 matching = set(selected)
1522 first = selected[0] if selected else None
1523 problems = (
1524 ([element for element in elements if element.matches(selector)] != selected, "matches() != select()"),
1525 (matcher.select(document) != selected or matcher.filter(elements) != selected, "Matcher != select()"),
1526 (first not in {matcher.select_one(document), document.select_one(selector)}, "select_one() != first match"),
1527 )
1528 if (problem := next((label for broken, label in problems if broken), None)) is not None:
1529 return problem
1530 for scope in elements[:8]:
1531 if scope.select(selector) != [element for element in scope.iter_elements() if element in matching]:
1532 return "scoped select() != matching descendants"
1533 nearest = next((element for element in [scope, *scope.ancestors] if element in matching), None)
1534 if matcher.closest(scope) != nearest:
1535 return "closest() != nearest matching ancestor"
1536 return None
1539def span_check(
1540 markup: str, locate: Callable[[Element], SourceLocation | None] = lambda element: element.source_location
1541) -> str | None:
1542 """
1543 Require source spans that slice their constructs out of the source.
1545 html5gum and Babel's position fuzzing check the same: every span lies inside the newline-normalized source with a
1546 line and column that match its offset; a start tag span begins with ``<`` and the tag name, an end tag span with
1547 ``</``, both end with ``>``; attribute spans sit inside their start tag and begin with the attribute name; distinct
1548 tag spans never overlap; ``source_line``/``source_col`` name the start tag; a comment prefix shifts every span by
1549 exactly its length; and an ``IncrementalParser`` fed in pieces reports the spans a one-shot parse does.
1550 """
1551 source = markup.replace("\r\n", "\n").replace("\r", "\n")
1552 document = parse(markup, source_locations=True)
1553 elements = [element for element in document.iter_elements() if locate(element) is not None]
1554 tokens: set[tuple[int, int]] = set()
1555 for element in elements:
1556 if (problem := _element_span_problem(source, element, locate)) is not None:
1557 return problem
1558 location = locate(element)
1559 assert location is not None # ruff: ignore[assert] # filtered above
1560 tokens.update((span.start_offset, span.end_offset) for span in (location.start_tag, location.end_tag) if span)
1561 ordered = sorted(tokens)
1562 if any(end > start for (_, end), (start, _) in pairwise(ordered)):
1563 return "tag spans overlap"
1564 prefix = f"<!--{chr(10) * (len(markup) % 3)}{'x' * (len(markup) % 5)}-->"
1565 shifted = parse(prefix + markup, source_locations=True)
1566 if [element.tag for element in shifted.iter_elements()] != [element.tag for element in document.iter_elements()]:
1567 return "a comment prefix changed the tree"
1568 expected = [_shift(locate(element), prefix) for element in document.iter_elements()]
1569 if [locate(element) for element in shifted.iter_elements()] != expected:
1570 return "a comment prefix shifted spans inexactly"
1571 incremental = IncrementalParser(source_locations=True)
1572 for start in range(0, len(markup), step := len(markup) // 3 + 1):
1573 incremental.feed(markup[start : start + step])
1574 if [locate(element) for element in incremental.close().iter_elements()] != [
1575 locate(element) for element in document.iter_elements()
1576 ]:
1577 return "IncrementalParser spans != parse() spans"
1578 return None
1581def _element_span_problem(
1582 source: str, element: Element, locate: Callable[[Element], SourceLocation | None]
1583) -> str | None:
1584 location = locate(element)
1585 assert location is not None # ruff: ignore[assert] # the caller passes located elements only
1586 start, end = location.start_tag, location.end_tag
1587 if (problem := _span_problem(source, start)) is not None:
1588 return f"start tag {problem}"
1589 # the tokenizer reads a NUL in a tag or attribute name as U+FFFD (WHATWG 13.2.5.8, 13.2.5.33)
1590 text = source[start.start_offset : start.end_offset].replace("\0", "\ufffd")
1591 # the tree builder renames an "image" start tag to "img" (WHATWG 13.2.6.4.7, in body)
1592 names = {element.tag.lower(), "image"} if element.tag == "img" else {element.tag.lower()}
1593 if not text.endswith(">") or not any(
1594 re.match(f"<{re.escape(name)}[\t\n\f />]", text, re.IGNORECASE) for name in names
1595 ):
1596 return "start tag span does not slice <name ...>"
1597 if (element.source_line, element.source_col) != (start.start_line, start.start_col):
1598 return "source_line/source_col != start tag start"
1599 if end is not None and (problem := _end_tag_problem(source, start, end)) is not None:
1600 return problem
1601 return _attribute_span_problem(source, start, location.attrs)
1604def _end_tag_problem(source: str, start: SourceSpan, end: SourceSpan) -> str | None:
1605 if (problem := _span_problem(source, end)) is not None:
1606 return f"end tag {problem}"
1607 if end.start_offset < start.end_offset or not source[end.start_offset : end.end_offset].startswith("</"):
1608 return "end tag span does not slice </name> after the start tag"
1609 return None
1612def _attribute_span_problem(source: str, start: SourceSpan, attrs: dict[str, SourceSpan]) -> str | None:
1613 previous = start.start_offset
1614 for name, span in sorted(attrs.items(), key=lambda item: item[1].start_offset):
1615 if (problem := _span_problem(source, span)) is not None:
1616 return f"attribute {problem}"
1617 if span.start_offset < previous or span.end_offset > start.end_offset:
1618 return "attribute span outside its start tag or overlapping"
1619 if not source[span.start_offset : span.end_offset].replace("\0", "\ufffd").lower().startswith(name.lower()):
1620 return "attribute span does not begin with its name"
1621 previous = span.end_offset
1622 return None
1625def _span_problem(source: str, span: SourceSpan) -> str | None:
1626 if not 0 <= span.start_offset < span.end_offset <= len(source):
1627 return "span empty or outside the source"
1628 for offset, line, column in (
1629 (span.start_offset, span.start_line, span.start_col),
1630 (span.end_offset, span.end_line, span.end_col),
1631 ):
1632 if (line, column) != (source.count("\n", 0, offset) + 1, offset - source.rfind("\n", 0, offset) - 1):
1633 return "line or column disagrees with offset"
1634 return None
1637def _shift(location: SourceLocation | None, prefix: str) -> SourceLocation | None:
1638 if location is None:
1639 return None
1640 lines = prefix.count("\n")
1641 tail = len(prefix) - prefix.rfind("\n") - 1
1643 def move(span: SourceSpan) -> SourceSpan:
1644 return SourceSpan(
1645 span.start_line + lines,
1646 span.start_col + tail * (span.start_line == 1),
1647 span.start_offset + len(prefix),
1648 span.end_line + lines,
1649 span.end_col + tail * (span.end_line == 1),
1650 span.end_offset + len(prefix),
1651 )
1653 return SourceLocation(
1654 move(location.start_tag),
1655 None if location.end_tag is None else move(location.end_tag),
1656 {name: move(span) for name, span in location.attrs.items()},
1657 )
1660def _css_fixpoint(text: str, printer: Callable[[str], str] = minify_css) -> str | None:
1661 if _nesting(text) > _CSS_MAX_NESTING:
1662 raise OutOfScopeError
1663 return fixpoint_check(text, printer, numeric=True)
1666def _css_valid_fixpoint(text: str, printer: Callable[[str], str] = minify_css) -> str | None:
1667 if not _css_valid_source(text):
1668 raise OutOfScopeError
1669 return _css_fixpoint(text, printer)
1672def _css_valid_source(text: str) -> bool:
1673 tokens = tinycss2.parse_component_value_list(text)
1674 recovered = tinycss2.serialize(tokens)
1675 return not (
1676 _css_has_parse_error(tokens)
1677 or any(recovered.count(delimiter) > text.count(delimiter) for delimiter in ")}]")
1678 or (recovered.replace("/**/", "").startswith(text.replace("/**/", "")) and recovered != text)
1679 or any(not _css_rule_valid(rule) for rule in tinycss2.parse_stylesheet(text))
1680 )
1683def _css_rule_valid(rule: object) -> bool:
1684 if isinstance(rule, ParseError):
1685 return False
1686 if isinstance(rule, QualifiedRule):
1687 try:
1688 css_to_xpath(tinycss2.serialize(rule.prelude))
1689 except (ExpressionError, SelectorSyntaxError):
1690 return False
1691 return all(not isinstance(item, ParseError) for item in tinycss2.parse_declaration_list(rule.content))
1692 if (
1693 isinstance(rule, AtRule)
1694 and rule.lower_at_keyword in {"media", "supports", "container", "layer"}
1695 and rule.content is not None
1696 ):
1697 return all(_css_rule_valid(nested) for nested in tinycss2.parse_rule_list(rule.content))
1698 return True
1701def _css_has_parse_error(tokens: list[object]) -> bool:
1702 return any(
1703 getattr(token, "type", "") == "error"
1704 or _css_has_parse_error(getattr(token, "content", getattr(token, "arguments", [])))
1705 for token in tokens
1706 )
1709def _nesting(text: str) -> int:
1710 depth = deepest = 0
1711 for char in text:
1712 depth += (char in "{([") - (char in "})]")
1713 deepest = max(deepest, depth)
1714 return deepest
1717def _js_fixpoint(text: str) -> str | None:
1718 # minify_js accepts some scripts acorn rejects (#984), whose output is not a program anyone wrote
1719 if _JS_NAMES.read(text) is None:
1720 raise OutOfScopeError
1721 return fixpoint_check(text, lambda source: minify_js(source, _JS_OFF), numeric=True)
1724_JS_NAMES: Final = JsNames()
1727def _js_names(text: str) -> str | None:
1728 return js_names_check(text, _JS_NAMES)
1731@dataclass(frozen=True)
1732class _Corpus:
1733 html: tuple[str, ...]
1734 css: tuple[str, ...]
1735 js: tuple[str, ...]
1738@cache
1739def _corpus() -> _Corpus:
1740 """Read the vendored seed inputs once; a missing source is a setup error, never a silent skip."""
1741 fuzz_corpus = _ROOT / "tools" / "fuzz" / "corpus"
1742 regressions = _ROOT / "tests" / "fuzz_regressions"
1743 tree_tests = _require(_ROOT / "tests" / "html5lib-tests" / "tree-construction")
1744 html = [
1745 data
1746 for path in sorted(tree_tests.glob("*.dat"))
1747 for data in re.findall(r"^#data\n(.*?)\n#errors", path.read_text(encoding="utf-8"), re.MULTILINE | re.DOTALL)
1748 ]
1749 html_files = [
1750 *(path for name in ("parse", "roundtrip", "minify_html") for path in (fuzz_corpus / name).iterdir()),
1751 *regressions.glob("*.html"),
1752 ]
1753 golden = _require(_ROOT / "tests" / "css" / "minify" / "css_minify_golden.json")
1754 css_files = [*(fuzz_corpus / "minify_css").iterdir(), *regressions.glob("*.css")]
1755 js = [
1756 row["input"]
1757 for path in sorted(_require(_ROOT / "tests" / "js" / "_corpus").glob("*.json"))
1758 for row in json.loads(path.read_text(encoding="utf-8"))
1759 ]
1760 return _Corpus(
1761 tuple(dict.fromkeys([*html, *_texts(html_files)])),
1762 tuple(dict.fromkeys([*(row[0] for row in json.loads(golden.read_text(encoding="utf-8"))), *_texts(css_files)])),
1763 tuple(dict.fromkeys([*js, *_texts(regressions.glob("*.js"))])),
1764 )
1767def _texts(paths: Iterable[Path]) -> list[str]:
1768 return [path.read_text(encoding="utf-8") for path in sorted(paths)]
1771def _require(path: Path) -> Path:
1772 if not path.exists():
1773 msg = f"{path} is missing; tox -e fuzz-round-trip checks out the seed submodules"
1774 raise FileNotFoundError(msg)
1775 return path
1778def _generated(make: Callable[[random.Random], str], count: int = 150) -> Callable[[], list[str]]:
1779 """Seed an oracle with a fixed batch from its generator, the same on every run."""
1780 return lambda: [make(rng) for rng in [random.Random(0)] for _ in range(count)]
1783def _mutate(rng: random.Random, text: str, tokens: Sequence[str]) -> str:
1784 """Apply one to six edits: insert a token, delete or duplicate a slice, or flip a character's case."""
1785 for _ in range(rng.randint(1, 6)):
1786 at = rng.randint(0, len(text))
1787 end = min(len(text), at + rng.randint(1, 8))
1788 draw = rng.random()
1789 if draw < 0.5 or not text:
1790 text = text[:at] + rng.choice(tokens) + text[at:]
1791 elif draw < 0.7:
1792 text = text[:at] + text[end:]
1793 elif draw < 0.9:
1794 text = text[:end] + text[at:end] + text[end:]
1795 else:
1796 text = text[:at] + text[at:end].swapcase() + text[end:]
1797 return text
1800_CLASSES: Final = ("a", "b", "c", "a b", "b c")
1801_IDS: Final = ("x", "y", "z")
1804_HTML_TOKENS: Final = (
1805 "<p>", "</p>", "<b>", "</b>", "<table>", "<td>", "<svg>", "<math>", "<template>", "<script>", "</script>", "<!--",
1806 "-->", "<![CDATA[", "]]>", "<a href=x>", "</a>", "<select>", "<option>", "<br>", "</br>", "<image>", "&",
1807 "�", "\r", "\n", "<!DOCTYPE html>", "<frameset>", "<plaintext>", "<textarea>", "<style>", "<noscript>",
1808 "<foreignObject>", "<desc>", "<mi>", "<annotation-xml encoding=text/html>", "<li>", "<dd>", "<form>", "</form>",
1809 "<?x?>", "<x:y>", 'a="b"', "\x00", "<head>", "<body>", "</html>", "<col>", "<tr>", "<caption>",
1810) # fmt: skip
1813def _generate_html(rng: random.Random) -> str:
1814 if rng.random() < 0.5:
1815 return _mutate(rng, rng.choice(_corpus().html), _HTML_TOKENS)
1816 return html_generate(rng).data.decode("utf-8")
1819_MARKDOWN_TEXT: Final = (
1820 "word", "two words", "*", "_", "#", "|", "[", "]", "(", ")", "\\", "`", "&", "<", ">", "1.", "2)", "-",
1821 "+", "=", "~", "!", "<", ">", "'", '"', " ", "---", "***", "```", "~~", "x*y", "a_b_c", "[x](y)", "\U0001f600",
1822 "é", "http://e.x", " ", " ",
1823) # fmt: skip
1826def _markdown_source_generate(rng: random.Random) -> str:
1827 return markdown_generate(rng).data.decode("utf-8")
1830def _markdown_html_generate(rng: random.Random) -> str:
1831 return markdown_generate(rng, html=True).data.decode("utf-8")
1834def _markdown_source_check(source: str) -> str | None:
1835 try:
1836 return markdown_source_check(source)
1837 except MarkdownProfileError as error:
1838 raise OutOfScopeError from error
1841def _markdown_html_check(source: str) -> str | None:
1842 try:
1843 return markdown_html_check(source)
1844 except MarkdownProfileError as error:
1845 raise OutOfScopeError from error
1848def _markdown_document(rng: random.Random) -> str:
1849 """
1850 Build conforming HTML that Markdown can express: flow blocks holding phrasing content, never a block inside it.
1852 CommonMark has no syntax for a block inside an inline, so a parser-made tree that nests one (a ``<pre>`` inside a
1853 ``<del>``) has no faithful Markdown form; the text in these blocks carries every Markdown-significant character.
1854 """
1855 return "".join(_markdown_block(rng, 0) for _ in range(rng.randint(1, 4)))
1858def _markdown_block(rng: random.Random, depth: int) -> str:
1859 inline = partial(_markdown_inline, rng, 0)
1860 choices: list[Callable[[], str]] = [
1861 lambda: f"<p>{inline()}</p>",
1862 lambda: f"<p>{inline()}</p>",
1863 lambda: f"<h{(level := rng.randint(1, 6))}>{inline()}</h{level}>",
1864 lambda: f"<pre><code>{rng.choice(_MARKDOWN_TEXT)}\n{rng.choice(_MARKDOWN_TEXT)}</code></pre>",
1865 lambda: "<hr>",
1866 lambda: _markdown_table(rng, rng.randint(1, 3)),
1867 ]
1868 if depth < 2:
1869 choices += [
1870 lambda: f"<blockquote>{_markdown_block(rng, depth + 1)}</blockquote>",
1871 lambda: (
1872 f"<{(kind := rng.choice(('ul', 'ol')))}>"
1873 + "".join(
1874 f"<li>{inline() if rng.random() < 0.7 else _markdown_block(rng, depth + 1)}</li>"
1875 for _ in range(rng.randint(1, 3))
1876 )
1877 + f"</{kind}>"
1878 ),
1879 ]
1880 return rng.choice(choices)()
1883def _markdown_table(rng: random.Random, columns: int) -> str:
1884 head = "".join(f"<th>{_markdown_inline(rng, 0)}</th>" for _ in range(columns))
1885 body = "".join(f"<td>{_markdown_inline(rng, 0)}</td>" for _ in range(columns))
1886 return f"<table><tr>{head}</tr><tr>{body}</tr></table>"
1889def _markdown_inline(rng: random.Random, depth: int) -> str:
1890 out = []
1891 for _ in range(rng.randint(1, 4)):
1892 draw = rng.random()
1893 if draw < 0.5 or depth >= 2:
1894 out.append(rng.choice(_MARKDOWN_TEXT))
1895 elif draw < 0.75:
1896 tag = rng.choice(("em", "strong", "del", "b", "i"))
1897 out.append(f"<{tag}>{_markdown_inline(rng, depth + 1)}</{tag}>")
1898 elif draw < 0.85:
1899 out.append(f"<code>{rng.choice(_MARKDOWN_TEXT)}</code>")
1900 elif draw < 0.93:
1901 out.append(
1902 f'<a href="{rng.choice(("x", "a b", "(", "<", "http://e.x/?a=1&b"))}">{rng.choice(_MARKDOWN_TEXT)}</a>'
1903 )
1904 elif draw < 0.97:
1905 out.append(f'<img src="i" alt="{rng.choice(_MARKDOWN_TEXT)}">')
1906 else:
1907 out.append("<br>")
1908 return "".join(out)
1911_CSS_TOKENS: Final = (
1912 "{", "}", ";", ":", "!important", "/*x*/", "/*!b*/", "@media screen{", "@supports (display:grid){", "url(a b)",
1913 'url("x")', '"s\\"t"', "'q'", "\\", "\\61", "#fff", "0px", "1e3", "+.5", "-0", "calc(1px + 2px)", "var(--x)",
1914 "--x:", "a,b", ">", "+", "~", "*", "[x=y]", ":not(", ")", "::before", "@import", "@charset", "\n", " ", "%",
1915) # fmt: skip
1916_SELECTOR_TAGS: Final = ("div", "p", "span", "a", "b", "i", "li", "ul", "section", "em")
1917_PSEUDOS: Final = (
1918 ":first-child", ":last-child", ":only-child", ":nth-child(2n+1)", ":nth-child(odd)", ":nth-child(even)",
1919 ":nth-last-child(-n+3)", ":nth-of-type(2)", ":first-of-type", ":last-of-type", ":empty", ":root", ":link",
1920 ":any-link", ":checked", ":disabled", ":enabled", ":lang(en)", ":nth-child(+0n+1)", ":nth-child(1 of .a)",
1921 ":only-of-type", ":defined", ":required", ":optional", ":read-only", ":read-write", ":placeholder-shown",
1922) # fmt: skip
1925def _selector(rng: random.Random, depth: int = 0) -> str:
1926 """Draw a selector list over the generators' tag, class, id and attribute vocabulary."""
1927 return ", ".join(_complex_selector(rng, depth) for _ in range(1 if rng.random() < 0.8 else rng.randint(2, 3)))
1930def _complex_selector(rng: random.Random, depth: int) -> str:
1931 parts = [_compound(rng, depth)]
1932 for _ in range(rng.choice((0, 0, 1, 1, 2, 3))):
1933 parts += [rng.choice((" ", " > ", ">", " + ", "~", " ")), _compound(rng, depth)]
1934 return "".join(parts)
1937def _compound(rng: random.Random, depth: int) -> str:
1938 out = rng.choice(("", "", "*", *_SELECTOR_TAGS, "DIV", "P"))
1939 for _ in range(rng.choice((0, 1, 1, 2, 3)) if out else rng.randint(1, 3)):
1940 draw = rng.random()
1941 if draw < 0.25:
1942 out += "." + rng.choice(("a", "b", "c", "\\61", "A"))
1943 elif draw < 0.35:
1944 out += "#" + rng.choice(_IDS)
1945 elif draw < 0.6:
1946 name = rng.choice(("title", "data-x", "lang", "class", "id", "href", "TITLE"))
1947 operator = rng.choice(("", "=", "~=", "|=", "^=", "$=", "*="))
1948 value = rng.choice(("a", '"b c"', "'x'", '""', "e", "A", "en")) if operator else ""
1949 flag = rng.choice(("", "", " i", " s", " I")) if operator else ""
1950 out += f"[{name}{operator}{value}{flag}]"
1951 elif draw < 0.85 or depth >= 2:
1952 out += rng.choice(_PSEUDOS)
1953 else:
1954 function = rng.choice((":not", ":is", ":where", ":has", ":has", ":not"))
1955 argument = _selector(rng, depth + 1)
1956 out += f"{function}({rng.choice(('> ', '+ ', '~ ', '')) if function == ':has' else ''}{argument})"
1957 return out
1960def _vocabulary_tree(rng: random.Random, budget: int = 25) -> str:
1961 """Markup whose tags, classes, ids and attributes are the ones the selector and XPath generators draw from."""
1962 out: list[str] = []
1963 stack: list[str] = []
1964 for _ in range(rng.randint(3, budget)):
1965 if (draw := rng.random()) < 0.5:
1966 tag = rng.choice(_SELECTOR_TAGS)
1967 attrs = "".join(
1968 f' {name}="{rng.choice(values)}"'
1969 for name, values in (
1970 ("class", _CLASSES),
1971 ("id", _IDS),
1972 ("title", ("a", "b c", "A", "x", "")),
1973 ("lang", ("en", "en-US", "fr")),
1974 ("data-x", ("e", "a", "")),
1975 ("href", ("#", "x")),
1976 )
1977 if rng.random() < 0.25
1978 )
1979 out.append(f"<{tag}{attrs}>")
1980 stack.append(tag)
1981 elif draw < 0.75 and stack:
1982 out.append(f"</{stack.pop()}>")
1983 else:
1984 out.append(rng.choice(("x", "hello", " ", "", "<!--c-->", "y z")))
1985 return "".join(out)
1988_LENGTHS: Final = (*_VALUE_SPELLINGS["0"], *_VALUE_SPELLINGS["1.5px"], *_VALUE_SPELLINGS["10px"], "2em", "5%", "auto")
1989_COLORS: Final = (
1990 *_VALUE_SPELLINGS["red"], *_VALUE_SPELLINGS["white"], *_VALUE_SPELLINGS["black"], *_VALUE_SPELLINGS["navy"],
1991 *_VALUE_SPELLINGS["transparent"], "currentcolor", "currentColor", "inherit",
1992) # fmt: skip
1993_SEMANTIC_VALUES: Final[dict[str, tuple[str, ...]]] = {
1994 "color": _COLORS,
1995 "background-color": _COLORS,
1996 "border-top-color": _COLORS,
1997 "outline-color": _COLORS,
1998 "margin-top": _LENGTHS,
1999 "margin-right": _LENGTHS,
2000 "margin-bottom": _LENGTHS,
2001 "margin-left": _LENGTHS,
2002 "padding-top": _LENGTHS,
2003 "padding-left": _LENGTHS,
2004 "width": _LENGTHS,
2005 "height": _LENGTHS,
2006 "top": _LENGTHS,
2007 "text-indent": _LENGTHS,
2008 "letter-spacing": (*_LENGTHS, "normal"),
2009 "font-weight": ("bold", "700", "normal", "400", "bolder", "lighter", "100", "900"),
2010 "opacity": ("1", "+1", "01", "0.5", ".5", "0.50", "50%", "0", "0.0", "inherit"),
2011 "z-index": ("1", "+1", "01", "auto", "-1", "2", "1.0"),
2012 "display": ("block", "inline", "none", "flex", "BLOCK", "initial"),
2013 "text-align": ("left", "center", "right", "start", "justify"),
2014 "visibility": ("visible", "hidden", "collapse", "unset"),
2015 "border-top-style": ("solid", "dashed", "none"),
2016 "line-height": ("1.5", "normal", "10px", "150%", "1.50"),
2017 "font-size": ("10px", "1e1px", "medium", "small", "1.5em", "0"),
2018 "margin": (),
2019 "padding": (),
2020 "border": (),
2021 "border-color": (),
2022 "border-width": (),
2023 "border-style": (),
2024 "background": (),
2025 "outline": (),
2026}
2029def _declaration(rng: random.Random) -> str:
2030 name = rng.choice(list(_SEMANTIC_VALUES))
2031 if values := _SEMANTIC_VALUES[name]:
2032 value = rng.choice(values)
2033 elif name in {"margin", "padding", "border-width"}:
2034 value = " ".join(rng.choice(_LENGTHS) for _ in range(rng.randint(1, 4)))
2035 elif name == "border-color":
2036 value = " ".join(rng.choice(_COLORS[:-1]) for _ in range(rng.randint(1, 4)))
2037 elif name == "border-style":
2038 value = " ".join(rng.choice(("solid", "dashed", "none")) for _ in range(rng.randint(1, 4)))
2039 elif name == "background":
2040 value = rng.choice(_COLORS[:-1])
2041 else:
2042 value = f"{rng.choice(_LENGTHS[:-1])} {rng.choice(('solid', 'dashed'))} {rng.choice(_COLORS[:-1])}"
2043 return f"{name}:{value}{' !important' if rng.random() < 0.15 else ''}"
2046def _stylesheet(rng: random.Random) -> str:
2047 """Draw rules over a small selector and declaration pool, so the minifier finds rules and longhands to merge."""
2048 selectors = [_selector(rng) for _ in range(rng.randint(1, 5))]
2049 rules = []
2050 for _ in range(rng.randint(1, 8)):
2051 body = ";".join(_declaration(rng) for _ in range(rng.randint(1, 5)))
2052 if rng.random() < 0.2:
2053 side = rng.choice(("margin", "padding"))
2054 body += "".join(f";{side}-{edge}:{rng.choice(_LENGTHS)}" for edge in ("top", "right", "bottom", "left"))
2055 rule = f"{rng.choice(selectors)}{{{body}}}"
2056 rules.append(f"@media all{{{rule}}}" if rng.random() < 0.1 else rule)
2057 return rng.choice(("", "\n", " /*c*/ ")).join(rules)
2060def _generate_css(rng: random.Random) -> str:
2061 if rng.random() < 0.25:
2062 return css_generate(rng).data.decode("utf-8")
2063 if rng.random() < 0.5:
2064 return _mutate(rng, rng.choice(_corpus().css), _CSS_TOKENS)
2065 return _mutate(rng, _stylesheet(rng), _CSS_TOKENS) if rng.random() < 0.3 else _stylesheet(rng)
2068def _generate_css_fixpoint(rng: random.Random) -> str:
2069 source = _generate_css(rng)
2070 while not _css_valid_source(source):
2071 source = _stylesheet(rng)
2072 return source
2075def _seeds_css_fixpoint() -> list[str]:
2076 return [source for source in _seeds_css() if _css_valid_source(source)]
2079def _generate_style(rng: random.Random) -> str:
2080 body = ";".join(_declaration(rng) for _ in range(rng.randint(1, 6)))
2081 return _mutate(rng, body, _CSS_TOKENS) if rng.random() < 0.6 else body
2084def _generate_semantic(rng: random.Random) -> str:
2085 inline = "" if rng.random() < 0.7 else f' style="{_declaration(rng)}"'
2086 return f"{_root(rng)}<style>{_stylesheet(rng)}</style><div{inline}>{_vocabulary_tree(rng)}</div>"
2089def _root(rng: random.Random) -> str:
2090 """
2091 Open a no-quirks document whose root element sometimes carries attributes, so root-only matches show.
2093 Quirks mode matches class and id selectors case-insensitively (Selectors 4 §6.6, DOM §4.2.6.1), which a selector
2094 translated to XPath cannot know, so every generated document declares the standard mode.
2095 """
2096 attrs = "".join(
2097 f' {name}="{value}"' for name, value in (("lang", "en"), ("class", "a"), ("title", "x")) if rng.random() < 0.3
2098 )
2099 return f"<!DOCTYPE html><html{attrs}>"
2102def _generate_xpath_case(rng: random.Random) -> str:
2103 return json.dumps({"html": _root(rng) + _vocabulary_tree(rng), "xpath": _xpath(rng)})
2106def _generate_selector_case(rng: random.Random) -> str:
2107 return json.dumps({"html": _root(rng) + _vocabulary_tree(rng), "css": _selector(rng)})
2110_AXES: Final = (
2111 "child", "descendant", "descendant-or-self", "parent", "ancestor", "ancestor-or-self", "following-sibling",
2112 "preceding-sibling", "following", "preceding", "self", "attribute",
2113) # fmt: skip
2114_NODE_TESTS: Final = ("*", "node()", "text()", "comment()", "processing-instruction()", *_SELECTOR_TAGS)
2115_PREDICATES: Final = (
2116 "1", "2", "last()", "last()-1", "position()<3", "position()=last()", "@id", "@class='a'", "@title", "not(@id)",
2117 "name()='p'", "count(*)>1", "contains(.,'x')", "starts-with(@class,'a')", "string-length()>2", "text()='x'",
2118 "*", "b", "0", "true()", ".//a", "@*", "lang('en')", "normalize-space()", "-1", "1.5", "@data-x='e'",
2119) # fmt: skip
2122def _xpath(rng: random.Random) -> str:
2123 """Draw an XPath 1.0 expression: a location path, a union, or a function or arithmetic over paths."""
2124 path = _path(rng)
2125 return rng.choice((
2126 path,
2127 path,
2128 path,
2129 f"{path} | {_path(rng)}",
2130 f"count({path})",
2131 f"string({path})",
2132 f"sum({path})",
2133 f"boolean({path})",
2134 f"name({path})",
2135 f"({path})[{rng.choice(_PREDICATES)}]",
2136 f"count({path}) * 2 - 1 div 0",
2137 f"concat({path}, '-', {_path(rng)})",
2138 f"normalize-space({path})",
2139 f"number({path}) mod 3",
2140 f"{path} = {_path(rng)}",
2141 f"{path} != 'x'",
2142 "0 div 0",
2143 "-0",
2144 ))
2147def _path(rng: random.Random) -> str:
2148 steps = []
2149 for _ in range(rng.randint(1, 3)):
2150 if (draw := rng.random()) < 0.15:
2151 step = rng.choice(("..", ".", "@*", "@id", "@class"))
2152 else:
2153 axis = rng.choice(_AXES) + "::" if draw < 0.55 else ""
2154 step = axis + rng.choice(_NODE_TESTS)
2155 steps.append(step + "".join(f"[{rng.choice(_PREDICATES)}]" for _ in range(rng.choice((0, 0, 1, 2)))))
2156 separators = [rng.choice(("/", "//")) for _ in steps]
2157 return rng.choice(("/", "//", "", "")) + "".join(
2158 step if index == 0 else separators[index] + step for index, step in enumerate(steps)
2159 )
2162_JS_FREE: Final = ("console", "window", "Math", "JSON", "undefined", "NaN", "Infinity", "globalThis", "zz")
2163_JS_BOUND: Final = ("a", "b", "c", "foo", "bar", "$", "_x", "\\u0061b", "of", "async", "yield", "let")
2164_JS_STRINGS: Final = (
2165 "'a'", '"b"', "'it\\'s'", '"\\x41"', "'\\u{1F600}'", '"\\n"', "''", '"use strict"', "'\\0'", '"q\\\r\nr"',
2166 "'</script>'", '"\\u2028"', "'a\"b'", '"1"', "'if'",
2167) # fmt: skip
2168_JS_NUMBERS: Final = (
2169 "0",
2170 "1",
2171 "1.50",
2172 ".5",
2173 "5.",
2174 "1e3",
2175 "1E-7",
2176 "0x1F",
2177 "0o17",
2178 "0b101",
2179 "1_000",
2180 "123n",
2181 "0.1",
2182 "-0",
2183)
2184_JS_PROPERTIES: Final = ("a", "b", "if", "class", "length", "constructor", "1", "'q'", '"r s"', "0x10", "[c]", "1.5")
2187@dataclass
2188class _Script:
2189 """
2190 The state one generated script threads through: a source of fresh names and whether a ``return`` is legal.
2192 Fresh names keep ``let``/``const``/``class`` declarations from colliding, which acorn rejects as early errors;
2193 references draw from the declared and free pools, so most of them bind (Domato's scope-threading rule).
2194 """
2196 rng: random.Random
2197 declared: list[str] = field(default_factory=lambda: list(_JS_BOUND[:7]))
2198 counter: int = 0
2200 def fresh(self) -> str:
2201 self.counter += 1
2202 name = f"v{self.counter}"
2203 self.declared.append(name)
2204 return name
2206 def reference(self) -> str:
2207 return self.rng.choice((*_JS_FREE, *self.declared))
2210def _program(rng: random.Random) -> str:
2211 """Draw a script from statements and expressions chosen for printer hazards: ASI, operator spacing, keys."""
2212 script = _Script(rng)
2213 return "".join(_statement(script, 0, in_function=False) for _ in range(rng.randint(1, 6)))
2216def _end(rng: random.Random) -> str:
2217 return rng.choice((";", ";", ";", "\n", ";\n"))
2220def _statement(script: _Script, depth: int, *, in_function: bool) -> str:
2221 rng = script.rng
2223 def block(*, function: bool = in_function) -> str:
2224 statements = "".join(_statement(script, depth + 1, in_function=function) for _ in range(rng.randint(0, 3)))
2225 return "{" + statements + "}"
2227 def expression() -> str:
2228 return _expression(script, depth)
2230 choices: list[Callable[[], str]] = [
2231 lambda: f"var {rng.choice(_JS_BOUND)} = {expression()}{_end(rng)}",
2232 lambda: f"{rng.choice(('let', 'const'))} {script.fresh()} = {expression()}{_end(rng)}",
2233 lambda: f"{expression()}{_end(rng)}",
2234 lambda: f"{expression()}{_end(rng)}",
2235 ]
2236 if depth < 3:
2237 choices += [
2238 lambda: f"function {script.fresh()}({_parameters(rng)}){block(function=True)}",
2239 lambda: f"if ({expression()}) {block()} else {block()}",
2240 lambda: f"for (let {script.fresh()} = 0; {script.reference()} < {expression()}; i++) {block()}",
2241 lambda: f"for (const {script.fresh()} {rng.choice(('in', 'of'))} {expression()}) {block()}",
2242 lambda: f"try {block()} catch ({script.fresh()}) {block()} finally {block()}",
2243 lambda: f"switch ({expression()}) {{case {expression()}: {block()} default: break}}",
2244 lambda: f"{(label := script.fresh())}: for (;;) {{ break {label} }}",
2245 lambda: _class(script, depth),
2246 block,
2247 ]
2248 if in_function:
2249 choices.append(lambda: f"return{rng.choice((' ', chr(10)))}{expression()}{_end(rng)}")
2250 return rng.choice(choices)()
2253def _parameters(rng: random.Random) -> str:
2254 return ", ".join(rng.sample(_JS_BOUND[:7], rng.randint(0, 3)))
2257def _class(script: _Script, depth: int) -> str:
2258 rng = script.rng
2259 members = rng.sample(
2260 [
2261 f"m() {{ return {_expression(script, depth + 1)} }}",
2262 f"static s = {_expression(script, depth + 1)};",
2263 "#p = 1; get g() { return this.#p }",
2264 "'q'() {}",
2265 "1() {}",
2266 f"[{script.reference()}]() {{}}",
2267 "set h(v) {}",
2268 ],
2269 rng.randint(1, 4),
2270 )
2271 return f"class {script.fresh()} {rng.choice(('', 'extends Object '))}{{ {' '.join(members)} }}"
2274def _expression(script: _Script, depth: int) -> str:
2275 rng = script.rng
2276 if depth >= 3 or rng.random() < 0.35:
2277 return rng.choice((
2278 script.reference,
2279 lambda: rng.choice(_JS_STRINGS),
2280 lambda: rng.choice(_JS_NUMBERS),
2281 lambda: rng.choice(("true", "false", "null", "this", "/a+/g", "/[/]/", "`t`", "``")),
2282 ))()
2284 def sub() -> str:
2285 return _expression(script, depth + 1)
2287 def target() -> str:
2288 return rng.choice(script.declared[:7])
2290 return rng.choice((
2291 lambda: f"{sub()}.{rng.choice(('a', 'if', 'class', 'length'))}",
2292 lambda: f"{sub()}[{rng.choice(_JS_STRINGS + _JS_NUMBERS)}]",
2293 lambda: f"{sub()}?.{rng.choice(('a', 'b'))}",
2294 lambda: f"{sub()}?.[{sub()}]",
2295 lambda: f"{sub()}({', '.join(sub() for _ in range(rng.randint(0, 2)))})",
2296 lambda: f"new {rng.choice(_JS_FREE[:3])}({sub()})",
2297 lambda: f"({rng.choice(('typeof ', 'void ', '!', '-', '+', '~'))}{sub()})",
2298 lambda: f"delete {sub()}.a",
2299 lambda: f"({sub()} {rng.choice(_JS_BINARY)} {sub()})",
2300 lambda: f"{target()} {rng.choice(('+ +', '- -', '+ ++', '- --', '/ /x/ /'))}{target()}",
2301 lambda: f"({sub()} ? {sub()} : {sub()})",
2302 lambda: f"(({', '.join(rng.sample(_JS_BOUND[:5], rng.randint(0, 2)))}) => {sub()})",
2303 lambda: f"({{{rng.choice(_JS_PROPERTIES)}: {sub()}, {target()}, m() {{ return {sub()} }}, ...{sub()}}})",
2304 lambda: f"[{sub()}, , ...{sub()}]",
2305 lambda: f"({target()} {rng.choice(('=', '+=', '??=', '||=', '**='))} {sub()})",
2306 lambda: f"`a${{{sub()}}}b${{{sub()}}}`",
2307 lambda: f"{rng.choice(_JS_FREE[:3])}`x\\x41${{{sub()}}}`",
2308 lambda: f"(function {rng.choice(('', 'f'))}() {{ return {sub()} }})()",
2309 lambda: f"{rng.choice(_JS_NUMBERS[:6])} .toString()",
2310 lambda: f"([{target()}, {target()}] = {sub()})",
2311 lambda: f"({{a: {target()}, b = 1}} = {sub()})",
2312 lambda: f"{target()}++",
2313 ))()
2316_JS_BINARY: Final = (
2317 "+", "-", "*", "/", "%", "**", "==", "===", "!=", "<", ">=", "&&", "||", "??", "&", "|", "^", "<<", ">>>", "in",
2318 "instanceof", ",",
2319) # fmt: skip
2322_JS_TOKENS: Final = ("\n", ";", "(", ")", "{", "}", ",", "+", "-", "/", "'", "`", "${", "in ", " of ", "=>", ".", "?.")
2325def _generate_js(rng: random.Random) -> str:
2326 if rng.random() < 0.25:
2327 return _mutate(rng, rng.choice(_corpus().js), _JS_TOKENS)
2328 return _program(rng)
2331def _generate_url(rng: random.Random) -> str:
2332 host = rng.choice(("Example.COM", "bücher.example", "127.0.0.1", "[::1]", "sub.example.org"))
2333 path = "/".join(rng.choices(("a", "..", ".", "%2e", "café", "a b", "%7e", ""), k=rng.randint(1, 6)))
2334 query = rng.choice(("", "?b=2&a=1", "?utm_source=x&id=2", "?q=a+b", "?q=%E2%9C%93"))
2335 fragment = rng.choice(("", "#frag", "#a b"))
2336 url = f"{rng.choice(('http', 'HTTPS'))}://{host}{rng.choice(('', ':80', ':443', ':8080'))}/{path}{query}{fragment}"
2337 return rng.choice((url, f" <{url}> ", url.replace("&", "&")))
2340# RFC 3986 section 5.4 supplies independent targets; safe HTTP leaves avoid scheme-specific edge policies.
2341_RESOLUTION_BASE: Final = "http://example.test/b/c/d;p?q"
2342_RESOLUTION_CASES: Final[dict[str, tuple[str, str]]] = {
2343 "child": ("{leaf}", "http://example.test/b/c/{leaf}"),
2344 "current": ("./{leaf}", "http://example.test/b/c/{leaf}"),
2345 "parent": ("../{leaf}", "http://example.test/b/{leaf}"),
2346 "grandparent": ("../../{leaf}", "http://example.test/{leaf}"),
2347 "root": ("/{leaf}", "http://example.test/{leaf}"),
2348 "query": ("?{leaf}", "http://example.test/b/c/d;p?{leaf}"),
2349 "fragment": ("#{leaf}", "http://example.test/b/c/d;p?q#{leaf}"),
2350 "directory": ("{leaf}/", "http://example.test/b/c/{leaf}/"),
2351}
2354def _generate_resolution(rng: random.Random) -> str:
2355 return f"{rng.choice(tuple(_RESOLUTION_CASES))}:g{rng.randrange(10000)}"
2358def _resolution_controls() -> dict[str, bool]:
2359 return {
2360 "unchanged relative link": resolve_links_check("parent:g", lambda _node, _base: None) is not None,
2361 "stable wrong target": resolve_links_check("parent:g", _resolve_wrong_target) is not None,
2362 "changed repeat": resolve_links_check("parent:g", _resolve_changed_repeat) is not None,
2363 }
2366def _resolve_wrong_target(node: Node, _base: str) -> None:
2367 node.select("a")[0].attrs["href"] = "http://wrong.example/"
2370def _resolve_changed_repeat(node: Node, base: str) -> None:
2371 if node.links()[0].url.startswith("http:"):
2372 node.select("a")[0].attrs["href"] = node.links()[0].url + "a"
2373 else:
2374 node.resolve_links(base)
2377def _generate_encoding(rng: random.Random) -> str:
2378 variant: Final = rng.choice(tuple(_ENCODING_FORMATS))
2379 alphabet: Final = "abc 123café€" if _ENCODING_FORMATS[variant][0] == "cp1252" else "abc 123café€中文𐐀"
2380 return f"{variant}\n{''.join(rng.choices(alphabet, k=rng.randint(0, 64)))}"
2383def _encoding_stream_controls() -> dict[str, bool]:
2384 missing: Final = EncodingMatch(None, 0.0, None)
2385 return {
2386 "wrong one-shot label": encoding_stream_check("utf-8-bom\ncafé", lambda _data: missing) is not None,
2387 "wrong chunked label": encoding_stream_check("utf-8-bom\ncafé", stream=lambda _data, _width: missing)
2388 is not None,
2389 }
2392def _encoding_decode_controls() -> dict[str, bool]:
2393 return {
2394 "wrong codec text": encoding_decode_check("utf-8-meta\ncafé", decode=lambda _data, _codec: "") is not None,
2395 "wrong parsed text": encoding_decode_check("utf-8-meta\ncafé", parse_text=lambda _data: "") is not None,
2396 "wrong detected label": encoding_decode_check(
2397 "utf-8-meta\ncafé", detect_bytes=lambda _data: EncodingMatch(None, 0.0, None)
2398 )
2399 is not None,
2400 }
2403def _encoding_seeds() -> list[str]:
2404 return ["empty\n", *(f"{variant}\ncafé €" for variant in _ENCODING_FORMATS), *_generated(_generate_encoding, 150)()]
2407def _generate_idna_host(rng: random.Random) -> str:
2408 return f"{rng.choice(tuple(_IDNA_LABELS))}\n{''.join(rng.choices('abc012', k=rng.randint(0, 8)))}"
2411def _idna_host_seeds() -> list[str]:
2412 return [*(f"{variant}\n" for variant in _IDNA_LABELS), *_generated(_generate_idna_host, 150)()]
2415def _idna_host_controls() -> dict[str, bool]:
2416 return {
2417 "unchanged Unicode host": idna_host_check("umlaut\n", lambda text: text) is not None,
2418 "wrong mapping": idna_host_check("umlaut\n", lambda _text: "https://wrong.example/") is not None,
2419 "non-NFC label": idna_host_check("umlaut\n", lambda _text: "https://xn--u-ccb.example/") is not None,
2420 "changed canonical output": idna_host_check(
2421 "umlaut\n", lambda text: normalize_url(text) if not text.isascii() else text + "a"
2422 )
2423 is not None,
2424 }
2427def _normalize_url_controls() -> dict[str, bool]:
2428 return {
2429 "growing output": normalize_url_check("https://example.org/", lambda text: text + "a") is not None,
2430 "rejects normalized output": normalize_url_check("raw", _reject_url_output) is not None,
2431 }
2434def _url_reparse_controls() -> dict[str, bool]:
2435 return {
2436 "stable uppercased scheme": url_reparse_check("raw", lambda _text: "HTTPS://example.test/") is not None,
2437 "changes after recomposition": url_reparse_check(
2438 "raw", lambda text: "https://example.test/" if text == "raw" else "https://changed.test/"
2439 )
2440 is not None,
2441 "rejects recomposed output": url_reparse_check("raw", _reject_url_output) is not None,
2442 }
2445def _reject_url_output(text: str) -> str:
2446 if text == "raw":
2447 return "https://example.org/"
2448 msg = "rejected"
2449 raise ValueError(msg)
2452def _clean_url_controls() -> dict[str, bool]:
2453 return {
2454 "growing output": clean_url_check("https://example.org/", lambda text: text + "a") is not None,
2455 "drops cleaned output": clean_url_check("raw", lambda text: "https://example.org/" if text == "raw" else None)
2456 is not None,
2457 "rejects cleaned output": clean_url_check("raw", _reject_url_output) is not None,
2458 }
2461def _seeds_url() -> list[str]:
2462 # WPT supplies inputs: crawl cleaning differs from WHATWG serialization.
2463 wpt = _require(_ROOT / "tools/fuzz-data/wpt/url/resources/urltestdata.json")
2464 return list(
2465 dict.fromkeys([
2466 *(row["input"] for row in json.loads(wpt.read_text(encoding="utf-8")) if isinstance(row, dict)),
2467 *_texts((_ROOT / "tools/fuzz/corpus/url").iterdir()),
2468 *_generated(_generate_url, 500)(),
2469 ])
2470 )
2473def _html_controls() -> dict[str, bool]:
2474 return {
2475 "unescaped text": html_check("<p><b>x</p>", lambda node: _unescape(node.serialize())) is not None,
2476 "text after an end tag": html_check("<b>x</b>y", lambda node: node.serialize().replace("</b>", "</b> "))
2477 is not None,
2478 }
2481def _unescape(text: str) -> str:
2482 return text.replace("<", "<").replace(">", ">")
2485def _xml_controls() -> dict[str, bool]:
2486 def xml(node: Node) -> str:
2487 return node.serialize(Html(xml=True))
2489 return {
2490 "unescaped ampersand": xml_check("<p>a&b</p>", lambda node: xml(node).replace("&", "&")) is not None,
2491 "dropped attribute": xml_check("<p a=1>x</p>", lambda node: xml(node).replace(' a="1"', "")) is not None,
2492 }
2495def _css_controls() -> dict[str, bool]:
2496 return {
2497 "growing output": _css_fixpoint("a{color:red}", lambda css: minify_css(css) + "/*!x*/") is not None,
2498 "late non-numeric convergence": fixpoint_check("aa", lambda text: text.replace("a", "b", 1), numeric=True)
2499 is not None,
2500 }
2503def _js_controls() -> dict[str, bool]:
2504 return {
2505 "growing output": fixpoint_check("x", lambda source: minify_js(source, _JS_OFF) + "\n0", numeric=True)
2506 is not None
2507 }
2510def _style_controls() -> dict[str, bool]:
2511 return {
2512 "growing output": style_check("a:b", lambda text: StyleDeclaration.parse(text + ";z:1")) is not None,
2513 "changed value": style_check("a:b", lambda text: StyleDeclaration.parse(text.replace(": ", ":x", 1)))
2514 is not None,
2515 }
2518def _markdown_controls() -> dict[str, bool]:
2519 return {"unescaped text": markdown_check("<p>*a*</p>", lambda node: node.text) is not None}
2522def _js_names_controls() -> dict[str, bool]:
2523 def broken(edit: Callable[[str], str]) -> Callable[[str, JSMinify], str]:
2524 return lambda source, options: edit(minify_js(source, options))
2526 return {
2527 "invented free identifier": js_names_check("x", _JS_NAMES, broken(lambda out: out + ";zz")) is not None,
2528 "lost string": js_names_check("f('q')", _JS_NAMES, broken(lambda out: re.sub(r"['\"]q['\"]", "0", out)))
2529 is not None,
2530 "captured free identifier": js_names_check(
2531 "function f(a){return b}", _JS_NAMES, lambda _source, _options: "function f(b){return b}"
2532 )
2533 is not None,
2534 "output not JavaScript": js_names_check("x", _JS_NAMES, broken(lambda out: out + "(")) is not None,
2535 }
2538def _css_semantics_controls() -> dict[str, bool]:
2539 return {
2540 "changed value": css_semantics_check(
2541 "<style>p{color:red}</style><p>x", lambda css: minify_css(css).replace("red", "#00f")
2542 )
2543 is not None,
2544 "rewritten selector": css_semantics_check(
2545 "<style>div>b{x-unknown:1}</style><div><p><b>x</b></p></div>", lambda css: minify_css(css).replace(">", " ")
2546 )
2547 is not None,
2548 }
2551def _xpath_controls() -> dict[str, bool]:
2552 case = json.dumps({"html": "<p></p><p></p>", "xpath": "//p"})
2553 return {
2554 "xpath_iter drops an item": xpath_entry_check(
2555 case, iterate=lambda node, expression: list(node.xpath_iter(expression))[1:]
2556 )
2557 is not None,
2558 "xpath_one returns the last": xpath_entry_check(
2559 case, first=lambda node, expression: list(node.xpath_iter(expression))[-1]
2560 )
2561 is not None,
2562 }
2565def _selector_controls() -> dict[str, bool]:
2566 case = json.dumps({"html": "<p>x</p>", "css": "p"})
2567 return {
2568 "translated to the child axis": selector_entry_check(
2569 case, lambda selector: css_to_xpath(selector, prefix="child::")
2570 )
2571 is not None
2572 }
2575def _span_controls() -> dict[str, bool]:
2576 def broken(edit: Callable[[SourceLocation], SourceLocation]) -> Callable[[Element], SourceLocation | None]:
2577 return lambda element: None if (location := element.source_location) is None else edit(location)
2579 def one_late(location: SourceLocation) -> SourceLocation:
2580 start = location.start_tag
2581 return location._replace(
2582 start_tag=start._replace(start_offset=start.start_offset + 1, start_col=start.start_col + 1)
2583 )
2585 return {
2586 "start one late": span_check("<p class=a>x</p>", broken(one_late)) is not None,
2587 "end tag at the start tag": span_check(
2588 "<p class=a>x</p>", broken(lambda location: location._replace(end_tag=location.start_tag))
2589 )
2590 is not None,
2591 }
2594def _seeds_html() -> list[str]:
2595 return list(_corpus().html)
2598def _seeds_css() -> list[str]:
2599 return list(_corpus().css)
2602def _seeds_js() -> list[str]:
2603 return list(_corpus().js)
2606def _seeds_style() -> list[str]:
2607 return [body for css in _corpus().css for body in re.findall(r"\{([^{}]*)\}", css)]
2610def _parser_bytes_check(case: str) -> str | None:
2611 try:
2612 return parser_bytes_check(case)
2613 except UnsupportedParserCaseError as error:
2614 raise OutOfScopeError from error
2617def _xml_island(case: str) -> str | None:
2618 try:
2619 return xml_island_check(case)
2620 except UnsupportedXmlIslandCaseError as error:
2621 raise OutOfScopeError(str(error)) from error
2624def _xml_literal(case: str) -> str | None:
2625 try:
2626 return xml_literal_check(case)
2627 except UnsupportedXmlLiteralCaseError as error:
2628 raise OutOfScopeError(str(error)) from error
2631ORACLES: Final[dict[str, Oracle]] = {
2632 "encoding-stream": Oracle(
2633 encoding_stream_check, _generate_encoding, _encoding_seeds, _encoding_stream_controls, Floor(100, 0.95)
2634 ),
2635 "parser-bytes": Oracle(
2636 _parser_bytes_check, parser_bytes_generate, parser_bytes_seeds, parser_bytes_controls, Floor(100, 1)
2637 ),
2638 "encoding-decode": Oracle(
2639 encoding_decode_check, _generate_encoding, _encoding_seeds, _encoding_decode_controls, Floor(100, 0.95)
2640 ),
2641 "css-custom-tokens": Oracle(
2642 _css_custom_check, css_custom_generate, css_custom_seeds, css_custom_controls, Floor(100, 1)
2643 ),
2644 "observer-sequence": Oracle(
2645 _observer_sequence_check,
2646 observer_sequence_generate,
2647 observer_sequence_seeds,
2648 observer_sequence_controls,
2649 Floor(100, 1),
2650 ),
2651 "dom-program": Oracle(
2652 _dom_program_check, dom_program_generate, dom_program_seeds, dom_program_controls, Floor(100, 1)
2653 ),
2654 "dom-lifecycle": Oracle(
2655 _dom_lifecycle_check, dom_lifecycle_generate, dom_lifecycle_seeds, dom_lifecycle_controls, Floor(200, 1)
2656 ),
2657 "xml-island-grammar": Oracle(_xml_island, xml_island_generate, xml_island_seeds, xml_island_controls, Floor(54, 1)),
2658 "xml-literal-grammar": Oracle(
2659 _xml_literal, xml_literal_generate, xml_literal_seeds, xml_literal_controls, Floor(96, 1)
2660 ),
2661 "iterator-sequence": Oracle(
2662 _iterator_sequence_check,
2663 iterator_sequence_generate,
2664 iterator_sequence_seeds,
2665 iterator_sequence_controls,
2666 Floor(100, 1),
2667 ),
2668 "normalize-url-fixpoint": Oracle(
2669 normalize_url_check, _generate_url, _seeds_url, _normalize_url_controls, Floor(100, 0.5)
2670 ),
2671 "url-split-reparse": Oracle(url_reparse_check, _generate_url, _seeds_url, _url_reparse_controls, Floor(100, 0.5)),
2672 "idna-host": Oracle(idna_host_check, _generate_idna_host, _idna_host_seeds, _idna_host_controls, Floor(100, 0.95)),
2673 "idna-nfc": Oracle(_idna_nfc_check, idna_nfc_generate, idna_nfc_seeds, idna_nfc_controls, Floor(100, 1)),
2674 "clean-url-fixpoint": Oracle(clean_url_check, _generate_url, _seeds_url, _clean_url_controls, Floor(100, 0.25)),
2675 "resolve-links": Oracle(
2676 resolve_links_check,
2677 _generate_resolution,
2678 _generated(_generate_resolution, 500),
2679 _resolution_controls,
2680 Floor(100, 1),
2681 syntax="html",
2682 ),
2683 "html-fixpoint": Oracle(html_check, _generate_html, _seeds_html, _html_controls, Floor(500, 0.85), syntax="html"),
2684 "html-table-grammar": Oracle(
2685 _html_table_check,
2686 html_table_generate,
2687 html_table_seeds,
2688 html_table_controls,
2689 Floor(24, 1),
2690 syntax="html",
2691 ),
2692 "html-sibling-grammar": Oracle(
2693 _html_sibling_check,
2694 html_sibling_generate,
2695 html_sibling_seeds,
2696 html_sibling_controls,
2697 Floor(100, 1),
2698 syntax="html",
2699 ),
2700 "html-foreign-grammar": Oracle(
2701 _html_foreign_check,
2702 html_foreign_generate,
2703 html_foreign_seeds,
2704 html_foreign_controls,
2705 Floor(36, 1),
2706 syntax="html",
2707 ),
2708 "html-list-grammar": Oracle(
2709 _html_list_check,
2710 html_list_generate,
2711 html_list_seeds,
2712 html_list_controls,
2713 Floor(24, 1),
2714 syntax="html",
2715 ),
2716 "xml-document-grammar": Oracle(
2717 xml_document_check, xml_source_generate, xml_source_seeds, xml_source_controls, Floor(60, 1)
2718 ),
2719 "xml-fixpoint": Oracle(xml_check, _generate_html, _seeds_html, _xml_controls, Floor(500, 0.95)),
2720 "css-sheet-grammar": Oracle(
2721 _css_fixpoint, css_source_generate, css_source_seeds, _css_controls, Floor(6356, 1), syntax="css"
2722 ),
2723 "css-fixpoint": Oracle(
2724 _css_valid_fixpoint, _generate_css_fixpoint, _seeds_css_fixpoint, _css_controls, Floor(500, 0.95), syntax="css"
2725 ),
2726 "js-fixpoint": Oracle(_js_fixpoint, _generate_js, _seeds_js, _js_controls, Floor(300, 0.6), syntax="js"),
2727 "style-fixpoint": Oracle(
2728 style_check, _generate_style, _seeds_style, _style_controls, Floor(300, 0.95), syntax="css"
2729 ),
2730 "markdown-source-grammar": Oracle(
2731 _markdown_source_check,
2732 _markdown_source_generate,
2733 markdown_source_seeds,
2734 markdown_controls,
2735 Floor(48, 1),
2736 ),
2737 "markdown-html-grammar": Oracle(
2738 _markdown_html_check,
2739 _markdown_html_generate,
2740 partial(markdown_source_seeds, html=True),
2741 partial(markdown_controls, html=True),
2742 Floor(29, 1),
2743 syntax="html",
2744 ),
2745 "markdown-fixpoint": Oracle(
2746 markdown_check,
2747 _markdown_document,
2748 _generated(_markdown_document, 500),
2749 _markdown_controls,
2750 Floor(500, 0.95),
2751 syntax="html",
2752 ),
2753 "js-names": Oracle(_js_names, _generate_js, _seeds_js, _js_names_controls, Floor(300, 0.6), syntax="js"),
2754 "css-semantics": Oracle(
2755 css_semantics_check,
2756 _generate_semantic,
2757 _generated(_generate_semantic),
2758 _css_semantics_controls,
2759 Floor(100, 0.95),
2760 syntax="html",
2761 ),
2762 "xpath-entry": Oracle(
2763 xpath_entry_check,
2764 _generate_xpath_case,
2765 _generated(_generate_xpath_case),
2766 _xpath_controls,
2767 Floor(100, 0.95),
2768 fields=("html", "xpath"),
2769 ),
2770 "selector-entry": Oracle(
2771 selector_entry_check,
2772 _generate_selector_case,
2773 _generated(_generate_selector_case),
2774 _selector_controls,
2775 Floor(100, 0.85),
2776 fields=("html", "css"),
2777 ),
2778 "spans": Oracle(span_check, _generate_html, _seeds_html, _span_controls, Floor(500, 0.95), syntax="html"),
2779}
2782if __name__ == "__main__":
2783 raise SystemExit(main())