Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import re
4from typing import (
5 Any,
6 Dict,
7 List,
8 Match,
9 MutableMapping,
10 Optional,
11 Set,
12 Tuple,
13)
15from ._inline.emphasis import finalize_emphasis_tokens, is_entity_boundary
16from ._inline.links import parse_link as parse_inline_link
17from .core import InlineState, Parser
18from .helpers import (
19 HTML_ATTRIBUTES,
20 HTML_TAGNAME,
21 PUNCTUATION,
22 unescape_char,
23)
24from .util import escape_url
26_REGEX_META_CHARS = set(r"()[]{}?*+|.^$")
27DEFAULT_MAX_EMPHASIS_DEPTH = 20
28DEFAULT_MAX_IMAGE_DEPTH = 20
30AUTO_EMAIL = (
31 r"""<[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]"""
32 r"(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?"
33 r"(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*>"
34)
36INLINE_HTML = (
37 r"<" + HTML_TAGNAME + HTML_ATTRIBUTES + r"\s*/?>|" # open tag
38 r"</" + HTML_TAGNAME + r"\s*>|" # close tag
39 r"<!--(?!>|->)(?:(?!--)[\s\S])+?(?<!-)-->|" # comment
40 r"<\?[\s\S]+?\?>|" # script like <?php?>
41 r"<![A-Z][\s\S]+?>|" # doctype
42 r"<!\[CDATA[\s\S]+?\]\]>" # cdata
43)
46class InlineParser(Parser[InlineState]):
47 sc_flag = 0
48 state_cls = InlineState
50 #: linebreak leaves two spaces at the end of line
51 STD_LINEBREAK = r"(?:\\| {2,})\n\s*"
53 #: every new line becomes <br>
54 HARD_LINEBREAK = r" *\n\s*"
56 # we only need to find the start pattern of an inline token
57 SPECIFICATION = {
58 # e.g. \`, \$
59 "escape": r"(?:\\" + PUNCTUATION + ")+",
60 # `code, ```code
61 "codespan": r"`{1,}",
62 # *w, **w, _w, __w
63 "emphasis": r"\*{1,3}(?=[^\s*])|\b_{1,3}(?=[^\s_])",
64 # [link], ![img]
65 "link": r"!?\[",
66 # <https://example.com>. regex copied from commonmark.js
67 "auto_link": r"<[A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*>",
68 "auto_email": AUTO_EMAIL,
69 "inline_html": INLINE_HTML,
70 "linebreak": STD_LINEBREAK,
71 "softbreak": HARD_LINEBREAK,
72 "prec_auto_link": r"<[A-Za-z][A-Za-z\d.+-]{1,31}:",
73 "prec_inline_html": r"</?" + HTML_TAGNAME + r"|<!|<\?",
74 }
75 DEFAULT_RULES = (
76 "escape",
77 "codespan",
78 "emphasis",
79 "link",
80 "auto_link",
81 "auto_email",
82 "inline_html",
83 "linebreak",
84 )
86 def __init__(
87 self,
88 hard_wrap: bool = False,
89 max_emphasis_depth: int = DEFAULT_MAX_EMPHASIS_DEPTH,
90 max_image_depth: int = DEFAULT_MAX_IMAGE_DEPTH,
91 ) -> None:
92 super(InlineParser, self).__init__()
94 self.hard_wrap = hard_wrap
95 self.max_emphasis_depth = max_emphasis_depth
96 self.max_image_depth = max_image_depth
97 self._fast_trigger_chars: Optional[Set[str]] = None
98 self._fast_trigger_re: Optional[re.Pattern[str]] = None
99 self._fast_trigger_re_chars: Optional[Tuple[str, ...]] = None
100 # lazy add linebreak
101 if hard_wrap:
102 self.specification["linebreak"] = self.HARD_LINEBREAK
103 else:
104 self.rules.append("softbreak")
106 self._methods = {name: getattr(self, "parse_" + name) for name in self.rules}
108 def register(
109 self,
110 name: str,
111 pattern: Optional[str],
112 func: Any,
113 before: Optional[str] = None,
114 ) -> None:
115 super().register(name, pattern, func, before=before)
116 self._fast_trigger_chars = None
117 self._fast_trigger_re = None
118 self._fast_trigger_re_chars = None
120 def parse_escape(self, m: Match[str], state: InlineState) -> int:
121 text = m.group(0)
122 text = unescape_char(text)
123 self.process_text(text, state, parse_emphasis=False)
124 return m.end()
126 def parse_link(self, m: Match[str], state: InlineState) -> Optional[int]:
127 return parse_inline_link(self, m, state)
129 def parse_auto_link(self, m: Match[str], state: InlineState) -> int:
130 text = m.group(0)
131 pos = m.end()
132 if state.in_link:
133 self.process_text(text, state)
134 return pos
136 text = text[1:-1]
137 self._add_auto_link(text, text, state)
138 return pos
140 def parse_auto_email(self, m: Match[str], state: InlineState) -> int:
141 text = m.group(0)
142 pos = m.end()
143 if state.in_link:
144 self.process_text(text, state)
145 return pos
147 text = text[1:-1]
148 url = "mailto:" + text
149 self._add_auto_link(url, text, state)
150 return pos
152 def _add_auto_link(self, url: str, text: str, state: InlineState) -> None:
153 state.append_token(
154 {
155 "type": "link",
156 "children": [{"type": "text", "raw": text}],
157 "attrs": {"url": escape_url(url)},
158 }
159 )
161 def parse_emphasis(self, m: Match[str], state: InlineState) -> int:
162 # Keep a delimiter run separate from the preceding text token. The
163 # emphasis finalizer needs to inspect these markers later, and merging
164 # one-character markers with a growing text token makes inputs such as
165 # ``*a*a*a`` repeatedly copy the whole accumulated string.
166 marker = m.group(0)
167 if len(marker) == 1:
168 state.append_token({"type": "text", "raw": marker})
169 else:
170 self.process_text(marker, state)
171 return m.end()
173 def parse_codespan(self, m: Match[str], state: InlineState) -> int:
174 marker = m.group(0)
175 # require same marker with same length at end
177 pattern = re.compile(r"(.*?[^`])" + marker + r"(?!`)", re.S)
179 pos = m.end()
180 m2 = pattern.match(state.src, pos)
181 if m2:
182 end_pos = m2.end()
183 code = m2.group(1)
184 # Line endings are treated like spaces
185 code = code.replace("\n", " ")
186 if len(code.strip()):
187 if code.startswith(" ") and code.endswith(" "):
188 code = code[1:-1]
189 state.append_token({"type": "codespan", "raw": code})
190 return end_pos
191 else:
192 state.append_token({"type": "text", "raw": marker})
193 return pos
195 def parse_linebreak(self, m: Match[str], state: InlineState) -> int:
196 state.append_token({"type": "linebreak"})
197 return m.end()
199 def parse_softbreak(self, m: Match[str], state: InlineState) -> int:
200 state.append_token({"type": "softbreak"})
201 return m.end()
203 def parse_inline_html(self, m: Match[str], state: InlineState) -> int:
204 end_pos = m.end()
205 html = m.group(0)
206 state.append_token({"type": "inline_html", "raw": html})
207 if html.startswith(("<a ", "<a>", "<A ", "<A>")):
208 state.in_link = True
209 elif html.startswith(("</a ", "</a>", "</A ", "</A>")):
210 state.in_link = False
211 return end_pos
213 def process_text(self, text: str, state: InlineState, parse_emphasis: bool = True) -> None:
214 if (
215 parse_emphasis
216 and state.tokens
217 and state.tokens[-1]["type"] == "text"
218 and state.tokens[-1].get("_emphasis", True)
219 and not is_entity_boundary(state.tokens[-1]["raw"], text)
220 ):
221 state.tokens[-1]["raw"] += text
222 else:
223 token: Dict[str, Any] = {"type": "text", "raw": text}
224 if not parse_emphasis:
225 token["_emphasis"] = False
226 state.append_token(token)
228 def parse(self, state: InlineState) -> List[Dict[str, Any]]:
229 pos = 0
230 sc = self.compile_sc()
231 while pos < len(state.src):
232 fast_end = self._find_fast_text_end(state.src, pos)
233 if fast_end is None:
234 m = sc.search(state.src, pos)
235 else:
236 if fast_end > pos:
237 self.process_text(state.src[pos:fast_end], state)
238 pos = fast_end
239 if pos >= len(state.src):
240 break
241 m = sc.match(state.src, pos)
243 if not m:
244 if fast_end is not None:
245 self.process_text(state.src[pos : pos + 1], state)
246 pos += 1
247 continue
248 break
250 end_pos = m.start()
251 if end_pos > pos:
252 hole = state.src[pos:end_pos]
253 self.process_text(hole, state)
255 new_pos = self.parse_method(m, state)
256 if not new_pos:
257 # move cursor 1 character forward
258 pos = end_pos + 1
259 hole = state.src[end_pos:pos]
260 self.process_text(hole, state)
261 else:
262 pos = new_pos
264 if pos == 0:
265 # special case, just pure text
266 self.process_text(state.src, state)
267 elif pos < len(state.src):
268 self.process_text(state.src[pos:], state)
269 state.tokens = finalize_emphasis_tokens(
270 state.tokens,
271 "emphasis" in self.rules,
272 self.max_emphasis_depth,
273 )
274 return state.tokens
276 def _find_fast_text_end(self, src: str, pos: int) -> Optional[int]:
277 chars = self._get_fast_trigger_chars()
278 if chars is None:
279 return None
281 trigger_re = self._get_fast_trigger_re(chars)
282 m = trigger_re.search(src, pos)
283 if m is None:
284 return len(src)
286 if m.group(0) == "\n":
287 return self._find_linebreak_start(src, pos, m.start())
288 return m.start()
290 def _get_fast_trigger_re(self, chars: Set[str]) -> re.Pattern[str]:
291 key = tuple(sorted(chars))
292 if self._fast_trigger_re is None or self._fast_trigger_re_chars != key:
293 pattern = "[" + re.escape("".join(key)) + "]"
294 self._fast_trigger_re = re.compile(pattern)
295 self._fast_trigger_re_chars = key
296 assert self._fast_trigger_re is not None
297 return self._fast_trigger_re
299 def _find_linebreak_start(self, src: str, min_pos: int, newline_pos: int) -> int:
300 pos = newline_pos
301 while pos > min_pos and src[pos - 1] == " ":
302 pos -= 1
303 if pos == newline_pos and pos > min_pos and src[pos - 1] == "\\":
304 return pos - 1
305 return pos
307 def _get_fast_trigger_chars(self) -> Optional[Set[str]]:
308 chars = self._fast_trigger_chars
309 if chars is not None:
310 return chars
312 chars = set()
313 for name in self.rules:
314 pattern = self.specification.get(name)
315 rule_chars = _get_rule_start_chars(name, pattern)
316 if rule_chars is None:
317 self._fast_trigger_chars = None
318 return None
319 chars.update(rule_chars)
320 self._fast_trigger_chars = chars
321 return chars
323 def precedence_scan(
324 self,
325 m: Match[str],
326 state: InlineState,
327 end_pos: int,
328 rules: Optional[List[str]] = None,
329 ) -> Optional[int]:
330 if rules is None:
331 rules = ["codespan", "link", "prec_auto_link", "prec_inline_html"]
333 mark_pos = m.end()
334 sc = self.compile_sc(rules)
335 m1 = sc.search(state.src, mark_pos, end_pos)
336 if not m1:
337 return None
339 lastgroup = m1.lastgroup
340 if not lastgroup:
341 return None
342 rule_name = lastgroup.replace("prec_", "")
343 sc = self.compile_sc([rule_name])
344 m2 = sc.match(state.src, m1.start())
345 if not m2:
346 return None
348 func = self._methods[rule_name]
349 new_state = state.copy()
350 new_state.src = state.src
351 m2_pos = func(m2, new_state)
352 if not m2_pos or m2_pos < end_pos:
353 return None
355 raw_text = state.src[m.start() : m2.start()]
356 state.append_token({"type": "text", "raw": raw_text})
357 for token in new_state.tokens:
358 state.append_token(token)
359 return m2_pos
361 def render(self, state: InlineState) -> List[Dict[str, Any]]:
362 self.parse(state)
363 return state.tokens
365 def __call__(self, s: str, env: MutableMapping[str, Any]) -> List[Dict[str, Any]]:
366 state = self.state_cls(env)
367 state.src = s
368 return self.render(state)
371def _get_rule_start_chars(name: str, pattern: Optional[str]) -> Optional[Set[str]]:
372 known = {
373 "escape": {"\\"},
374 "codespan": {"`"},
375 "emphasis": {"*", "_"},
376 "link": {"!", "["},
377 "auto_link": {"<"},
378 "auto_email": {"<"},
379 "inline_html": {"<"},
380 "linebreak": {"\n"},
381 "softbreak": {"\n"},
382 "prec_auto_link": {"<"},
383 "prec_inline_html": {"<"},
384 # built-in plugins
385 "url_link": {"h"},
386 "strikethrough": {"~"},
387 "mark": {"="},
388 "insert": {"^"},
389 "superscript": {"^"},
390 "subscript": {"~"},
391 "footnote": {"["},
392 "inline_math": {"$"},
393 "ruby": {"["},
394 "inline_spoiler": {">"},
395 }
396 if name in known:
397 return known[name]
398 if not pattern:
399 return set()
400 return _guess_pattern_start_chars(pattern)
403def _guess_pattern_start_chars(pattern: str) -> Optional[Set[str]]:
404 if not pattern:
405 return set()
407 if pattern.startswith("\\") and len(pattern) > 1:
408 c = pattern[1]
409 if c in _REGEX_META_CHARS or c in PUNCTUATION:
410 return {c}
411 return None
413 c = pattern[0]
414 if c in _REGEX_META_CHARS or c.isspace():
415 return None
416 return {c}