Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Python Markdown
3# A Python implementation of John Gruber's Markdown.
5# Documentation: https://python-markdown.github.io/
6# GitHub: https://github.com/Python-Markdown/markdown/
7# PyPI: https://pypi.org/project/Markdown/
9# Started by Manfred Stienstra (http://www.dwerg.net/).
10# Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org).
11# Currently maintained by Waylan Limberg (https://github.com/waylan),
12# Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser).
14# Copyright 2007-2023 The Python Markdown Project (v. 1.7 and later)
15# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
16# Copyright 2004 Manfred Stienstra (the original version)
18# License: BSD (see LICENSE.md for details).
20"""
21In version 3.0, a new, more flexible inline processor was added, [`markdown.inlinepatterns.InlineProcessor`][]. The
22original inline patterns, which inherit from [`markdown.inlinepatterns.Pattern`][] or one of its children are still
23supported, though users are encouraged to migrate.
25The new `InlineProcessor` provides two major enhancements to `Patterns`:
271. Inline Processors no longer need to match the entire block, so regular expressions no longer need to start with
28 `r'^(.*?)'` and end with `r'(.*?)%'`. This runs faster. The returned [`Match`][re.Match] object will only contain
29 what is explicitly matched in the pattern, and extension pattern groups now start with `m.group(1)`.
312. The `handleMatch` method now takes an additional input called `data`, which is the entire block under analysis,
32 not just what is matched with the specified pattern. The method now returns the element *and* the indexes relative
33 to `data` that the return element is replacing (usually `m.start(0)` and `m.end(0)`). If the boundaries are
34 returned as `None`, it is assumed that the match did not take place, and nothing will be altered in `data`.
36 This allows handling of more complex constructs than regular expressions can handle, e.g., matching nested
37 brackets, and explicit control of the span "consumed" by the processor.
39"""
41from __future__ import annotations
43from . import util
44from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast
45from collections import deque
46import re
47import xml.etree.ElementTree as etree
48from html import entities
50if TYPE_CHECKING: # pragma: no cover
51 from markdown import Markdown
54def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlineProcessor]:
55 """
56 Build the default set of inline patterns for Markdown.
58 The order in which processors and/or patterns are applied is very important - e.g. if we first replace
59 `http://.../` links with `<a>` tags and _then_ try to replace inline HTML, we would end up with a mess. So, we
60 apply the expressions in the following order:
62 * backticks and escaped characters have to be handled before everything else so that we can preempt any markdown
63 patterns by escaping them;
65 * then we handle the various types of links (auto-links must be handled before inline HTML);
67 * then we handle inline HTML. At this point we will simply replace all inline HTML strings with a placeholder
68 and add the actual HTML to a stash;
70 * finally we apply strong, emphasis, etc.
72 """
73 inlinePatterns = util.Registry()
74 inlinePatterns.register(BacktickInlineProcessor(BACKTICK_RE), 'backtick', 190)
75 inlinePatterns.register(EscapeInlineProcessor(ESCAPE_RE, md), 'escape', 180)
76 inlinePatterns.register(ReferenceInlineProcessor(REFERENCE_RE, md), 'reference', 170)
77 inlinePatterns.register(LinkInlineProcessor(LINK_RE, md), 'link', 160)
78 inlinePatterns.register(ImageInlineProcessor(IMAGE_LINK_RE, md), 'image_link', 150)
79 inlinePatterns.register(
80 ImageReferenceInlineProcessor(IMAGE_REFERENCE_RE, md), 'image_reference', 140
81 )
82 inlinePatterns.register(
83 ShortReferenceInlineProcessor(REFERENCE_RE, md), 'short_reference', 130
84 )
85 inlinePatterns.register(
86 ShortImageReferenceInlineProcessor(IMAGE_REFERENCE_RE, md), 'short_image_ref', 125
87 )
88 inlinePatterns.register(AutolinkInlineProcessor(AUTOLINK_RE, md), 'autolink', 120)
89 inlinePatterns.register(AutomailInlineProcessor(AUTOMAIL_RE, md), 'automail', 110)
90 inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100)
91 inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90)
92 inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80)
93 inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60)
94 md.delimiters.add('_', 'strong,em', smart=True)
95 return inlinePatterns
98# The actual regular expressions for patterns
99# -----------------------------------------------------------------------------
101NOIMG = r'(?<!\!)'
102""" Match not an image. Partial regular expression which matches if not preceded by `!`. """
104BACKTICK_RE = r'(?:(?<!\\)((?:\\{2})+)(?=`+)|(?<!\\)`)'
105""" Match backtick quoted string (`` `e=f()` `` or ``` ``e=f("`")`` ```). """
107ESCAPE_RE = r'\\(.)'
108""" Match a backslash escaped character (`\\<` or `\\*`). """
110LINK_RE = NOIMG + r'\['
111""" Match start of in-line link (`[text](url)` or `[text](<url>)` or `[text](url "title")`). """
113IMAGE_LINK_RE = r'\!\['
114""" Match start of in-line image link (`` or ``). """
116REFERENCE_RE = LINK_RE
117""" Match start of reference link (`[Label][3]`). """
119IMAGE_REFERENCE_RE = IMAGE_LINK_RE
120""" Match start of image reference (`![alt text][2]`). """
122AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>'
123""" Match an automatic link (`<http://www.example.com>`). """
125AUTOMAIL_RE = r'<([^<> !]+@[^@<> ]+)>'
126""" Match an automatic email link (`<me@example.com>`). """
128HTML_RE = (
129 # Tag
130 r'''(<(\/?+[a-zA-Z][^\s"'<>@]*+(?:\s+[^\s"'=<>]++(?:\s*+=\s*+(?:"[^"]*+"|'[^']*+'|[^\s"'=<>]++))?+)*+\s*+/?|'''
131 # Comment
132 r'!--(?:(?!<!--|-->).)*--|'
133 # Processing instruction
134 r'[?](?:(?!<[?]|[?]>).)*[?]|'
135 # `CDATA`
136 r'!\[CDATA\[(?:(?!<!\[CDATA\[|\]\]>).)*\]\]'
137 ')>)'
138)
139""" Match an HTML tag (`<...>`). """
141ENTITY_RE = r'(&(?:\#[0-9]+|\#x[0-9a-fA-F]+|[a-zA-Z0-9]+);)'
142""" Match an HTML entity (`&` (decimal) or `&` (hex) or `&` (named)). """
144LINE_BREAK_RE = r' \n'
145""" Match two spaces at end of line. """
148def dequote(string: str) -> str:
149 """Remove quotes from around a string."""
150 if ((string.startswith('"') and string.endswith('"')) or
151 (string.startswith("'") and string.endswith("'"))):
152 return string[1:-1]
153 else:
154 return string
157class EmStrongItem(NamedTuple):
158 """Emphasis/strong pattern item."""
159 pattern: re.Pattern[str]
160 builder: str
161 tags: str
164# The pattern classes
165# -----------------------------------------------------------------------------
168class Pattern: # pragma: no cover
169 """
170 Base class that inline patterns subclass.
172 Inline patterns are handled by means of `Pattern` subclasses, one per regular expression.
173 Each pattern object uses a single regular expression and must support the following methods:
174 [`getCompiledRegExp`][markdown.inlinepatterns.Pattern.getCompiledRegExp] and
175 [`handleMatch`][markdown.inlinepatterns.Pattern.handleMatch].
177 All the regular expressions used by `Pattern` subclasses must capture the whole block. For this
178 reason, they all start with `^(.*)` and end with `(.*)!`. When passing a regular expression on
179 class initialization, the `^(.*)` and `(.*)!` are added automatically and the regular expression
180 is pre-compiled.
182 It is strongly suggested that the newer style [`markdown.inlinepatterns.InlineProcessor`][] that
183 use a more efficient and flexible search approach be used instead. However, the older style
184 `Pattern` remains for backward compatibility with many existing third-party extensions.
186 """
188 ANCESTOR_EXCLUDES: Collection[str] = tuple()
189 """
190 A collection of elements which are undesirable ancestors. The processor will be skipped if it
191 would cause the content to be a descendant of one of the listed tag names.
192 """
194 compiled_re: re.Pattern[str]
195 md: Markdown | None
197 def __init__(self, pattern: str, md: Markdown | None = None):
198 """
199 Create an instant of an inline pattern.
201 Arguments:
202 pattern: A regular expression that matches a pattern.
203 md: An optional pointer to the instance of `markdown.Markdown` and is available as
204 `self.md` on the class instance.
207 """
208 self.pattern = pattern
209 self.compiled_re = re.compile(r"^(.*?)%s(.*)$" % pattern,
210 re.DOTALL | re.UNICODE)
212 self.md = md
214 def getCompiledRegExp(self) -> re.Pattern:
215 """ Return a compiled regular expression. """
216 return self.compiled_re
218 def handleMatch(self, m: re.Match[str]) -> etree.Element | str:
219 """Return a ElementTree element from the given match.
221 Subclasses should override this method.
223 Arguments:
224 m: A match object containing a match of the pattern.
226 Returns: An ElementTree Element object.
228 """
229 pass # pragma: no cover
231 def type(self) -> str:
232 """ Return class name, to define pattern type """
233 return self.__class__.__name__
235 def unescape(self, text: str) -> str:
236 """ Return unescaped text given text with an inline placeholder. """
237 try:
238 stash = self.md.treeprocessors['inline'].stashed_nodes
239 except KeyError: # pragma: no cover
240 return text
242 def get_stash(m):
243 id = m.group(1)
244 if id in stash:
245 value = stash.get(id)
246 if isinstance(value, str):
247 return value
248 else:
249 # An `etree` Element - return text content only
250 return ''.join(value.itertext())
251 return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
254class InlineProcessor(Pattern):
255 """
256 Base class that inline processors subclass.
258 This is the newer style inline processor that uses a more
259 efficient and flexible search approach.
261 """
263 def __init__(self, pattern: str, md: Markdown | None = None):
264 """
265 Create an instant of an inline processor.
267 Arguments:
268 pattern: A regular expression that matches a pattern.
269 md: An optional pointer to the instance of `markdown.Markdown` and is available as
270 `self.md` on the class instance.
272 """
273 self.pattern = pattern
274 self.compiled_re = re.compile(pattern, re.DOTALL | re.UNICODE)
276 # API for Markdown to pass `safe_mode` into instance
277 self.safe_mode = False
278 self.md = md
280 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | str | None, int | None, int | None]:
281 """Return a ElementTree element from the given match and the
282 start and end index of the matched text.
284 If `start` and/or `end` are returned as `None`, it will be
285 assumed that the processor did not find a valid region of text.
287 Subclasses should override this method.
289 Arguments:
290 m: A re match object containing a match of the pattern.
291 data: The buffer currently under analysis.
293 Returns:
294 el: The ElementTree element, text or None.
295 start: The start of the region that has been matched or None.
296 end: The end of the region that has been matched or None.
298 """
299 pass # pragma: no cover
302class SimpleTextPattern(Pattern): # pragma: no cover
303 """ Return a simple text of `group(2)` of a Pattern. """
304 def handleMatch(self, m: re.Match[str]) -> str:
305 """ Return string content of `group(2)` of a matching pattern. """
306 return m.group(2)
309class SimpleTextInlineProcessor(InlineProcessor):
310 """ Return a simple text of `group(1)` of a Pattern. """
311 def handleMatch(self, m: re.Match[str], data: str) -> tuple[str, int, int]:
312 """ Return string content of `group(1)` of a matching pattern. """
313 return m.group(1), m.start(0), m.end(0)
316class EscapeInlineProcessor(InlineProcessor):
317 """ Return an escaped character. """
319 def handleMatch(self, m: re.Match[str], data: str) -> tuple[str | None, int, int]:
320 """
321 If the character matched by `group(1)` of a pattern is in [`ESCAPED_CHARS`][markdown.Markdown.ESCAPED_CHARS]
322 then return the integer representing the character's Unicode code point (as returned by [`ord`][]) wrapped
323 in [`util.STX`][markdown.util.STX] and [`util.ETX`][markdown.util.ETX].
325 If the matched character is not in [`ESCAPED_CHARS`][markdown.Markdown.ESCAPED_CHARS], then return `None`.
326 """
328 char = m.group(1)
329 if char in self.md.ESCAPED_CHARS:
330 return '{}{}{}'.format(util.STX, ord(char), util.ETX), m.start(0), m.end(0)
331 else:
332 return None, m.start(0), m.end(0)
335class SimpleTagPattern(Pattern): # pragma: no cover
336 """
337 Return element of type `tag` with a text attribute of `group(3)`
338 of a Pattern.
340 """
341 def __init__(self, pattern: str, tag: str):
342 """
343 Create an instant of an simple tag pattern.
345 Arguments:
346 pattern: A regular expression that matches a pattern.
347 tag: Tag of element.
349 """
350 Pattern.__init__(self, pattern)
351 self.tag = tag
352 """ The tag of the rendered element. """
354 def handleMatch(self, m: re.Match[str]) -> etree.Element:
355 """
356 Return [`Element`][xml.etree.ElementTree.Element] of type `tag` with the string in `group(3)` of a
357 matching pattern as the Element's text.
358 """
359 el = etree.Element(self.tag)
360 el.text = m.group(3)
361 return el
364class SimpleTagInlineProcessor(InlineProcessor):
365 """
366 Return element of type `tag` with a text attribute of `group(2)`
367 of a Pattern.
369 """
370 def __init__(self, pattern: str, tag: str):
371 """
372 Create an instant of an simple tag processor.
374 Arguments:
375 pattern: A regular expression that matches a pattern.
376 tag: Tag of element.
378 """
379 InlineProcessor.__init__(self, pattern)
380 self.tag = tag
381 """ The tag of the rendered element. """
383 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element, int, int]: # pragma: no cover
384 """
385 Return [`Element`][xml.etree.ElementTree.Element] of type `tag` with the string in `group(2)` of a
386 matching pattern as the Element's text.
387 """
388 el = etree.Element(self.tag)
389 el.text = m.group(2)
390 return el, m.start(0), m.end(0)
393class SubstituteTagPattern(SimpleTagPattern): # pragma: no cover
394 """ Return an element of type `tag` with no children. """
395 def handleMatch(self, m: re.Match[str]) -> etree.Element:
396 """ Return empty [`Element`][xml.etree.ElementTree.Element] of type `tag`. """
397 return etree.Element(self.tag)
400class SubstituteTagInlineProcessor(SimpleTagInlineProcessor):
401 """ Return an element of type `tag` with no children. """
402 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element, int, int]:
403 """ Return empty [`Element`][xml.etree.ElementTree.Element] of type `tag`. """
404 return etree.Element(self.tag), m.start(0), m.end(0)
407class BacktickInlineProcessor(InlineProcessor):
408 """ Return a `<code>` element containing the escaped matching text. """
410 def __init__(self, pattern: str):
411 InlineProcessor.__init__(self, pattern)
412 self.ESCAPED_BSLASH = '{}{}{}'.format(util.STX, ord('\\'), util.ETX)
413 self.tag = 'code'
414 """ The tag of the rendered element. """
416 def find_code_spans(self, start: int, text: str) -> tuple[int, int] | None:
417 """Find code spans."""
419 last = len(text)
421 # Get the maximum starting ticks
422 max_ticks = 0
423 while start < last and text[start] == '`':
424 max_ticks += 1
425 start += 1
427 if not max_ticks: # pragma: no cover
428 # This is not ever expected to happen.
429 return None
431 longest_span = 0
432 end = 0
434 # Find an ending span of backticks that matches our opening
435 i = start
436 while i < last:
437 span_length = 0
438 while i < last and text[i] == '`':
439 span_length += 1
440 i += 1
441 if not span_length:
442 i += 1
443 continue
445 # Did we find the end?
446 if max_ticks == span_length:
447 return start, i - span_length
449 # Track the longest span of backticks we find as a fallback.
450 if span_length > longest_span:
451 longest_span = span_length
452 end = i
454 # Since we didn't find an exact matching start and end,
455 # adjust start to match the largest end we could calculate.
456 if longest_span:
457 return start - (max_ticks - longest_span), end - longest_span
459 # We could not find a suitable pairing
460 return None
462 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | str | None, int | None, int | None]:
463 """
464 If the match contains `group(3)` of a pattern, then return a `code`
465 [`Element`][xml.etree.ElementTree.Element] which contains HTML escaped text (with
466 [`code_escape`][markdown.util.code_escape]) as an [`AtomicString`][markdown.util.AtomicString].
468 If the match contains `group(1)` then return the text of `group(1)` as backslash escaped.
470 """
471 if m.group(1):
472 return m.group(1).replace('\\\\', self.ESCAPED_BSLASH), m.start(0), m.end(0)
474 begin = m.start(0)
475 result = self.find_code_spans(begin, data)
476 if result is not None:
477 start, end = result
478 el = etree.Element(self.tag)
479 el.text = util.AtomicString(util.code_escape(data[start:end].strip()))
480 return el, begin, result[1] + (start - begin)
481 return None, None, None
484class DoubleTagPattern(SimpleTagPattern): # pragma: no cover
485 """Return a ElementTree element nested in tag2 nested in tag1.
487 Useful for strong emphasis etc.
489 """
490 def handleMatch(self, m: re.Match[str]) -> etree.Element:
491 """
492 Return [`Element`][xml.etree.ElementTree.Element] in following format:
493 `<tag1><tag2>group(3)</tag2>group(4)</tag2>` where `group(4)` is optional.
495 """
496 tag1, tag2 = self.tag.split(",")
497 el1 = etree.Element(tag1)
498 el2 = etree.SubElement(el1, tag2)
499 el2.text = m.group(3)
500 if len(m.groups()) == 5:
501 el2.tail = m.group(4)
502 return el1
505class DoubleTagInlineProcessor(SimpleTagInlineProcessor):
506 """Return a ElementTree element nested in tag2 nested in tag1.
508 Useful for strong emphasis etc.
510 """
511 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element, int, int]: # pragma: no cover
512 """
513 Return [`Element`][xml.etree.ElementTree.Element] in following format:
514 `<tag1><tag2>group(2)</tag2>group(3)</tag2>` where `group(3)` is optional.
516 """
517 tag1, tag2 = self.tag.split(",")
518 el1 = etree.Element(tag1)
519 el2 = etree.SubElement(el1, tag2)
520 el2.text = m.group(2)
521 if len(m.groups()) == 3:
522 el2.tail = m.group(3)
523 return el1, m.start(0), m.end(0)
526class HtmlInlineProcessor(InlineProcessor):
527 """ Store raw inline html and return a placeholder. """
528 def handleMatch(self, m: re.Match[str], data: str) -> tuple[str, int, int]:
529 """ Store the text of `group(1)` of a pattern and return a placeholder string. """
530 rawhtml = self.backslash_unescape(self.unescape(m.group(1)))
531 place_holder = self.md.htmlStash.store(rawhtml)
532 return place_holder, m.start(0), m.end(0)
534 def unescape(self, text: str) -> str:
535 """ Return unescaped text given text with an inline placeholder. """
536 try:
537 stash = self.md.treeprocessors['inline'].stashed_nodes
538 except KeyError: # pragma: no cover
539 return text
541 def get_stash(m: re.Match[str]) -> str:
542 id = m.group(1)
543 value = stash.get(id)
544 if value is not None:
545 if isinstance(value, etree.Element):
546 # Ensure we don't have a placeholder inside a placeholder
547 return self.unescape(self.md.serializer(value))
548 else:
549 return r'\%s' % value
551 return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
553 def backslash_unescape(self, text: str) -> str:
554 """ Return text with backslash escapes undone (backslashes are restored). """
555 try:
556 RE = self.md.treeprocessors['unescape'].RE
557 except KeyError: # pragma: no cover
558 return text
560 def _unescape(m: re.Match[str]) -> str:
561 return chr(int(m.group(1)))
563 return RE.sub(_unescape, text)
566class Delimiter:
567 """Delimiter."""
569 def __init__(self, token: str, tags: str, smart: bool, double: bool):
570 """Initialize."""
572 self.stack: deque[tuple[int, int, bool, int]] = deque()
573 temp = tags.split(',')
574 self.tag_count = len(temp)
575 self.tags: tuple[str, str] = (temp[0], temp[1]) if self.tag_count == 2 else (temp[0], temp[0])
576 self.double = len(temp) != 2 and double
577 self.single = len(temp) != 2 and not double
578 self.smart = smart
579 self._build_patterns(token)
581 def _build_patterns(self, token: str) -> str:
582 """Build regular expression patterns."""
584 # Build up patterns
585 self.token = token
586 etoken = re.escape(token)
587 # Avoid at start and end
588 xstart = fr'(?:(?<=_)|(?<![\w{etoken}]))' if token != '_' else fr'(?<![\w{etoken}])'
589 xend = fr'(?:(?=_)|(?![\w{etoken}]))' if token != '_' else fr'(?![\w{etoken}])'
590 self.max_size = 2
591 if self.tag_count != 2 and not self.double:
592 self.max_size = 1
594 # Patterns for "smart" cases.
595 if self.smart:
596 self.boundary = re.compile(
597 fr'''(?x)
598 (?P<ambiguous>(?<!^)(?<![\s{etoken}]){xstart}{etoken}{{1,}}{xend}(?![\s{etoken}])(?!$))|
599 (?P<end>(?<!^)(?<![\s{etoken}]){etoken}{{1,}}{xend})|
600 (?P<start>{xstart}{etoken}{{1,}}(?![\s{etoken}])(?!$))
601 ''',
602 flags=re.UNICODE
603 )
604 # Patterns for "dumb" cases.
605 else:
606 self.boundary = re.compile(
607 fr'''(?x)
608 (?P<ambiguous>(?<!^)(?<![\s{etoken}]){etoken}{{1,}}(?![\s{etoken}])(?!$))|
609 (?P<end>(?<!^)(?<![\s{etoken}]){etoken}{{1,}})|
610 (?P<start>{etoken}{{1,}}(?![\s{etoken}])(?!$))
611 ''',
612 flags=re.UNICODE
613 )
615 self.bad = re.compile(fr'{etoken}+')
617 return fr'{etoken}'
619 def reset(self) -> None:
620 """Reset."""
622 # Cache info
623 self.stack.clear()
626class DelimiterProcessor(InlineProcessor):
627 """Processor for handling complex nested patterns such as strong and em matches."""
629 SPACE = re.compile(r'\s')
631 def __init__(
632 self,
633 token: str,
634 tags: str,
635 md: Markdown,
636 smart: bool = False,
637 double: bool = False
638 ) -> None:
639 """Initialize."""
641 md.delimiters = self
642 self.regions: list[tuple[int, int, int, int, tuple[str, str], int]] = []
643 self.stack: list[tuple[int, int, bool, int]] = []
644 self.tokens: list[str] = []
645 self.delimiters: dict[str, Delimiter] = {}
646 self.cache_index = 0
647 self.cache_pos = 0
648 self.md = md
649 # API for Markdown to pass `safe_mode` into instance
650 self.safe_mode = False
651 self.add(token, tags, smart, double)
653 def add(
654 self,
655 token: str,
656 tags: str,
657 smart: bool = False,
658 double: bool = False
659 ) -> None:
660 """Add a delimiter."""
662 if token not in self.tokens:
663 self.tokens.append(token)
664 self.delimiters[token] = Delimiter(token, tags, smart, double)
665 self.pattern = '|'.join([re.escape(t) for t in self.tokens])
666 self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE)
668 def remove(self, token: str) -> None:
669 """Remove a token."""
671 try:
672 i = self.tokens.index(token)
673 del self.tokens[i]
674 del self.delimiters[token]
675 except Exception:
676 pass
677 self.pattern = '|'.join([re.escape(t) for t in self.tokens])
678 self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE) if self.pattern else re.compile(r'(?!)')
680 def reset(self) -> None:
681 """Reset."""
683 # Cache info
684 for v in self.delimiters.values():
685 v.reset()
686 self.regions.clear()
687 self.stack.clear()
688 self.cache_index = 0
689 self.cache_pos = 0
691 def _build_element(
692 self,
693 data: str,
694 start: int = 0,
695 offset: int = 0
696 ) -> tuple[etree.Element, int]:
697 """Element builder."""
699 regions = self.regions
700 el: etree.Element | None = None
701 last: Any = None
702 previous: Any = None
704 outer: list[etree.Element] = []
705 outer_r: list[tuple[int, int, int, int, tuple[str, str], int]] = []
707 # Iterate regions creating the elements they represent
708 end = len(regions)
709 idx = 0
710 for idx, i in enumerate(range(start, end), 1):
711 r = regions[i]
712 # Not contained within region
713 if idx and r[0] >= regions[start][3]:
714 idx -= 1
715 break
717 # Get the appropriate element(s)
718 if r[-1] == 2:
719 el1 = etree.Element(r[4][0])
720 else:
721 el1 = etree.Element(r[4][1])
723 # Populate the elements with their text
724 if idx > 1:
725 if last.text is None:
726 if previous[2] < r[0]:
727 last.text = data[previous[1]+offset:previous[2]+offset]
728 else:
729 last.text = data[previous[1]+offset:r[0]+offset]
730 if last is not outer[-1] and last.tail is None:
731 if r[0] < outer_r[-1][3]:
732 last.tail = data[previous[3]+offset:r[0]+offset]
733 else:
734 last.tail = data[previous[3]+offset:outer_r[-1][2]+offset]
735 outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset]
737 # First element
738 if el is None:
739 el = el1
740 last = el
741 outer.append(el)
742 outer_r.append(r)
744 # Subsequent elements
745 else:
746 # Is the current outer element no longer wrapping this one?
747 while len(outer_r) > 1 and r[3] > outer_r[-1][3]:
748 outer.pop()
749 outer_r.pop()
751 # Non-nested
752 else:
753 outer[-1].append(el1)
755 # Is this element wrapping the next?
756 if i + 1 < end:
757 if r[3] > regions[i + 1][3]:
758 outer.append(el1)
759 outer_r.append(r)
761 # Track the last element we parsed.
762 last = el1
764 # Track the previous region.
765 previous = r
767 # Populate remaining elements with their text
768 while outer:
769 if last.text is None:
770 last.text = data[previous[1]+offset:previous[2]+offset]
771 if last.tail is None and last is not outer[-1]:
772 last.tail = data[previous[3]+offset:outer_r[-1][2]+offset]
773 last = outer.pop()
774 previous = outer_r.pop()
776 return cast('etree.Element', el), idx
778 def increment_next_position(self, start: int, count: int) -> None:
779 """
780 Increment cache position to the next location that we can initiate an insertion.
782 Cache position should be the first match after our current replacement.
783 This gives us an anchor to calculate the new offset after insertion.
784 """
786 # Determine next offset
787 self.cache_index += count
788 if self.cache_index < len(self.regions):
789 self.cache_pos = self.regions[self.cache_index][0]
790 while self.stack:
791 entry = self.stack.pop(0)
792 if start < entry[0] <= self.cache_pos:
793 self.cache_pos = entry[0]
794 break
796 # Nothing left to process
797 else:
798 self.reset()
800 def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]:
801 """Get a cached result."""
803 # Process the next region(s) in the cache
804 regions = self.regions
805 offset = pos - self.cache_pos if pos != self.cache_pos else pos - regions[self.cache_index][0]
806 start, end = regions[self.cache_index][0], regions[self.cache_index][3]
807 el, count = self._build_element(data, self.cache_index, offset)
808 self.increment_next_position(start, count)
809 return el, start + offset, end + offset
811 def get_match(self, data: str, start: int) -> re.Match[str] | None:
812 """Get match."""
814 for d in self.delimiters.values():
815 m = d.boundary.match(data, start)
816 if m is not None:
817 return m
818 return None
820 def search(self, data: str, start: int) -> re.Match[str] | None:
821 """Search."""
823 for i in range(start, len(data)):
824 if data[i] in self.delimiters:
825 d = self.delimiters[data[i]]
826 m = d.boundary.match(data, i)
827 if m is not None:
828 if not d.stack and m.lastgroup[0] == 'e': # type: ignore[index]
829 continue
830 return m
831 return None
833 def add_region(self, delim: Delimiter, a: int, b: int, c: int, d: int, size: int) -> None:
834 """Add region."""
836 self.regions.append((a, b, c, d, delim.tags, size))
837 for delim in self.delimiters.values():
838 while delim.stack:
839 p = delim.stack[-1][0]
840 if max(p, a) <= min(p, d - 1):
841 delim.stack.pop()
842 continue
843 break
845 def handleMatch( # type: ignore[override]
846 self,
847 m: re.Match[str],
848 data: str
849 ) -> tuple[etree.Element | None, int | None, int | None]:
850 """Parse delimiter pattern."""
852 # Do we have entries we haven't returned yet?
853 if self.regions:
854 return self.get_cached_result(m.start(0), data)
856 # If token is not an opening, quit
857 m2 = self.get_match(data, m.start(0))
858 if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index]
859 if m2 is None:
860 delim = self.delimiters[m.group(0)]
861 m2 = delim.bad.match(data, m.start(0))
862 # Advance past the full length of the delimiter found
863 return None, m2.start(0), m2.end(0)
865 # Get the stack and regions
866 token = data[m.start(0)]
867 delim = self.delimiters[token]
868 stack = delim.stack
870 start = m2.start(0)
871 end = m2.end(0)
872 length = end - start
874 # Double needs at least a size of 2
875 if delim.double and length < 2:
876 return None, m.start(0), m.end(0)
878 is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index]
879 stack.append((start, start + length, is_ambiguous, length))
881 # Pair tokens until the stack is empty or we can no longer find tokens.
882 while any(d.stack for d in self.delimiters.values()):
883 m2 = self.search(data, end)
884 if m2 is None:
885 break
887 token = data[m2.start(0)]
888 delim = self.delimiters[token]
889 stack = delim.stack
891 start = m2.start(0)
892 end = m2.end(0)
894 # Get current and last delimiter size
895 current = len(m2.group(0))
897 # Some delimiters may be ambiguous and look like both a start or an end
898 is_start = m2.lastgroup[0] != 'e' # type: ignore[index]
899 is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index]
900 is_ambiguous = is_start and is_end
902 last = stack[-1][-1] if stack else 0
904 # Find closing tokens
905 # Looking for:
906 # - `*em*`
907 # - `**strong**`
908 # - `***strong,em***`
909 # - `*em**`
910 # - `*em***`
911 # - `**strong***`
912 #
913 # Avoid ambiguous tokens that could be a start or an end.
914 # Consume starts until the end token is fully consumed.
915 # If we don't consume the entire end, see if next rule consumes it.
916 if stack and is_end and ((not is_ambiguous and current > last) or current == last or current >= 3):
917 is_start = False
919 # Consume previous points until the delimiter is consumed
920 original = current
921 while stack and current and last <= current:
922 delimiter = stack.pop()
924 # Build up region for pair and adjust accounting.
925 size = min(delimiter[-1], 1 if delim.tag_count == 2 and delimiter[-1] == 3 else delim.max_size)
926 self.add_region(delim, delimiter[1] - size, delimiter[1], start, start + size, size)
927 start += size
928 current -= size
929 new = 0
930 if size < delimiter[-1] and (not delim.double or (delimiter[-1] - size) != 1):
931 new = delimiter[-1] - size
932 stack.append((delimiter[0], delimiter[1] - size, delimiter[2], new))
934 if not stack:
935 if any(d.stack for d in self.delimiters.values() if d is not delim):
936 delim.reset()
937 continue
938 is_end = False
939 break
941 last = stack[-1][-1]
943 # Should remainder be treated as a new start?
944 if original >= 3 and current and is_ambiguous:
945 delim.stack.append((m2.start(0) + (original - current), end, False, current))
946 is_end = False
948 # Do we still have more to consume?
949 else:
950 is_end = current and stack and last > current
952 # Looking for:
953 # - `***em*`
954 # - `***strong**`
955 # - `**em*`
956 if stack and is_end and (last >= 3 or not is_ambiguous) and last > current:
957 delimiter = stack.pop()
959 # Don't pair with an ambiguous opening
960 while stack and delimiter[-1] != 3 and delimiter[2]:
961 delimiter = stack.pop()
962 last = delimiter[-1]
964 if delimiter[2] and delimiter[-1] != 3:
965 if any(d.stack for d in self.delimiters.values() if d is not delim):
966 delim.reset()
967 continue
968 break
970 # Create new regions
971 is_start = False
972 ds, de = delimiter[:2]
973 while current and (not delim.double or current != 1):
974 size = min(current, 1 if delim.tag_count == 2 and current == 3 else delim.max_size)
975 new = last - size
976 self.add_region(delim, ds + new, de, start, start + size, size)
977 start += size
978 current -= size
979 last -= size
980 de -= size
981 if last and (not delim.double or last > 1):
982 stack.append((ds, de, False, last))
984 # Find opening tokens
985 # Looking for:
986 # - `*em ...*`
987 # - `**strong ...*`
988 # - `***em ...*`
989 if is_start and (not delim.double or current != 1):
990 stack.append((start, end, is_ambiguous, current))
992 # Combine the stacks and order them
993 for delim in self.delimiters.values():
994 self.stack.extend(delim.stack)
995 self.stack.sort(key=lambda x: x[0])
997 # Build the HTML elements
998 if self.regions:
999 # Regions may be out of order.
1000 self.regions.sort(key=lambda x: x[0])
1001 start, end = self.regions[0][0], self.regions[0][3]
1002 el, count = self._build_element(data)
1003 self.increment_next_position(start, count)
1004 return el, start, end
1006 # We failed to pair any valid start/end delimiters, avoid the parsed range next pass.
1007 start = m.start(0)
1008 end = self.stack[-1][1] if self.stack else m.end(0)
1009 self.reset()
1010 return None, start, end
1013class LinkInlineProcessor(InlineProcessor):
1014 """ Return a link element from the given match. """
1015 RE_LINK = re.compile(r'''\(\s*(?:(<[^<>]*>)\s*(?:('[^']*'|"[^"]*")\s*)?\))?''', re.DOTALL | re.UNICODE)
1016 RE_TITLE_CLEAN = re.compile(r'\s')
1018 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]:
1019 """ Return an `a` [`Element`][xml.etree.ElementTree.Element] or `(None, None, None)`. """
1020 text, index, handled = self.getText(data, m.end(0))
1022 if not handled:
1023 return None, None, None
1025 href, title, index, handled = self.getLink(data, index)
1026 if not handled:
1027 return None, None, None
1029 el = etree.Element("a")
1030 el.text = text
1032 el.set("href", href)
1034 if title is not None:
1035 el.set("title", title)
1037 return el, m.start(0), index
1039 def getLink(self, data: str, index: int) -> tuple[str, str | None, int, bool]:
1040 """Parse data between `()` of `[Text]()` allowing recursive `()`. """
1042 href = ''
1043 title: str | None = None
1044 handled = False
1046 m = self.RE_LINK.match(data, pos=index)
1047 if m and m.group(1):
1048 # Matches [Text](<link> "title")
1049 href = m.group(1)[1:-1].strip()
1050 if m.group(2):
1051 title = m.group(2)[1:-1]
1052 index = m.end(0)
1053 handled = True
1054 elif m:
1055 # Track bracket nesting and index in string
1056 bracket_count = 1
1057 backtrack_count = 1
1058 start_index = m.end()
1059 index = start_index
1060 last_bracket = -1
1062 # Primary (first found) quote tracking.
1063 quote: str | None = None
1064 start_quote = -1
1065 exit_quote = -1
1066 ignore_matches = False
1068 # Secondary (second found) quote tracking.
1069 alt_quote = None
1070 start_alt_quote = -1
1071 exit_alt_quote = -1
1073 # Track last character
1074 last = ''
1076 for pos in range(index, len(data)):
1077 c = data[pos]
1078 if c == '(':
1079 # Count nested (
1080 # Don't increment the bracket count if we are sure we're in a title.
1081 if not ignore_matches:
1082 bracket_count += 1
1083 elif backtrack_count > 0:
1084 backtrack_count -= 1
1085 elif c == ')':
1086 # Match nested ) to (
1087 # Don't decrement if we are sure we are in a title that is unclosed.
1088 if ((exit_quote != -1 and quote == last) or (exit_alt_quote != -1 and alt_quote == last)):
1089 bracket_count = 0
1090 elif not ignore_matches:
1091 bracket_count -= 1
1092 elif backtrack_count > 0:
1093 backtrack_count -= 1
1094 # We've found our backup end location if the title doesn't resolve.
1095 if backtrack_count == 0:
1096 last_bracket = index + 1
1098 elif c in ("'", '"'):
1099 # Quote has started
1100 if not quote:
1101 # We'll assume we are now in a title.
1102 # Brackets are quoted, so no need to match them (except for the final one).
1103 ignore_matches = True
1104 backtrack_count = bracket_count
1105 bracket_count = 1
1106 start_quote = index + 1
1107 quote = c
1108 # Secondary quote (in case the first doesn't resolve): [text](link'"title")
1109 elif c != quote and not alt_quote:
1110 start_alt_quote = index + 1
1111 alt_quote = c
1112 # Update primary quote match
1113 elif c == quote:
1114 exit_quote = index + 1
1115 # Update secondary quote match
1116 elif alt_quote and c == alt_quote:
1117 exit_alt_quote = index + 1
1119 index += 1
1121 # Link is closed, so let's break out of the loop
1122 if bracket_count == 0:
1123 # Get the title if we closed a title string right before link closed
1124 if exit_quote >= 0 and quote == last:
1125 href = data[start_index:start_quote - 1]
1126 title = ''.join(data[start_quote:exit_quote - 1])
1127 elif exit_alt_quote >= 0 and alt_quote == last:
1128 href = data[start_index:start_alt_quote - 1]
1129 title = ''.join(data[start_alt_quote:exit_alt_quote - 1])
1130 else:
1131 href = data[start_index:index - 1]
1132 break
1134 if c != ' ':
1135 last = c
1137 # We have a scenario: `[test](link"notitle)`
1138 # When we enter a string, we stop tracking bracket resolution in the main counter,
1139 # but we do keep a backup counter up until we discover where we might resolve all brackets
1140 # if the title string fails to resolve.
1141 if bracket_count != 0 and backtrack_count == 0:
1142 href = data[start_index:last_bracket - 1]
1143 index = last_bracket
1144 bracket_count = 0
1146 handled = bracket_count == 0
1148 if title is not None:
1149 title = self.RE_TITLE_CLEAN.sub(' ', dequote(self.unescape(title.strip())))
1151 href = self.unescape(href).strip()
1153 return href, title, index, handled
1155 def getText(self, data: str, index: int) -> tuple[str, int, bool]:
1156 """Parse the content between `[]` of the start of an image or link
1157 resolving nested square brackets.
1159 """
1160 bracket_count = 1
1161 text = []
1162 for pos in range(index, len(data)):
1163 c = data[pos]
1164 if c == ']':
1165 bracket_count -= 1
1166 elif c == '[':
1167 bracket_count += 1
1168 index += 1
1169 if bracket_count == 0:
1170 break
1171 text.append(c)
1172 return ''.join(text), index, bracket_count == 0
1175class ImageInlineProcessor(LinkInlineProcessor):
1176 """ Return a `img` element from the given match. """
1178 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]:
1179 """ Return an `img` [`Element`][xml.etree.ElementTree.Element] or `(None, None, None)`. """
1180 text, index, handled = self.getText(data, m.end(0))
1181 if not handled:
1182 return None, None, None
1184 src, title, index, handled = self.getLink(data, index)
1185 if not handled:
1186 return None, None, None
1188 el = etree.Element("img")
1190 el.set("src", src)
1192 if title is not None:
1193 el.set("title", title)
1195 el.set('alt', self.unescape(text))
1196 return el, m.start(0), index
1199class ReferenceInlineProcessor(LinkInlineProcessor):
1200 """ Match to a stored reference and return link element. """
1201 NEWLINE_CLEANUP_RE = re.compile(r'\s+', re.MULTILINE)
1203 RE_LINK = re.compile(r'\s?\[([^\]]*)\]', re.DOTALL | re.UNICODE)
1205 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]:
1206 """
1207 Return [`Element`][xml.etree.ElementTree.Element] returned by `makeTag` method or `(None, None, None)`.
1209 """
1210 text, index, handled = self.getText(data, m.end(0))
1211 if not handled:
1212 return None, None, None
1214 id, end, handled = self.evalId(data, index, text)
1215 if not handled:
1216 return None, None, None
1218 # Clean up line breaks in id
1219 id = self.NEWLINE_CLEANUP_RE.sub(' ', id)
1220 if id not in self.md.references: # ignore undefined refs
1221 return None, m.start(0), end
1223 href, title = self.md.references[id]
1225 return self.makeTag(href, title, text), m.start(0), end
1227 def evalId(self, data: str, index: int, text: str) -> tuple[str | None, int, bool]:
1228 """
1229 Evaluate the id portion of `[ref][id]`.
1231 If `[ref][]` use `[ref]`.
1232 """
1233 m = self.RE_LINK.match(data, pos=index)
1234 if not m:
1235 return None, index, False
1236 else:
1237 id = m.group(1).lower()
1238 end = m.end(0)
1239 if not id:
1240 id = text.lower()
1241 return id, end, True
1243 def makeTag(self, href: str, title: str, text: str) -> etree.Element:
1244 """ Return an `a` [`Element`][xml.etree.ElementTree.Element]. """
1245 el = etree.Element('a')
1247 el.set('href', href)
1248 if title:
1249 el.set('title', title)
1251 el.text = text
1252 return el
1255class ShortReferenceInlineProcessor(ReferenceInlineProcessor):
1256 """Short form of reference: `[google]`. """
1257 def evalId(self, data: str, index: int, text: str) -> tuple[str, int, bool]:
1258 """Evaluate the id of `[ref]`. """
1260 return text.lower(), index, True
1263class ImageReferenceInlineProcessor(ReferenceInlineProcessor):
1264 """ Match to a stored reference and return `img` element. """
1265 def makeTag(self, href: str, title: str, text: str) -> etree.Element:
1266 """ Return an `img` [`Element`][xml.etree.ElementTree.Element]. """
1267 el = etree.Element("img")
1268 el.set("src", href)
1269 if title:
1270 el.set("title", title)
1271 el.set("alt", self.unescape(text))
1272 return el
1275class ShortImageReferenceInlineProcessor(ImageReferenceInlineProcessor):
1276 """ Short form of image reference: `![ref]`. """
1277 def evalId(self, data: str, index: int, text: str) -> tuple[str, int, bool]:
1278 """Evaluate the id of `[ref]`. """
1280 return text.lower(), index, True
1283class AutolinkInlineProcessor(InlineProcessor):
1284 """ Return a link Element given an auto-link (`<http://example/com>`). """
1285 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element, int, int]:
1286 """ Return an `a` [`Element`][xml.etree.ElementTree.Element] of `group(1)`. """
1287 el = etree.Element("a")
1288 el.set('href', self.unescape(m.group(1)))
1289 el.text = util.AtomicString(m.group(1))
1290 return el, m.start(0), m.end(0)
1293class AutomailInlineProcessor(InlineProcessor):
1294 """
1295 Return a `mailto` link Element given an auto-mail link (`<foo@example.com>`).
1296 """
1297 def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element, int, int]:
1298 """ Return an [`Element`][xml.etree.ElementTree.Element] containing a `mailto` link of `group(1)`. """
1299 el = etree.Element('a')
1300 email = self.unescape(m.group(1))
1301 if email.startswith("mailto:"):
1302 email = email[len("mailto:"):]
1304 def codepoint2name(code: int) -> str:
1305 """Return entity definition by code, or the code if not defined."""
1306 entity = entities.codepoint2name.get(code)
1307 if entity:
1308 return "{}{};".format(util.AMP_SUBSTITUTE, entity)
1309 else:
1310 return "%s#%d;" % (util.AMP_SUBSTITUTE, code)
1312 letters = [codepoint2name(ord(letter)) for letter in email]
1313 el.text = util.AtomicString(''.join(letters))
1315 mailto = "mailto:" + email
1316 mailto = "".join([util.AMP_SUBSTITUTE + '#%d;' %
1317 ord(letter) for letter in mailto])
1318 el.set('href', mailto)
1319 return el, m.start(0), m.end(0)