Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Python Markdown
3# A Python implementation of John Gruber's Markdown.
5# Documentation: https://python-markdown.github.io/
6# GitHub: https://github.com/Python-Markdown/markdown/
7# PyPI: https://pypi.org/project/Markdown/
9# Started by Manfred Stienstra (http://www.dwerg.net/).
10# Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org).
11# Currently maintained by Waylan Limberg (https://github.com/waylan),
12# Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser).
14# Copyright 2007-2023 The Python Markdown Project (v. 1.7 and later)
15# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
16# Copyright 2004 Manfred Stienstra (the original version)
18# License: BSD (see LICENSE.md for details).
20"""
21Tree processors manipulate the tree created by block processors. They can even create an entirely
22new `ElementTree` object. This is an excellent place for creating summaries, adding collected
23references, or last minute adjustments.
25"""
27from __future__ import annotations
29import re
30import xml.etree.ElementTree as etree
31from typing import TYPE_CHECKING, Any
32from . import util
33from . import inlinepatterns
35if TYPE_CHECKING: # pragma: no cover
36 from markdown import Markdown
39def build_treeprocessors(md: Markdown, **kwargs: Any) -> util.Registry[Treeprocessor]:
40 """ Build the default `treeprocessors` for Markdown. """
41 treeprocessors = util.Registry()
42 treeprocessors.register(InlineProcessor(md), 'inline', 20)
43 treeprocessors.register(PrettifyTreeprocessor(md), 'prettify', 10)
44 treeprocessors.register(UnescapeTreeprocessor(md), 'unescape', 0)
45 return treeprocessors
48def isString(s: object) -> bool:
49 """ Return `True` if object is a string but not an [`AtomicString`][markdown.util.AtomicString]. """
50 if not isinstance(s, util.AtomicString):
51 return isinstance(s, str)
52 return False
55class Treeprocessor(util.Processor):
56 """
57 `Treeprocessor`s are run on the `ElementTree` object before serialization.
59 Each `Treeprocessor` implements a `run` method that takes a pointer to an
60 `Element` and modifies it as necessary.
62 `Treeprocessors` must extend `markdown.Treeprocessor`.
64 """
65 def run(self, root: etree.Element) -> etree.Element | None:
66 """
67 Subclasses of `Treeprocessor` should implement a `run` method, which
68 takes a root `Element`. This method can return another `Element`
69 object, and the existing root `Element` will be replaced, or it can
70 modify the current tree and return `None`.
71 """
72 pass # pragma: no cover
75class InlineProcessor(Treeprocessor):
76 """
77 A `Treeprocessor` that traverses a tree, applying inline patterns.
78 """
80 def __init__(self, md: Markdown):
81 self.__placeholder_prefix = util.INLINE_PLACEHOLDER_PREFIX
82 self.__placeholder_suffix = util.ETX
83 self.__placeholder_length = 4 + len(self.__placeholder_prefix) \
84 + len(self.__placeholder_suffix)
85 self.__placeholder_re = util.INLINE_PLACEHOLDER_RE
86 self.md = md
87 self.inlinePatterns = md.inlinePatterns
88 self.ancestors: list[str] = []
90 def __makePlaceholder(self, type: str) -> tuple[str, str]:
91 """ Generate a placeholder """
92 id = "%04d" % len(self.stashed_nodes)
93 hash = util.INLINE_PLACEHOLDER % id
94 return hash, id
96 def __findPlaceholder(self, data: str, index: int) -> tuple[str | None, int]:
97 """
98 Extract id from data string, start from index.
100 Arguments:
101 data: String.
102 index: Index, from which we start search.
104 Returns:
105 Placeholder id and string index, after the found placeholder.
107 """
108 m = self.__placeholder_re.search(data, index)
109 if m:
110 return m.group(1), m.end()
111 else:
112 return None, index + 1
114 def __stashNode(self, node: etree.Element | str, type: str) -> str:
115 """ Add node to stash. """
116 placeholder, id = self.__makePlaceholder(type)
117 self.stashed_nodes[id] = node
118 return placeholder
120 def __handleInline(self, data: str, patternIndex: int = 0) -> str:
121 """
122 Process string with inline patterns and replace it with placeholders.
124 Arguments:
125 data: A line of Markdown text.
126 patternIndex: The index of the `inlinePattern` to start with.
128 Returns:
129 String with placeholders.
131 """
132 if not isinstance(data, util.AtomicString):
133 startIndex = 0
134 count = len(self.inlinePatterns)
135 while patternIndex < count:
136 data, matched, startIndex = self.__applyPattern(
137 self.inlinePatterns[patternIndex], data, patternIndex, startIndex
138 )
139 if not matched:
140 patternIndex += 1
141 return data
143 def __processElementText(self, node: etree.Element, subnode: etree.Element, isText: bool = True) -> None:
144 """
145 Process placeholders in `Element.text` or `Element.tail`
146 of Elements popped from `self.stashed_nodes`.
148 Arguments:
149 node: Parent node.
150 subnode: Processing node.
151 isText: Boolean variable, True - it's text, False - it's a tail.
153 """
154 if isText:
155 text = subnode.text
156 subnode.text = None
157 else:
158 text = subnode.tail
159 subnode.tail = None
161 childResult = self.__processPlaceholders(text, subnode, isText)
163 if not isText and node is not subnode:
164 pos = list(node).index(subnode) + 1
165 else:
166 pos = 0
168 childResult.reverse()
169 for newChild in childResult:
170 node.insert(pos, newChild[0])
172 def __processPlaceholders(
173 self,
174 data: str | None,
175 parent: etree.Element,
176 isText: bool = True
177 ) -> list[tuple[etree.Element, list[str]]]:
178 """
179 Process string with placeholders and generate `ElementTree` tree.
181 Arguments:
182 data: String with placeholders instead of `ElementTree` elements.
183 parent: Element, which contains processing inline data.
184 isText: Boolean variable, True - it's text, False - it's a tail.
186 Returns:
187 List with `ElementTree` elements with applied inline patterns.
189 """
190 def linkText(text: str | None) -> None:
191 if text:
192 if result:
193 if result[-1][0].tail:
194 result[-1][0].tail += text
195 else:
196 result[-1][0].tail = text
197 elif not isText:
198 if parent.tail:
199 parent.tail += text
200 else:
201 parent.tail = text
202 else:
203 if parent.text:
204 parent.text += text
205 else:
206 parent.text = text
207 result = []
208 strartIndex = 0
209 while data:
210 index = data.find(self.__placeholder_prefix, strartIndex)
211 if index != -1:
212 id, phEndIndex = self.__findPlaceholder(data, index)
214 if id in self.stashed_nodes:
215 node = self.stashed_nodes.get(id)
217 if index > 0:
218 text = data[strartIndex:index]
219 linkText(text)
221 if not isinstance(node, str): # it's Element
222 for child in [node] + list(node):
223 if child.tail:
224 if child.tail.strip():
225 self.__processElementText(
226 node, child, False
227 )
228 if child.text:
229 if child.text.strip():
230 self.__processElementText(child, child)
231 else: # it's just a string
232 linkText(node)
233 strartIndex = phEndIndex
234 continue
236 strartIndex = phEndIndex
237 result.append((node, self.ancestors[:]))
239 else: # wrong placeholder
240 end = index + len(self.__placeholder_prefix)
241 linkText(data[strartIndex:end])
242 strartIndex = end
243 else:
244 text = data[strartIndex:]
245 if isinstance(data, util.AtomicString):
246 # We don't want to loose the `AtomicString`
247 text = util.AtomicString(text)
248 linkText(text)
249 data = ""
251 return result
253 def __applyPattern(
254 self,
255 pattern: inlinepatterns.Pattern,
256 data: str,
257 patternIndex: int,
258 startIndex: int = 0
259 ) -> tuple[str, bool, int]:
260 """
261 Check if the line fits the pattern, create the necessary
262 elements, add it to `stashed_nodes`.
264 Arguments:
265 data: The text to be processed.
266 pattern: The pattern to be checked.
267 patternIndex: Index of current pattern.
268 startIndex: String index, from which we start searching.
270 Returns:
271 String with placeholders instead of `ElementTree` elements.
273 """
274 new_style = isinstance(pattern, inlinepatterns.InlineProcessor)
276 for exclude in pattern.ANCESTOR_EXCLUDES:
277 if exclude.lower() in self.ancestors:
278 return data, False, 0
280 if new_style:
281 match = None
282 # Since `handleMatch` may reject our first match,
283 # we iterate over the buffer looking for matches
284 # until we can't find any more.
285 for match in pattern.getCompiledRegExp().finditer(data, startIndex):
286 node, start, end = pattern.handleMatch(match, data)
287 if start is None or end is None:
288 startIndex += match.end(0)
289 match = None
290 continue
291 break
292 else: # pragma: no cover
293 match = pattern.getCompiledRegExp().match(data[startIndex:])
294 leftData = data[:startIndex]
296 if not match:
297 return data, False, 0
299 if not new_style: # pragma: no cover
300 node = pattern.handleMatch(match)
301 start = match.start(0)
302 end = match.end(0)
304 if node is None:
305 return data, True, end
307 if not isinstance(node, str):
308 if not isinstance(node.text, util.AtomicString):
309 # We need to process current node too
310 for child in [node] + list(node):
311 if not isString(node):
312 if child.text:
313 self.ancestors.append(child.tag.lower())
314 child.text = self.__handleInline(
315 child.text, patternIndex + 1
316 )
317 self.ancestors.pop()
318 if child.tail:
319 child.tail = self.__handleInline(
320 child.tail, patternIndex
321 )
323 placeholder = self.__stashNode(node, pattern.type())
325 if new_style:
326 # Return the index just past the inserted placeholder so the
327 # next call scans only the unprocessed tail. Scanning from 0
328 # after every match makes repeated inline patterns quadratic.
329 return "{}{}{}".format(data[:start],
330 placeholder, data[end:]), True, start + len(placeholder)
331 else: # pragma: no cover
332 return "{}{}{}{}".format(leftData,
333 match.group(1),
334 placeholder, match.groups()[-1]), True, (
335 len(leftData) + len(match.group(1)) + len(placeholder)
336 )
338 def __build_ancestors(self, parent: etree.Element | None, parents: list[str]) -> None:
339 """Build the ancestor list."""
340 ancestors = []
341 while parent is not None:
342 if parent is not None:
343 ancestors.append(parent.tag.lower())
344 parent = self.parent_map.get(parent)
345 ancestors.reverse()
346 parents.extend(ancestors)
348 def run(self, tree: etree.Element, ancestors: list[str] | None = None) -> etree.Element:
349 """Apply inline patterns to a parsed Markdown tree.
351 Iterate over `Element`, find elements with inline tag, apply inline
352 patterns and append newly created Elements to tree. To avoid further
353 processing of string with inline patterns, instead of normal string,
354 use subclass [`AtomicString`][markdown.util.AtomicString]:
356 node.text = markdown.util.AtomicString("This will not be processed.")
358 Arguments:
359 tree: `Element` object, representing Markdown tree.
360 ancestors: List of parent tag names that precede the tree node (if needed).
362 Returns:
363 An element tree object with applied inline patterns.
365 """
366 self.stashed_nodes: dict[str, etree.Element | str] = {}
368 # Ensure a valid parent list, but copy passed in lists
369 # to ensure we don't have the user accidentally change it on us.
370 tree_parents = [] if ancestors is None else ancestors[:]
372 self.parent_map = {c: p for p in tree.iter() for c in p}
373 stack = [(tree, tree_parents)]
375 while stack:
376 currElement, parents = stack.pop(0)
378 self.ancestors = parents
379 self.__build_ancestors(currElement, self.ancestors)
381 insertQueue = []
382 for child in currElement:
383 if child.text and not isinstance(
384 child.text, util.AtomicString
385 ):
386 self.ancestors.append(child.tag.lower())
387 text = child.text
388 child.text = None
389 lst = self.__processPlaceholders(
390 self.__handleInline(text), child
391 )
392 for item in lst:
393 self.parent_map[item[0]] = child
394 stack += lst
395 insertQueue.append((child, lst))
396 self.ancestors.pop()
397 if child.tail:
398 tail = self.__handleInline(child.tail)
399 dumby = etree.Element('d')
400 child.tail = None
401 tailResult = self.__processPlaceholders(tail, dumby, False)
402 if dumby.tail:
403 child.tail = dumby.tail
404 pos = list(currElement).index(child) + 1
405 tailResult.reverse()
406 for newChild in tailResult:
407 self.parent_map[newChild[0]] = currElement
408 currElement.insert(pos, newChild[0])
409 if len(child):
410 self.parent_map[child] = currElement
411 stack.append((child, self.ancestors[:]))
413 for element, lst in insertQueue:
414 for i, obj in enumerate(lst):
415 newChild = obj[0]
416 element.insert(i, newChild)
417 return tree
420class PrettifyTreeprocessor(Treeprocessor):
421 """ Add line breaks to the html document. """
423 def _prettifyETree(self, elem: etree.Element) -> None:
424 """ Recursively add line breaks to `ElementTree` children. """
426 i = "\n"
427 if self.md.is_block_level(elem.tag) and elem.tag not in ['code', 'pre']:
428 if (not elem.text or not elem.text.strip()) \
429 and len(elem) and self.md.is_block_level(elem[0].tag):
430 elem.text = i
431 for e in elem:
432 if self.md.is_block_level(e.tag):
433 self._prettifyETree(e)
434 if not elem.tail or not elem.tail.strip():
435 elem.tail = i
437 def run(self, root: etree.Element) -> None:
438 """ Add line breaks to `Element` object and its children. """
440 self._prettifyETree(root)
441 # Do `<br />`'s separately as they are often in the middle of
442 # inline content and missed by `_prettifyETree`.
443 brs = root.iter('br')
444 for br in brs:
445 if not br.tail or not br.tail.strip():
446 br.tail = '\n'
447 else:
448 br.tail = '\n%s' % br.tail
449 # Clean up extra empty lines at end of code blocks.
450 pres = root.iter('pre')
451 for pre in pres:
452 if len(pre) and pre[0].tag == 'code':
453 code = pre[0]
454 # Only prettify code containing text only
455 if not len(code) and code.text is not None:
456 code.text = util.AtomicString(code.text.rstrip() + '\n')
459class UnescapeTreeprocessor(Treeprocessor):
460 """ Restore escaped chars """
462 RE = re.compile(r'{}(\d+){}'.format(util.STX, util.ETX))
464 def _unescape(self, m: re.Match[str]) -> str:
465 return chr(int(m.group(1)))
467 def unescape(self, text: str) -> str:
468 return self.RE.sub(self._unescape, text)
470 def run(self, root: etree.Element) -> None:
471 """ Loop over all elements and unescape all text. """
472 for elem in root.iter():
473 # Unescape text content
474 if elem.text and not elem.tag == 'code':
475 elem.text = self.unescape(elem.text)
476 # Unescape tail content
477 if elem.tail:
478 elem.tail = self.unescape(elem.tail)
479 # Unescape attribute values
480 for key, value in elem.items():
481 elem.set(key, self.unescape(value))