Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/markdown/preprocessors.py: 97%

1# Python Markdown 

2 

3# A Python implementation of John Gruber's Markdown. 

4 

5# Documentation: https://python-markdown.github.io/ 

6# GitHub: https://github.com/Python-Markdown/markdown/ 

7# PyPI: https://pypi.org/project/Markdown/ 

8 

9# Started by Manfred Stienstra (http://www.dwerg.net/). 

10# Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org). 

11# Currently maintained by Waylan Limberg (https://github.com/waylan), 

12# Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser). 

13 

14# Copyright 2007-2023 The Python Markdown Project (v. 1.7 and later) 

15# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) 

16# Copyright 2004 Manfred Stienstra (the original version) 

17 

18# License: BSD (see LICENSE.md for details). 

19 

20""" 

21Preprocessors work on source text before it is broken down into its individual parts. 

22This is an excellent place to clean up bad characters or to extract portions for later 

23processing that the parser may otherwise choke on. 

24""" 

25 

26from __future__ import annotations 

27 

28from typing import TYPE_CHECKING, Any 

29from . import util 

30from .htmlparser import HTMLExtractor 

31import re 

32 

33if TYPE_CHECKING: # pragma: no cover 

34 from markdown import Markdown 

35 

36 

37def build_preprocessors(md: Markdown, **kwargs: Any) -> util.Registry[Preprocessor]: 

38 """ Build and return the default set of preprocessors used by Markdown. """ 

39 preprocessors = util.Registry() 

40 preprocessors.register(NormalizeWhitespace(md), 'normalize_whitespace', 30) 

41 preprocessors.register(HtmlBlockPreprocessor(md), 'html_block', 20) 

42 return preprocessors 

43 

44 

45class Preprocessor(util.Processor): 

46 """ 

47 Preprocessors are run after the text is broken into lines. 

48 

49 Each preprocessor implements a `run` method that takes a pointer to a 

50 list of lines of the document, modifies it as necessary and returns 

51 either the same pointer or a pointer to a new list. 

52 

53 Preprocessors must extend `Preprocessor`. 

54 

55 """ 

56 def run(self, lines: list[str]) -> list[str]: 

57 """ 

58 Each subclass of `Preprocessor` should override the `run` method, which 

59 takes the document as a list of strings split by newlines and returns 

60 the (possibly modified) list of lines. 

61 

62 """ 

63 pass # pragma: no cover 

64 

65 

66class NormalizeWhitespace(Preprocessor): 

67 """ Normalize whitespace for consistent parsing. """ 

68 

69 def run(self, lines: list[str]) -> list[str]: 

70 source = '\n'.join(lines) 

71 source = source.replace(util.STX, "").replace(util.ETX, "") 

72 source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n" 

73 source = source.expandtabs(self.md.tab_length) 

74 source = re.sub(r'(?<=\n) +\n', '\n', source) 

75 return source.split('\n') 

76 

77 

78class HtmlBlockPreprocessor(Preprocessor): 

79 """ 

80 Remove html blocks from the text and store them for later retrieval. 

81 

82 The raw HTML is stored in the [`htmlStash`][markdown.util.HtmlStash] of the 

83 [`Markdown`][markdown.Markdown] instance. 

84 """ 

85 

86 def run(self, lines: list[str]) -> list[str]: 

87 source = '\n'.join(lines) 

88 parser = HTMLExtractor(self.md) 

89 parser.feed(source) 

90 parser.close() 

91 return ''.join(parser.cleandoc).split('\n')