Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/markdown/htmlparser.py: 99%

1# Python Markdown 

2 

3# A Python implementation of John Gruber's Markdown. 

4 

5# Documentation: https://python-markdown.github.io/ 

6# GitHub: https://github.com/Python-Markdown/markdown/ 

7# PyPI: https://pypi.org/project/Markdown/ 

8 

9# Started by Manfred Stienstra (http://www.dwerg.net/). 

10# Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org). 

11# Currently maintained by Waylan Limberg (https://github.com/waylan), 

12# Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser). 

13 

14# Copyright 2007-2023 The Python Markdown Project (v. 1.7 and later) 

15# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) 

16# Copyright 2004 Manfred Stienstra (the original version) 

17 

18# License: BSD (see LICENSE.md for details). 

19 

20""" 

21This module imports a copy of [`html.parser.HTMLParser`][] and modifies it heavily through monkey-patches. 

22A copy is imported rather than the module being directly imported as this ensures that the user can import 

23and use the unmodified library for their own needs. 

24""" 

25 

26from __future__ import annotations 

27 

28import re 

29import importlib.util 

30import sys 

31from typing import TYPE_CHECKING, Sequence 

32 

33if TYPE_CHECKING: # pragma: no cover 

34 from markdown import Markdown 

35 

36# Included for versions which do not have current comment fix 

37commentclose = re.compile(r'--!?>') 

38 

39# Import a copy of the html.parser lib as `htmlparser` so we can monkeypatch it. 

40# Users can still do `from html import parser` and get the default behavior. 

41spec = importlib.util.find_spec('html.parser') 

42htmlparser = importlib.util.module_from_spec(spec) 

43spec.loader.exec_module(htmlparser) 

44sys.modules['htmlparser'] = htmlparser 

45 

46# This is a hack. We are sneaking in `</>` so we can capture it without the HTML parser 

47# throwing it away. When we see it, we will process it as data. 

48htmlparser.starttagopen = re.compile('<[a-zA-Z]|</>') 

49 

50htmlparser.endtagopen = re.compile('</[a-zA-Z]?') 

51 

52# Monkeypatch `HTMLParser` to only accept `?>` to close Processing Instructions. 

53htmlparser.piclose = re.compile(r'\?>') 

54# Monkeypatch `HTMLParser` to only recognize entity references with a closing semicolon. 

55htmlparser.entityref = re.compile(r'&([a-zA-Z][-.a-zA-Z0-9]*);') 

56# Monkeypatch `HTMLParser` to no longer support partial entities. We are always feeding a complete block, 

57# so the 'incomplete' functionality is unnecessary. As the `entityref` regex is run right before incomplete, 

58# and the two regex are the same, then incomplete will simply never match and we avoid the logic within. 

59htmlparser.incomplete = htmlparser.entityref 

60# Monkeypatch `HTMLParser` to not accept a backtick in a tag name, attribute name, or bare value. 

61htmlparser.locatestarttagend_tolerant = re.compile(r""" 

62 <[a-zA-Z][^`\t\n\r\f />\x00]* # tag name <= added backtick here 

63 (?:[\s/]* # optional whitespace before attribute name 

64 (?:(?<=['"\s/])[^`\s/>][^\s/=>]* # attribute name <= added backtick here 

65 (?:\s*=+\s* # value indicator 

66 (?:'[^']*' # LITA-enclosed value 

67 |"[^"]*" # LIT-enclosed value 

68 |(?!['"])[^`>\s]* # bare value <= added backtick here 

69 ) 

70 (?:\s*,)* # possibly followed by a comma 

71 )?(?:\s|/(?!>))* 

72 )* 

73 )? 

74 \s* # trailing whitespace 

75""", re.VERBOSE) 

76htmlparser.locatetagend = re.compile(r""" 

77 [a-zA-Z][^`\t\n\r\f />]* # tag name 

78 [\t\n\r\f /]* # optional whitespace before attribute name 

79 (?:(?<=['"\t\n\r\f /])[^`\t\n\r\f />][^\t\n\r\f /=>]* # attribute name 

80 (?:= # value indicator 

81 (?:'[^']*' # LITA-enclosed value 

82 |"[^"]*" # LIT-enclosed value 

83 |(?!['"])[^>\t\n\r\f ]* # bare value 

84 ) 

85 )? 

86 [\t\n\r\f /]* # possibly followed by a space 

87 )* 

88 >? 

89""", re.VERBOSE) 

90 

91# Match a blank line at the start of a block of text (two newlines). 

92# The newlines may be preceded by additional whitespace. 

93blank_line_re = re.compile(r'^([ ]*\n){2}') 

94 

95# Match a blank line anywhere in a run of text. 

96inner_blank_line_re = re.compile(r'\n[ \t]*\n') 

97 

98# Elements which never have an end tag and so cannot contain a comment. 

99void_tags = frozenset([ 

100 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr' 

101]) 

102 

103 

104class _HTMLParser(htmlparser.HTMLParser): 

105 """Handle special start and end tags.""" 

106 

107 def parse_endtag(self, i): 

108 start = self.rawdata[i:i+3] 

109 c = ord(start[-1]) 

110 is_data = len(start) < 3 or not (65 <= c <= 90 or 97 <= c <= 122) 

111 pos = super().parse_endtag(i) if not is_data else -1 

112 if pos == -1: 

113 self.handle_data(self.rawdata[i:i + 2]) 

114 return i + 2 

115 return pos 

116 

117 def parse_starttag(self, i: int) -> int: # pragma: no cover 

118 # Treat `</>` as normal data as it is not a real tag. 

119 if self.rawdata[i:i + 3] == '</>': 

120 self.handle_data(self.rawdata[i:i + 3]) 

121 return i + 3 

122 

123 return super().parse_starttag(i) 

124 

125 

126# Overwrite our custom one for people like MkDocs that pull it in 

127htmlparser.HTMLParser = _HTMLParser 

128 

129 

130class HTMLExtractor(htmlparser.HTMLParser): 

131 """ 

132 Extract raw HTML from text. 

133 

134 The raw HTML is stored in the [`htmlStash`][markdown.util.HtmlStash] of the 

135 [`Markdown`][markdown.Markdown] instance passed to `md` and the remaining text 

136 is stored in `cleandoc` as a list of strings. 

137 """ 

138 

139 def __init__(self, md: Markdown, *args, **kwargs): 

140 if 'convert_charrefs' not in kwargs: 

141 kwargs['convert_charrefs'] = False 

142 

143 # Block tags that should contain no content (self closing) 

144 self.empty_tags = set(['hr']) 

145 

146 self.lineno_start_cache = [0] 

147 

148 # This calls self.reset 

149 super().__init__(*args, **kwargs) 

150 self.md = md 

151 

152 def reset(self): 

153 """Reset this instance. Loses all unprocessed data.""" 

154 self.inraw = False 

155 self.intail = False 

156 self.stack: list[str] = [] # When `inraw==True`, stack contains a list of tags 

157 self.inline_stack: list[str] = [] # Unclosed raw inline tags in the current paragraph 

158 self._cache: list[str] = [] 

159 self.cleandoc: list[str] = [] 

160 self.lineno_start_cache = [0] 

161 

162 super().reset() 

163 

164 def close(self): 

165 """Handle any buffered data.""" 

166 super().close() 

167 if len(self.rawdata): 

168 # Temp fix for https://bugs.python.org/issue41989 

169 # TODO: remove this when the bug is fixed in all supported Python versions. 

170 if self.convert_charrefs and not self.cdata_elem: # pragma: no cover 

171 self.handle_data(htmlparser.unescape(self.rawdata)) 

172 else: 

173 self.handle_data(self.rawdata) 

174 # Handle any unclosed tags. 

175 if len(self._cache): 

176 self.cleandoc.append(self.md.htmlStash.store(''.join(self._cache))) 

177 self._cache = [] 

178 

179 @property 

180 def line_offset(self) -> int: 

181 """Returns char index in `self.rawdata` for the start of the current line. """ 

182 for ii in range(len(self.lineno_start_cache)-1, self.lineno-1): 

183 last_line_start_pos = self.lineno_start_cache[ii] 

184 lf_pos = self.rawdata.find('\n', last_line_start_pos) 

185 if lf_pos == -1: 

186 # No more newlines found. Use end of raw data as start of line beyond end. 

187 lf_pos = len(self.rawdata) 

188 self.lineno_start_cache.append(lf_pos+1) 

189 

190 return self.lineno_start_cache[self.lineno-1] 

191 

192 def at_line_start(self) -> bool: 

193 """ 

194 Returns True if current position is at start of line. 

195 

196 Allows for up to three blank spaces at start of line. 

197 """ 

198 if self.offset == 0: 

199 return True 

200 if self.offset > 3: 

201 return False 

202 # Confirm up to first 3 chars are whitespace 

203 return self.rawdata[self.line_offset:self.line_offset + self.offset].strip() == '' 

204 

205 def get_endtag_text(self, tag: str) -> str: 

206 """ 

207 Returns the text of the end tag. 

208 

209 If it fails to extract the actual text from the raw data, it builds a closing tag with `tag`. 

210 """ 

211 # Attempt to extract actual tag from raw source text 

212 start = self.line_offset + self.offset 

213 m = htmlparser.endendtag.search(self.rawdata, start) 

214 if m: 

215 return self.rawdata[start:m.end()] 

216 else: # pragma: no cover 

217 # Failed to extract from raw data. Assume well formed and lowercase. 

218 return '</{}>'.format(tag) 

219 

220 def handle_starttag(self, tag: str, attrs: Sequence[tuple[str, str]]): 

221 # Handle tags that should always be empty and do not specify a closing tag 

222 if tag in self.empty_tags: 

223 self.handle_startendtag(tag, attrs) 

224 return 

225 

226 if self.md.is_block_level(tag) and (self.intail or (self.at_line_start() and not self.inraw)): 

227 # Started a new raw block. Prepare stack. 

228 self.inraw = True 

229 self.cleandoc.append('\n') 

230 

231 text = self.get_starttag_text() 

232 if self.inraw: 

233 self.stack.append(tag) 

234 self._cache.append(text) 

235 else: 

236 self.cleandoc.append(text) 

237 self.open_inline_tag(tag) 

238 if tag in self.CDATA_CONTENT_ELEMENTS: 

239 # This is presumably a standalone tag in a code span (see #1036). 

240 self.clear_cdata_mode() 

241 

242 def handle_endtag(self, tag: str): 

243 text = self.get_endtag_text(tag) 

244 

245 if self.inraw: 

246 self._cache.append(text) 

247 if tag in self.stack: 

248 # Remove tag from stack 

249 while self.stack: 

250 if self.stack.pop() == tag: 

251 break 

252 if len(self.stack) == 0: 

253 # End of raw block. 

254 if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(text):]): 

255 # Preserve blank line and end of raw block. 

256 self._cache.append('\n') 

257 else: 

258 # More content exists after `endtag`. 

259 self.intail = True 

260 # Reset stack. 

261 self.inraw = False 

262 self.cleandoc.append(self.md.htmlStash.store(''.join(self._cache))) 

263 # Insert blank line between this and next line. 

264 self.cleandoc.append('\n\n') 

265 self._cache = [] 

266 else: 

267 self.cleandoc.append(text) 

268 self.close_inline_tag(tag) 

269 

270 def open_inline_tag(self, tag: str): 

271 """ Track an unclosed raw inline tag so a comment inside it is not treated as a block. """ 

272 if not self.md.is_block_level(tag) and tag not in void_tags: 

273 self.inline_stack.append(tag) 

274 

275 def close_inline_tag(self, tag: str): 

276 """ Stop tracking an inline tag and any unclosed inline tags opened after it. """ 

277 if tag in self.inline_stack: 

278 while self.inline_stack: 

279 if self.inline_stack.pop() == tag: 

280 break 

281 

282 def inline_close_follows(self, text: str) -> bool: 

283 """ Return `True` if an open inline tag is closed in the same paragraph after the comment `text`. """ 

284 if not self.inline_stack: 

285 return False 

286 rest = inner_blank_line_re.split(self.rawdata[self.line_offset + self.offset + len(text):], 1)[0] 

287 return any(re.search(r'</\s*{}\s*>'.format(re.escape(tag)), rest, re.I) for tag in self.inline_stack) 

288 

289 def handle_data(self, data: str): 

290 if self.intail and '\n' in data: 

291 self.intail = False 

292 if self.inraw: 

293 self._cache.append(data) 

294 else: 

295 if inner_blank_line_re.search(data): 

296 # A blank line ends the paragraph, so open inline tags no longer enclose what follows. 

297 self.inline_stack = [] 

298 self.cleandoc.append(data) 

299 

300 def handle_empty_tag(self, data: str, is_block: bool): 

301 """ Handle empty tags (`<data>`). """ 

302 if self.inraw or self.intail: 

303 # Append this to the existing raw block 

304 self._cache.append(data) 

305 elif self.at_line_start() and is_block: 

306 # Handle this as a standalone raw block 

307 if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(data):]): 

308 # Preserve blank line after tag in raw block. 

309 data += '\n' 

310 else: 

311 # More content exists after tag. 

312 self.intail = True 

313 item = self.cleandoc[-1] if self.cleandoc else '' 

314 # If we only have one newline before block element, add another 

315 if not item.endswith('\n\n') and item.endswith('\n'): 

316 self.cleandoc.append('\n') 

317 self.cleandoc.append(self.md.htmlStash.store(data)) 

318 # Insert blank line between this and next line. 

319 self.cleandoc.append('\n\n') 

320 else: 

321 self.cleandoc.append(data) 

322 

323 def handle_startendtag(self, tag: str, attrs): 

324 self.handle_empty_tag(self.get_starttag_text(), is_block=self.md.is_block_level(tag)) 

325 

326 def handle_charref(self, name: str): 

327 self.handle_empty_tag('&#{};'.format(name), is_block=False) 

328 

329 def handle_entityref(self, name: str): 

330 self.handle_empty_tag('&{};'.format(name), is_block=False) 

331 

332 def handle_comment(self, data: str): 

333 # Check if the comment is unclosed, if so, we need to override position 

334 text = '<!--{}-->'.format(data) 

335 # A comment inside a raw inline element is part of that inline content, unless it contains a blank line. 

336 is_block = bool(inner_blank_line_re.search(data)) or not self.inline_close_follows(text) 

337 self.handle_empty_tag(text, is_block=is_block) 

338 

339 def handle_decl(self, data: str): 

340 self.handle_empty_tag('<!{}>'.format(data), is_block=True) 

341 

342 def handle_pi(self, data: str): 

343 self.handle_empty_tag('<?{}?>'.format(data), is_block=True) 

344 

345 def unknown_decl(self, data: str): 

346 end = ']]>' if data.startswith('CDATA[') else ']>' 

347 self.handle_empty_tag('<![{}{}'.format(data, end), is_block=True) 

348 

349 def parse_pi(self, i: int) -> int: 

350 if self.at_line_start() or self.intail: 

351 return super().parse_pi(i) 

352 # This is not the beginning of a raw block so treat as plain data 

353 # and avoid consuming any tags which may follow (see #1066). 

354 self.handle_data('<?') 

355 return i + 2 

356 

357 # Internal -- parse comment, return length or -1 if not terminated 

358 # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state 

359 def parse_comment(self, i, report=True): 

360 rawdata = self.rawdata 

361 assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()' 

362 match = commentclose.search(rawdata, i+4) 

363 if not match: 

364 self.handle_data('<') 

365 return i + 1 

366 if report: 

367 j = match.start() 

368 self.handle_comment(rawdata[i+4: j]) 

369 return match.end() 

370 

371 def parse_html_declaration(self, i: int) -> int: 

372 if self.at_line_start() or self.intail: 

373 if self.rawdata[i:i+3] == '<![' and not self.rawdata[i:i+9] == '<![CDATA[': 

374 # We have encountered the bug in #1534 (Python bug `gh-77057`). 

375 # Provide an override until we drop support for Python < 3.13. 

376 result = self.parse_bogus_comment(i) 

377 if result == -1: 

378 self.handle_data(self.rawdata[i:i + 1]) 

379 return i + 1 

380 return result 

381 return super().parse_html_declaration(i) 

382 # This is not the beginning of a raw block so treat as plain data 

383 # and avoid consuming any tags which may follow (see #1066). 

384 self.handle_data('<!') 

385 return i + 2 

386 

387 def parse_bogus_comment(self, i: int, report: int = 0) -> int: 

388 # Override the default behavior so that bogus comments get passed 

389 # through unaltered by setting `report` to `0` (see #1425). 

390 pos = super().parse_bogus_comment(i, report) 

391 if pos == -1: # pragma: no cover 

392 return -1 

393 self.handle_empty_tag(self.rawdata[i:pos], is_block=False) 

394 return pos 

395 

396 # The rest has been copied from base class in standard lib to address #1036. 

397 # As `__startag_text` is private, all references to it must be in this subclass. 

398 # The last few lines of `parse_starttag` are reversed so that `handle_starttag` 

399 # can override `cdata_mode` in certain situations (in a code span). 

400 __starttag_text: str | None = None 

401 

402 def get_starttag_text(self) -> str: 

403 """Return full source of start tag: `<...>`.""" 

404 return self.__starttag_text 

405 

406 def parse_starttag(self, i: int) -> int: # pragma: no cover 

407 # Treat `</>` as normal data as it is not a real tag. 

408 if self.rawdata[i:i + 3] == '</>': 

409 self.handle_data(self.rawdata[i:i + 3]) 

410 return i + 3 

411 

412 self.__starttag_text = None 

413 endpos = self.check_for_whole_start_tag(i) 

414 if endpos < 0: 

415 self.handle_data(self.rawdata[i:i + 1]) 

416 return i + 1 

417 rawdata = self.rawdata 

418 self.__starttag_text = rawdata[i:endpos] 

419 

420 # Now parse the data between `i+1` and `j` into a tag and `attrs` 

421 attrs = [] 

422 match = htmlparser.tagfind_tolerant.match(rawdata, i+1) 

423 assert match, 'unexpected call to parse_starttag()' 

424 k = match.end() 

425 self.lasttag = tag = match.group(1).lower() 

426 while k < endpos: 

427 m = htmlparser.attrfind_tolerant.match(rawdata, k) 

428 if not m: 

429 break 

430 attrname, rest, attrvalue = m.group(1, 2, 3) 

431 if not rest: 

432 attrvalue = None 

433 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ 

434 attrvalue[:1] == '"' == attrvalue[-1:]: # noqa: E127 

435 attrvalue = attrvalue[1:-1] 

436 if attrvalue: 

437 attrvalue = htmlparser.unescape(attrvalue) 

438 attrs.append((attrname.lower(), attrvalue)) 

439 k = m.end() 

440 

441 end = rawdata[k:endpos].strip() 

442 if end not in (">", "/>"): 

443 lineno, offset = self.getpos() 

444 if "\n" in self.__starttag_text: 

445 lineno = lineno + self.__starttag_text.count("\n") 

446 offset = len(self.__starttag_text) \ 

447 - self.__starttag_text.rfind("\n") # noqa: E127 

448 else: 

449 offset = offset + len(self.__starttag_text) 

450 self.handle_data(rawdata[i:endpos]) 

451 return endpos 

452 if end.endswith('/>'): 

453 # XHTML-style empty tag: `<span attr="value" />` 

454 self.handle_startendtag(tag, attrs) 

455 else: 

456 # *** set `cdata_mode` first so we can override it in `handle_starttag` (see #1036) *** 

457 if tag in self.CDATA_CONTENT_ELEMENTS: 

458 self.set_cdata_mode(tag) 

459 self.handle_starttag(tag, attrs) 

460 return endpos