Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/markdown/htmlparser.py: 99%

1# Python Markdown 

2 

3# A Python implementation of John Gruber's Markdown. 

4 

5# Documentation: https://python-markdown.github.io/ 

6# GitHub: https://github.com/Python-Markdown/markdown/ 

7# PyPI: https://pypi.org/project/Markdown/ 

8 

9# Started by Manfred Stienstra (http://www.dwerg.net/). 

10# Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org). 

11# Currently maintained by Waylan Limberg (https://github.com/waylan), 

12# Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser). 

13 

14# Copyright 2007-2023 The Python Markdown Project (v. 1.7 and later) 

15# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) 

16# Copyright 2004 Manfred Stienstra (the original version) 

17 

18# License: BSD (see LICENSE.md for details). 

19 

20""" 

21This module imports a copy of [`html.parser.HTMLParser`][] and modifies it heavily through monkey-patches. 

22A copy is imported rather than the module being directly imported as this ensures that the user can import 

23and use the unmodified library for their own needs. 

24""" 

25 

26from __future__ import annotations 

27 

28import re 

29import importlib.util 

30import sys 

31from typing import TYPE_CHECKING, Sequence 

32 

33if TYPE_CHECKING: # pragma: no cover 

34 from markdown import Markdown 

35 

36# Included for versions which do not have current comment fix 

37commentclose = re.compile(r'--!?>') 

38 

39# Import a copy of the html.parser lib as `htmlparser` so we can monkeypatch it. 

40# Users can still do `from html import parser` and get the default behavior. 

41spec = importlib.util.find_spec('html.parser') 

42htmlparser = importlib.util.module_from_spec(spec) 

43spec.loader.exec_module(htmlparser) 

44sys.modules['htmlparser'] = htmlparser 

45 

46# This is a hack. We are sneaking in `</>` so we can capture it without the HTML parser 

47# throwing it away. When we see it, we will process it as data. 

48htmlparser.starttagopen = re.compile('<[a-zA-Z]|</>') 

49 

50htmlparser.endtagopen = re.compile('</[a-zA-Z]?') 

51 

52# Monkeypatch `HTMLParser` to only accept `?>` to close Processing Instructions. 

53htmlparser.piclose = re.compile(r'\?>') 

54# Monkeypatch `HTMLParser` to only recognize entity references with a closing semicolon. 

55htmlparser.entityref = re.compile(r'&([a-zA-Z][-.a-zA-Z0-9]*);') 

56# Monkeypatch `HTMLParser` to no longer support partial entities. We are always feeding a complete block, 

57# so the 'incomplete' functionality is unnecessary. As the `entityref` regex is run right before incomplete, 

58# and the two regex are the same, then incomplete will simply never match and we avoid the logic within. 

59htmlparser.incomplete = htmlparser.entityref 

60# Monkeypatch `HTMLParser` to not accept a backtick in a tag name, attribute name, or bare value. 

61htmlparser.locatestarttagend_tolerant = re.compile(r""" 

62 <[a-zA-Z][^`\t\n\r\f />\x00]* # tag name <= added backtick here 

63 (?:[\s/]* # optional whitespace before attribute name 

64 (?:(?<=['"\s/])[^`\s/>][^\s/=>]* # attribute name <= added backtick here 

65 (?:\s*=+\s* # value indicator 

66 (?:'[^']*' # LITA-enclosed value 

67 |"[^"]*" # LIT-enclosed value 

68 |(?!['"])[^`>\s]* # bare value <= added backtick here 

69 ) 

70 (?:\s*,)* # possibly followed by a comma 

71 )?(?:\s|/(?!>))* 

72 )* 

73 )? 

74 \s* # trailing whitespace 

75""", re.VERBOSE) 

76htmlparser.locatetagend = re.compile(r""" 

77 [a-zA-Z][^`\t\n\r\f />]* # tag name 

78 [\t\n\r\f /]* # optional whitespace before attribute name 

79 (?:(?<=['"\t\n\r\f /])[^`\t\n\r\f />][^\t\n\r\f /=>]* # attribute name 

80 (?:= # value indicator 

81 (?:'[^']*' # LITA-enclosed value 

82 |"[^"]*" # LIT-enclosed value 

83 |(?!['"])[^>\t\n\r\f ]* # bare value 

84 ) 

85 )? 

86 [\t\n\r\f /]* # possibly followed by a space 

87 )* 

88 >? 

89""", re.VERBOSE) 

90 

91# Match a blank line at the start of a block of text (two newlines). 

92# The newlines may be preceded by additional whitespace. 

93blank_line_re = re.compile(r'^([ ]*\n){2}') 

94 

95# Match a blank line anywhere in a run of text. 

96inner_blank_line_re = re.compile(r'\n[ \t]*\n') 

97 

98# Elements which never have an end tag and so cannot contain a comment. 

99void_tags = frozenset([ 

100 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr' 

101]) 

102 

103 

104class _HTMLParser(htmlparser.HTMLParser): 

105 """Handle special start and end tags.""" 

106 

107 def parse_endtag(self, i): 

108 start = self.rawdata[i:i+3] 

109 c = ord(start[-1]) 

110 is_data = len(start) < 3 or not (65 <= c <= 90 or 97 <= c <= 122) 

111 pos = super().parse_endtag(i) if not is_data else -1 

112 if pos == -1: 

113 self.handle_data(self.rawdata[i:i + 2]) 

114 return i + 2 

115 return pos 

116 

117 def parse_starttag(self, i: int) -> int: # pragma: no cover 

118 # Treat `</>` as normal data as it is not a real tag. 

119 if self.rawdata[i:i + 3] == '</>': 

120 self.handle_data(self.rawdata[i:i + 3]) 

121 return i + 3 

122 

123 return super().parse_starttag(i) 

124 

125 

126# Overwrite our custom one for people like MkDocs that pull it in 

127htmlparser.HTMLParser = _HTMLParser 

128 

129 

130class HTMLExtractor(htmlparser.HTMLParser): 

131 """ 

132 Extract raw HTML from text. 

133 

134 The raw HTML is stored in the [`htmlStash`][markdown.util.HtmlStash] of the 

135 [`Markdown`][markdown.Markdown] instance passed to `md` and the remaining text 

136 is stored in `cleandoc` as a list of strings. 

137 """ 

138 

139 def __init__(self, md: Markdown, *args, **kwargs): 

140 if 'convert_charrefs' not in kwargs: 

141 kwargs['convert_charrefs'] = False 

142 

143 # Block tags that should contain no content (self closing) 

144 self.empty_tags = set(['hr']) 

145 

146 self.lineno_start_cache = [0] 

147 

148 # This calls self.reset 

149 super().__init__(*args, **kwargs) 

150 self.md = md 

151 

152 def reset(self): 

153 """Reset this instance. Loses all unprocessed data.""" 

154 self.inraw = False 

155 self.intail = False 

156 self.stack: list[str] = [] # When `inraw==True`, stack contains a list of tags 

157 self.inline_stack: list[str] = [] # Unclosed raw inline tags in the current paragraph 

158 self._cache: list[str] = [] 

159 self.cleandoc: list[str] = [] 

160 self.lineno_start_cache = [0] 

161 # The `rawdata` and position of the last failed search for a comment close. 

162 self._no_commentclose: tuple[str | None, int] = (None, 0) 

163 

164 super().reset() 

165 

166 def close(self): 

167 """Handle any buffered data.""" 

168 super().close() 

169 if len(self.rawdata): 

170 # Temp fix for https://bugs.python.org/issue41989 

171 # TODO: remove this when the bug is fixed in all supported Python versions. 

172 if self.convert_charrefs and not self.cdata_elem: # pragma: no cover 

173 self.handle_data(htmlparser.unescape(self.rawdata)) 

174 else: 

175 self.handle_data(self.rawdata) 

176 # Handle any unclosed tags. 

177 if len(self._cache): 

178 self.cleandoc.append(self.md.htmlStash.store(''.join(self._cache))) 

179 self._cache = [] 

180 

181 @property 

182 def line_offset(self) -> int: 

183 """Returns char index in `self.rawdata` for the start of the current line. """ 

184 for ii in range(len(self.lineno_start_cache)-1, self.lineno-1): 

185 last_line_start_pos = self.lineno_start_cache[ii] 

186 lf_pos = self.rawdata.find('\n', last_line_start_pos) 

187 if lf_pos == -1: 

188 # No more newlines found. Use end of raw data as start of line beyond end. 

189 lf_pos = len(self.rawdata) 

190 self.lineno_start_cache.append(lf_pos+1) 

191 

192 return self.lineno_start_cache[self.lineno-1] 

193 

194 def at_line_start(self) -> bool: 

195 """ 

196 Returns True if current position is at start of line. 

197 

198 Allows for up to three blank spaces at start of line. 

199 """ 

200 if self.offset == 0: 

201 return True 

202 if self.offset > 3: 

203 return False 

204 # Confirm up to first 3 chars are whitespace 

205 return self.rawdata[self.line_offset:self.line_offset + self.offset].strip() == '' 

206 

207 def get_endtag_text(self, tag: str) -> str: 

208 """ 

209 Returns the text of the end tag. 

210 

211 If it fails to extract the actual text from the raw data, it builds a closing tag with `tag`. 

212 """ 

213 # Attempt to extract actual tag from raw source text 

214 start = self.line_offset + self.offset 

215 m = htmlparser.endendtag.search(self.rawdata, start) 

216 if m: 

217 return self.rawdata[start:m.end()] 

218 else: # pragma: no cover 

219 # Failed to extract from raw data. Assume well formed and lowercase. 

220 return '</{}>'.format(tag) 

221 

222 def handle_starttag(self, tag: str, attrs: Sequence[tuple[str, str]]): 

223 # Handle tags that should always be empty and do not specify a closing tag 

224 if tag in self.empty_tags: 

225 self.handle_startendtag(tag, attrs) 

226 return 

227 

228 if self.md.is_block_level(tag) and (self.intail or (self.at_line_start() and not self.inraw)): 

229 # Started a new raw block. Prepare stack. 

230 self.inraw = True 

231 self.cleandoc.append('\n') 

232 

233 text = self.get_starttag_text() 

234 if self.inraw: 

235 self.stack.append(tag) 

236 self._cache.append(text) 

237 else: 

238 self.cleandoc.append(text) 

239 self.open_inline_tag(tag) 

240 if tag in self.CDATA_CONTENT_ELEMENTS: 

241 # This is presumably a standalone tag in a code span (see #1036). 

242 self.clear_cdata_mode() 

243 

244 def handle_endtag(self, tag: str): 

245 text = self.get_endtag_text(tag) 

246 

247 if self.inraw: 

248 self._cache.append(text) 

249 if tag in self.stack: 

250 # Remove tag from stack 

251 while self.stack: 

252 if self.stack.pop() == tag: 

253 break 

254 if len(self.stack) == 0: 

255 # End of raw block. 

256 if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(text):]): 

257 # Preserve blank line and end of raw block. 

258 self._cache.append('\n') 

259 else: 

260 # More content exists after `endtag`. 

261 self.intail = True 

262 # Reset stack. 

263 self.inraw = False 

264 self.cleandoc.append(self.md.htmlStash.store(''.join(self._cache))) 

265 # Insert blank line between this and next line. 

266 self.cleandoc.append('\n\n') 

267 self._cache = [] 

268 else: 

269 self.cleandoc.append(text) 

270 self.close_inline_tag(tag) 

271 

272 def open_inline_tag(self, tag: str): 

273 """ Track an unclosed raw inline tag so a comment inside it is not treated as a block. """ 

274 if not self.md.is_block_level(tag) and tag not in void_tags: 

275 self.inline_stack.append(tag) 

276 

277 def close_inline_tag(self, tag: str): 

278 """ Stop tracking an inline tag and any unclosed inline tags opened after it. """ 

279 if tag in self.inline_stack: 

280 while self.inline_stack: 

281 if self.inline_stack.pop() == tag: 

282 break 

283 

284 def inline_close_follows(self, text: str) -> bool: 

285 """ Return `True` if an open inline tag is closed in the same paragraph after the comment `text`. """ 

286 if not self.inline_stack: 

287 return False 

288 rest = inner_blank_line_re.split(self.rawdata[self.line_offset + self.offset + len(text):], 1)[0] 

289 return any(re.search(r'</\s*{}\s*>'.format(re.escape(tag)), rest, re.I) for tag in self.inline_stack) 

290 

291 def handle_data(self, data: str): 

292 if self.intail and '\n' in data: 

293 self.intail = False 

294 if self.inraw: 

295 self._cache.append(data) 

296 else: 

297 if inner_blank_line_re.search(data): 

298 # A blank line ends the paragraph, so open inline tags no longer enclose what follows. 

299 self.inline_stack = [] 

300 self.cleandoc.append(data) 

301 

302 def handle_empty_tag(self, data: str, is_block: bool): 

303 """ Handle empty tags (`<data>`). """ 

304 if self.inraw or self.intail: 

305 # Append this to the existing raw block 

306 self._cache.append(data) 

307 elif self.at_line_start() and is_block: 

308 # Handle this as a standalone raw block 

309 if blank_line_re.match(self.rawdata[self.line_offset + self.offset + len(data):]): 

310 # Preserve blank line after tag in raw block. 

311 data += '\n' 

312 else: 

313 # More content exists after tag. 

314 self.intail = True 

315 item = self.cleandoc[-1] if self.cleandoc else '' 

316 # If we only have one newline before block element, add another 

317 if not item.endswith('\n\n') and item.endswith('\n'): 

318 self.cleandoc.append('\n') 

319 self.cleandoc.append(self.md.htmlStash.store(data)) 

320 # Insert blank line between this and next line. 

321 self.cleandoc.append('\n\n') 

322 else: 

323 self.cleandoc.append(data) 

324 

325 def handle_startendtag(self, tag: str, attrs): 

326 self.handle_empty_tag(self.get_starttag_text(), is_block=self.md.is_block_level(tag)) 

327 

328 def handle_charref(self, name: str): 

329 self.handle_empty_tag('&#{};'.format(name), is_block=False) 

330 

331 def handle_entityref(self, name: str): 

332 self.handle_empty_tag('&{};'.format(name), is_block=False) 

333 

334 def handle_comment(self, data: str): 

335 # Check if the comment is unclosed, if so, we need to override position 

336 text = '<!--{}-->'.format(data) 

337 # A comment inside a raw inline element is part of that inline content, unless it contains a blank line. 

338 is_block = bool(inner_blank_line_re.search(data)) or not self.inline_close_follows(text) 

339 self.handle_empty_tag(text, is_block=is_block) 

340 

341 def handle_decl(self, data: str): 

342 self.handle_empty_tag('<!{}>'.format(data), is_block=True) 

343 

344 def handle_pi(self, data: str): 

345 self.handle_empty_tag('<?{}?>'.format(data), is_block=True) 

346 

347 def unknown_decl(self, data: str): 

348 end = ']]>' if data.startswith('CDATA[') else ']>' 

349 self.handle_empty_tag('<![{}{}'.format(data, end), is_block=True) 

350 

351 def parse_pi(self, i: int) -> int: 

352 if self.at_line_start() or self.intail: 

353 return super().parse_pi(i) 

354 # This is not the beginning of a raw block so treat as plain data 

355 # and avoid consuming any tags which may follow (see #1066). 

356 self.handle_data('<?') 

357 return i + 2 

358 

359 # Internal -- parse comment, return length or -1 if not terminated 

360 # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state 

361 def parse_comment(self, i, report=True): 

362 rawdata = self.rawdata 

363 assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()' 

364 # If no close was found after an earlier position in the same `rawdata`, there 

365 # is none after this one either. Don't rescan the rest of the document again. 

366 searched, pos = self._no_commentclose 

367 if searched is rawdata and i + 4 >= pos: 

368 match = None 

369 else: 

370 match = commentclose.search(rawdata, i+4) 

371 if not match: 

372 self._no_commentclose = (rawdata, i + 4) 

373 self.handle_data('<') 

374 return i + 1 

375 if report: 

376 j = match.start() 

377 self.handle_comment(rawdata[i+4: j]) 

378 return match.end() 

379 

380 def parse_html_declaration(self, i: int) -> int: 

381 if self.at_line_start() or self.intail: 

382 if self.rawdata[i:i+3] == '<![' and not self.rawdata[i:i+9] == '<![CDATA[': 

383 # We have encountered the bug in #1534 (Python bug `gh-77057`). 

384 # Provide an override until we drop support for Python < 3.13. 

385 result = self.parse_bogus_comment(i) 

386 if result == -1: 

387 self.handle_data(self.rawdata[i:i + 1]) 

388 return i + 1 

389 return result 

390 return super().parse_html_declaration(i) 

391 # This is not the beginning of a raw block so treat as plain data 

392 # and avoid consuming any tags which may follow (see #1066). 

393 self.handle_data('<!') 

394 return i + 2 

395 

396 def parse_bogus_comment(self, i: int, report: int = 0) -> int: 

397 # Override the default behavior so that bogus comments get passed 

398 # through unaltered by setting `report` to `0` (see #1425). 

399 pos = super().parse_bogus_comment(i, report) 

400 if pos == -1: # pragma: no cover 

401 return -1 

402 self.handle_empty_tag(self.rawdata[i:pos], is_block=False) 

403 return pos 

404 

405 # The rest has been copied from base class in standard lib to address #1036. 

406 # As `__startag_text` is private, all references to it must be in this subclass. 

407 # The last few lines of `parse_starttag` are reversed so that `handle_starttag` 

408 # can override `cdata_mode` in certain situations (in a code span). 

409 __starttag_text: str | None = None 

410 

411 def get_starttag_text(self) -> str: 

412 """Return full source of start tag: `<...>`.""" 

413 return self.__starttag_text 

414 

415 def parse_starttag(self, i: int) -> int: # pragma: no cover 

416 # Treat `</>` as normal data as it is not a real tag. 

417 if self.rawdata[i:i + 3] == '</>': 

418 self.handle_data(self.rawdata[i:i + 3]) 

419 return i + 3 

420 

421 self.__starttag_text = None 

422 endpos = self.check_for_whole_start_tag(i) 

423 if endpos < 0: 

424 self.handle_data(self.rawdata[i:i + 1]) 

425 return i + 1 

426 rawdata = self.rawdata 

427 self.__starttag_text = rawdata[i:endpos] 

428 

429 # Now parse the data between `i+1` and `j` into a tag and `attrs` 

430 attrs = [] 

431 match = htmlparser.tagfind_tolerant.match(rawdata, i+1) 

432 assert match, 'unexpected call to parse_starttag()' 

433 k = match.end() 

434 self.lasttag = tag = match.group(1).lower() 

435 while k < endpos: 

436 m = htmlparser.attrfind_tolerant.match(rawdata, k) 

437 if not m: 

438 break 

439 attrname, rest, attrvalue = m.group(1, 2, 3) 

440 if not rest: 

441 attrvalue = None 

442 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ 

443 attrvalue[:1] == '"' == attrvalue[-1:]: # noqa: E127 

444 attrvalue = attrvalue[1:-1] 

445 if attrvalue: 

446 attrvalue = htmlparser.unescape(attrvalue) 

447 attrs.append((attrname.lower(), attrvalue)) 

448 k = m.end() 

449 

450 end = rawdata[k:endpos].strip() 

451 if end not in (">", "/>"): 

452 lineno, offset = self.getpos() 

453 if "\n" in self.__starttag_text: 

454 lineno = lineno + self.__starttag_text.count("\n") 

455 offset = len(self.__starttag_text) \ 

456 - self.__starttag_text.rfind("\n") # noqa: E127 

457 else: 

458 offset = offset + len(self.__starttag_text) 

459 self.handle_data(rawdata[i:endpos]) 

460 return endpos 

461 if end.endswith('/>'): 

462 # XHTML-style empty tag: `<span attr="value" />` 

463 self.handle_startendtag(tag, attrs) 

464 else: 

465 # *** set `cdata_mode` first so we can override it in `handle_starttag` (see #1036) *** 

466 if tag in self.CDATA_CONTENT_ELEMENTS: 

467 self.set_cdata_mode(tag) 

468 self.handle_starttag(tag, attrs) 

469 return endpos