Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/bs4/__init__.py: 66%

1"""Beautiful Soup Elixir and Tonic - "The Screen-Scraper's Friend". 

2 

3http://www.crummy.com/software/BeautifulSoup/ 

4 

5Beautiful Soup uses a pluggable XML or HTML parser to parse a 

6(possibly invalid) document into a tree representation. Beautiful Soup 

7provides methods and Pythonic idioms that make it easy to navigate, 

8search, and modify the parse tree. 

9 

10Beautiful Soup works with Python 3.7 and up. It works better if lxml 

11and/or html5lib is installed, but they are not required. 

12 

13For more than you ever wanted to know about Beautiful Soup, see the 

14documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ 

15""" 

16 

17__author__ = "Leonard Richardson (leonardr@segfault.org)" 

18__version__ = "4.13.4" 

19__copyright__ = "Copyright (c) 2004-2025 Leonard Richardson" 

20# Use of this source code is governed by the MIT license. 

21__license__ = "MIT" 

22 

23__all__ = [ 

24 "AttributeResemblesVariableWarning", 

25 "BeautifulSoup", 

26 "Comment", 

27 "Declaration", 

28 "ProcessingInstruction", 

29 "ResultSet", 

30 "CSS", 

31 "Script", 

32 "Stylesheet", 

33 "Tag", 

34 "TemplateString", 

35 "ElementFilter", 

36 "UnicodeDammit", 

37 "CData", 

38 "Doctype", 

39 

40 # Exceptions 

41 "FeatureNotFound", 

42 "ParserRejectedMarkup", 

43 "StopParsing", 

44 

45 # Warnings 

46 "AttributeResemblesVariableWarning", 

47 "GuessedAtParserWarning", 

48 "MarkupResemblesLocatorWarning", 

49 "UnusualUsageWarning", 

50 "XMLParsedAsHTMLWarning", 

51] 

52 

53from collections import Counter 

54import sys 

55import warnings 

56 

57# The very first thing we do is give a useful error if someone is 

58# running this code under Python 2. 

59if sys.version_info.major < 3: 

60 raise ImportError( 

61 "You are trying to use a Python 3-specific version of Beautiful Soup under Python 2. This will not work. The final version of Beautiful Soup to support Python 2 was 4.9.3." 

62 ) 

63 

64from .builder import ( 

65 builder_registry, 

66 TreeBuilder, 

67) 

68from .builder._htmlparser import HTMLParserTreeBuilder 

69from .dammit import UnicodeDammit 

70from .css import CSS 

71from ._deprecation import ( 

72 _deprecated, 

73) 

74from .element import ( 

75 CData, 

76 Comment, 

77 DEFAULT_OUTPUT_ENCODING, 

78 Declaration, 

79 Doctype, 

80 NavigableString, 

81 PageElement, 

82 ProcessingInstruction, 

83 PYTHON_SPECIFIC_ENCODINGS, 

84 ResultSet, 

85 Script, 

86 Stylesheet, 

87 Tag, 

88 TemplateString, 

89) 

90from .formatter import Formatter 

91from .filter import ( 

92 ElementFilter, 

93 SoupStrainer, 

94) 

95from typing import ( 

96 Any, 

97 cast, 

98 Counter as CounterType, 

99 Dict, 

100 Iterator, 

101 List, 

102 Sequence, 

103 Optional, 

104 Type, 

105 Union, 

106) 

107 

108from bs4._typing import ( 

109 _Encoding, 

110 _Encodings, 

111 _IncomingMarkup, 

112 _InsertableElement, 

113 _RawAttributeValue, 

114 _RawAttributeValues, 

115 _RawMarkup, 

116) 

117 

118# Import all warnings and exceptions into the main package. 

119from bs4.exceptions import ( 

120 FeatureNotFound, 

121 ParserRejectedMarkup, 

122 StopParsing, 

123) 

124from bs4._warnings import ( 

125 AttributeResemblesVariableWarning, 

126 GuessedAtParserWarning, 

127 MarkupResemblesLocatorWarning, 

128 UnusualUsageWarning, 

129 XMLParsedAsHTMLWarning, 

130) 

131 

132 

133class BeautifulSoup(Tag): 

134 """A data structure representing a parsed HTML or XML document. 

135 

136 Most of the methods you'll call on a BeautifulSoup object are inherited from 

137 PageElement or Tag. 

138 

139 Internally, this class defines the basic interface called by the 

140 tree builders when converting an HTML/XML document into a data 

141 structure. The interface abstracts away the differences between 

142 parsers. To write a new tree builder, you'll need to understand 

143 these methods as a whole. 

144 

145 These methods will be called by the BeautifulSoup constructor: 

146 * reset() 

147 * feed(markup) 

148 

149 The tree builder may call these methods from its feed() implementation: 

150 * handle_starttag(name, attrs) # See note about return value 

151 * handle_endtag(name) 

152 * handle_data(data) # Appends to the current data node 

153 * endData(containerClass) # Ends the current data node 

154 

155 No matter how complicated the underlying parser is, you should be 

156 able to build a tree using 'start tag' events, 'end tag' events, 

157 'data' events, and "done with data" events. 

158 

159 If you encounter an empty-element tag (aka a self-closing tag, 

160 like HTML's <br> tag), call handle_starttag and then 

161 handle_endtag. 

162 """ 

163 

164 #: Since `BeautifulSoup` subclasses `Tag`, it's possible to treat it as 

165 #: a `Tag` with a `Tag.name`. Hoever, this name makes it clear the 

166 #: `BeautifulSoup` object isn't a real markup tag. 

167 ROOT_TAG_NAME: str = "[document]" 

168 

169 #: If the end-user gives no indication which tree builder they 

170 #: want, look for one with these features. 

171 DEFAULT_BUILDER_FEATURES: Sequence[str] = ["html", "fast"] 

172 

173 #: A string containing all ASCII whitespace characters, used in 

174 #: during parsing to detect data chunks that seem 'empty'. 

175 ASCII_SPACES: str = "\x20\x0a\x09\x0c\x0d" 

176 

177 # FUTURE PYTHON: 

178 element_classes: Dict[Type[PageElement], Type[PageElement]] #: :meta private: 

179 builder: TreeBuilder #: :meta private: 

180 is_xml: bool 

181 known_xml: Optional[bool] 

182 parse_only: Optional[SoupStrainer] #: :meta private: 

183 

184 # These members are only used while parsing markup. 

185 markup: Optional[_RawMarkup] #: :meta private: 

186 current_data: List[str] #: :meta private: 

187 currentTag: Optional[Tag] #: :meta private: 

188 tagStack: List[Tag] #: :meta private: 

189 open_tag_counter: CounterType[str] #: :meta private: 

190 preserve_whitespace_tag_stack: List[Tag] #: :meta private: 

191 string_container_stack: List[Tag] #: :meta private: 

192 _most_recent_element: Optional[PageElement] #: :meta private: 

193 

194 #: Beautiful Soup's best guess as to the character encoding of the 

195 #: original document. 

196 original_encoding: Optional[_Encoding] 

197 

198 #: The character encoding, if any, that was explicitly defined 

199 #: in the original document. This may or may not match 

200 #: `BeautifulSoup.original_encoding`. 

201 declared_html_encoding: Optional[_Encoding] 

202 

203 #: This is True if the markup that was parsed contains 

204 #: U+FFFD REPLACEMENT_CHARACTER characters which were not present 

205 #: in the original markup. These mark character sequences that 

206 #: could not be represented in Unicode. 

207 contains_replacement_characters: bool 

208 

209 def __init__( 

210 self, 

211 markup: _IncomingMarkup = "", 

212 features: Optional[Union[str, Sequence[str]]] = None, 

213 builder: Optional[Union[TreeBuilder, Type[TreeBuilder]]] = None, 

214 parse_only: Optional[SoupStrainer] = None, 

215 from_encoding: Optional[_Encoding] = None, 

216 exclude_encodings: Optional[_Encodings] = None, 

217 element_classes: Optional[Dict[Type[PageElement], Type[PageElement]]] = None, 

218 **kwargs: Any, 

219 ): 

220 """Constructor. 

221 

222 :param markup: A string or a file-like object representing 

223 markup to be parsed. 

224 

225 :param features: Desirable features of the parser to be 

226 used. This may be the name of a specific parser ("lxml", 

227 "lxml-xml", "html.parser", or "html5lib") or it may be the 

228 type of markup to be used ("html", "html5", "xml"). It's 

229 recommended that you name a specific parser, so that 

230 Beautiful Soup gives you the same results across platforms 

231 and virtual environments. 

232 

233 :param builder: A TreeBuilder subclass to instantiate (or 

234 instance to use) instead of looking one up based on 

235 `features`. You only need to use this if you've implemented a 

236 custom TreeBuilder. 

237 

238 :param parse_only: A SoupStrainer. Only parts of the document 

239 matching the SoupStrainer will be considered. This is useful 

240 when parsing part of a document that would otherwise be too 

241 large to fit into memory. 

242 

243 :param from_encoding: A string indicating the encoding of the 

244 document to be parsed. Pass this in if Beautiful Soup is 

245 guessing wrongly about the document's encoding. 

246 

247 :param exclude_encodings: A list of strings indicating 

248 encodings known to be wrong. Pass this in if you don't know 

249 the document's encoding but you know Beautiful Soup's guess is 

250 wrong. 

251 

252 :param element_classes: A dictionary mapping BeautifulSoup 

253 classes like Tag and NavigableString, to other classes you'd 

254 like to be instantiated instead as the parse tree is 

255 built. This is useful for subclassing Tag or NavigableString 

256 to modify default behavior. 

257 

258 :param kwargs: For backwards compatibility purposes, the 

259 constructor accepts certain keyword arguments used in 

260 Beautiful Soup 3. None of these arguments do anything in 

261 Beautiful Soup 4; they will result in a warning and then be 

262 ignored. 

263 

264 Apart from this, any keyword arguments passed into the 

265 BeautifulSoup constructor are propagated to the TreeBuilder 

266 constructor. This makes it possible to configure a 

267 TreeBuilder by passing in arguments, not just by saying which 

268 one to use. 

269 """ 

270 if "convertEntities" in kwargs: 

271 del kwargs["convertEntities"] 

272 warnings.warn( 

273 "BS4 does not respect the convertEntities argument to the " 

274 "BeautifulSoup constructor. Entities are always converted " 

275 "to Unicode characters." 

276 ) 

277 

278 if "markupMassage" in kwargs: 

279 del kwargs["markupMassage"] 

280 warnings.warn( 

281 "BS4 does not respect the markupMassage argument to the " 

282 "BeautifulSoup constructor. The tree builder is responsible " 

283 "for any necessary markup massage." 

284 ) 

285 

286 if "smartQuotesTo" in kwargs: 

287 del kwargs["smartQuotesTo"] 

288 warnings.warn( 

289 "BS4 does not respect the smartQuotesTo argument to the " 

290 "BeautifulSoup constructor. Smart quotes are always converted " 

291 "to Unicode characters." 

292 ) 

293 

294 if "selfClosingTags" in kwargs: 

295 del kwargs["selfClosingTags"] 

296 warnings.warn( 

297 "Beautiful Soup 4 does not respect the selfClosingTags argument to the " 

298 "BeautifulSoup constructor. The tree builder is responsible " 

299 "for understanding self-closing tags." 

300 ) 

301 

302 if "isHTML" in kwargs: 

303 del kwargs["isHTML"] 

304 warnings.warn( 

305 "Beautiful Soup 4 does not respect the isHTML argument to the " 

306 "BeautifulSoup constructor. Suggest you use " 

307 "features='lxml' for HTML and features='lxml-xml' for " 

308 "XML." 

309 ) 

310 

311 def deprecated_argument(old_name: str, new_name: str) -> Optional[Any]: 

312 if old_name in kwargs: 

313 warnings.warn( 

314 'The "%s" argument to the BeautifulSoup constructor ' 

315 'was renamed to "%s" in Beautiful Soup 4.0.0' 

316 % (old_name, new_name), 

317 DeprecationWarning, 

318 stacklevel=3, 

319 ) 

320 return kwargs.pop(old_name) 

321 return None 

322 

323 parse_only = parse_only or deprecated_argument("parseOnlyThese", "parse_only") 

324 if parse_only is not None: 

325 # Issue a warning if we can tell in advance that 

326 # parse_only will exclude the entire tree. 

327 if parse_only.excludes_everything: 

328 warnings.warn( 

329 f"The given value for parse_only will exclude everything: {parse_only}", 

330 UserWarning, 

331 stacklevel=3, 

332 ) 

333 

334 from_encoding = from_encoding or deprecated_argument( 

335 "fromEncoding", "from_encoding" 

336 ) 

337 

338 if from_encoding and isinstance(markup, str): 

339 warnings.warn( 

340 "You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored." 

341 ) 

342 from_encoding = None 

343 

344 self.element_classes = element_classes or dict() 

345 

346 # We need this information to track whether or not the builder 

347 # was specified well enough that we can omit the 'you need to 

348 # specify a parser' warning. 

349 original_builder = builder 

350 original_features = features 

351 

352 builder_class: Type[TreeBuilder] 

353 if isinstance(builder, type): 

354 # A builder class was passed in; it needs to be instantiated. 

355 builder_class = builder 

356 builder = None 

357 elif builder is None: 

358 if isinstance(features, str): 

359 features = [features] 

360 if features is None or len(features) == 0: 

361 features = self.DEFAULT_BUILDER_FEATURES 

362 possible_builder_class = builder_registry.lookup(*features) 

363 if possible_builder_class is None: 

364 raise FeatureNotFound( 

365 "Couldn't find a tree builder with the features you " 

366 "requested: %s. Do you need to install a parser library?" 

367 % ",".join(features) 

368 ) 

369 builder_class = possible_builder_class 

370 

371 # At this point either we have a TreeBuilder instance in 

372 # builder, or we have a builder_class that we can instantiate 

373 # with the remaining **kwargs. 

374 if builder is None: 

375 builder = builder_class(**kwargs) 

376 if ( 

377 not original_builder 

378 and not ( 

379 original_features == builder.NAME 

380 or ( 

381 isinstance(original_features, str) 

382 and original_features in builder.ALTERNATE_NAMES 

383 ) 

384 ) 

385 and markup 

386 ): 

387 # The user did not tell us which TreeBuilder to use, 

388 # and we had to guess. Issue a warning. 

389 if builder.is_xml: 

390 markup_type = "XML" 

391 else: 

392 markup_type = "HTML" 

393 

394 # This code adapted from warnings.py so that we get the same line 

395 # of code as our warnings.warn() call gets, even if the answer is wrong 

396 # (as it may be in a multithreading situation). 

397 caller = None 

398 try: 

399 caller = sys._getframe(1) 

400 except ValueError: 

401 pass 

402 if caller: 

403 globals = caller.f_globals 

404 line_number = caller.f_lineno 

405 else: 

406 globals = sys.__dict__ 

407 line_number = 1 

408 filename = globals.get("__file__") 

409 if filename: 

410 fnl = filename.lower() 

411 if fnl.endswith((".pyc", ".pyo")): 

412 filename = filename[:-1] 

413 if filename: 

414 # If there is no filename at all, the user is most likely in a REPL, 

415 # and the warning is not necessary. 

416 values = dict( 

417 filename=filename, 

418 line_number=line_number, 

419 parser=builder.NAME, 

420 markup_type=markup_type, 

421 ) 

422 warnings.warn( 

423 GuessedAtParserWarning.MESSAGE % values, 

424 GuessedAtParserWarning, 

425 stacklevel=2, 

426 ) 

427 else: 

428 if kwargs: 

429 warnings.warn( 

430 "Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`." 

431 ) 

432 

433 self.builder = builder 

434 self.is_xml = builder.is_xml 

435 self.known_xml = self.is_xml 

436 self._namespaces = dict() 

437 self.parse_only = parse_only 

438 

439 if hasattr(markup, "read"): # It's a file-type object. 

440 markup = markup.read() 

441 elif not isinstance(markup, (bytes, str)) and not hasattr(markup, "__len__"): 

442 raise TypeError( 

443 f"Incoming markup is of an invalid type: {markup!r}. Markup must be a string, a bytestring, or an open filehandle." 

444 ) 

445 elif len(markup) <= 256 and ( 

446 (isinstance(markup, bytes) and b"<" not in markup and b"\n" not in markup) 

447 or (isinstance(markup, str) and "<" not in markup and "\n" not in markup) 

448 ): 

449 # Issue warnings for a couple beginner problems 

450 # involving passing non-markup to Beautiful Soup. 

451 # Beautiful Soup will still parse the input as markup, 

452 # since that is sometimes the intended behavior. 

453 if not self._markup_is_url(markup): 

454 self._markup_resembles_filename(markup) 

455 

456 # At this point we know markup is a string or bytestring. If 

457 # it was a file-type object, we've read from it. 

458 markup = cast(_RawMarkup, markup) 

459 

460 rejections = [] 

461 success = False 

462 for ( 

463 self.markup, 

464 self.original_encoding, 

465 self.declared_html_encoding, 

466 self.contains_replacement_characters, 

467 ) in self.builder.prepare_markup( 

468 markup, from_encoding, exclude_encodings=exclude_encodings 

469 ): 

470 self.reset() 

471 self.builder.initialize_soup(self) 

472 try: 

473 self._feed() 

474 success = True 

475 break 

476 except ParserRejectedMarkup as e: 

477 rejections.append(e) 

478 pass 

479 

480 if not success: 

481 other_exceptions = [str(e) for e in rejections] 

482 raise ParserRejectedMarkup( 

483 "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " 

484 + "\n ".join(other_exceptions) 

485 ) 

486 

487 # Clear out the markup and remove the builder's circular 

488 # reference to this object. 

489 self.markup = None 

490 self.builder.soup = None 

491 

492 def copy_self(self) -> "BeautifulSoup": 

493 """Create a new BeautifulSoup object with the same TreeBuilder, 

494 but not associated with any markup. 

495 

496 This is the first step of the deepcopy process. 

497 """ 

498 clone = type(self)("", None, self.builder) 

499 

500 # Keep track of the encoding of the original document, 

501 # since we won't be parsing it again. 

502 clone.original_encoding = self.original_encoding 

503 return clone 

504 

505 def __getstate__(self) -> Dict[str, Any]: 

506 # Frequently a tree builder can't be pickled. 

507 d = dict(self.__dict__) 

508 if "builder" in d and d["builder"] is not None and not self.builder.picklable: 

509 d["builder"] = type(self.builder) 

510 # Store the contents as a Unicode string. 

511 d["contents"] = [] 

512 d["markup"] = self.decode() 

513 

514 # If _most_recent_element is present, it's a Tag object left 

515 # over from initial parse. It might not be picklable and we 

516 # don't need it. 

517 if "_most_recent_element" in d: 

518 del d["_most_recent_element"] 

519 return d 

520 

521 def __setstate__(self, state: Dict[str, Any]) -> None: 

522 # If necessary, restore the TreeBuilder by looking it up. 

523 self.__dict__ = state 

524 if isinstance(self.builder, type): 

525 self.builder = self.builder() 

526 elif not self.builder: 

527 # We don't know which builder was used to build this 

528 # parse tree, so use a default we know is always available. 

529 self.builder = HTMLParserTreeBuilder() 

530 self.builder.soup = self 

531 self.reset() 

532 self._feed() 

533 

534 @classmethod 

535 @_deprecated( 

536 replaced_by="nothing (private method, will be removed)", version="4.13.0" 

537 ) 

538 def _decode_markup(cls, markup: _RawMarkup) -> str: 

539 """Ensure `markup` is Unicode so it's safe to send into warnings.warn. 

540 

541 warnings.warn had this problem back in 2010 but fortunately 

542 not anymore. This has not been used for a long time; I just 

543 noticed that fact while working on 4.13.0. 

544 """ 

545 if isinstance(markup, bytes): 

546 decoded = markup.decode("utf-8", "replace") 

547 else: 

548 decoded = markup 

549 return decoded 

550 

551 @classmethod 

552 def _markup_is_url(cls, markup: _RawMarkup) -> bool: 

553 """Error-handling method to raise a warning if incoming markup looks 

554 like a URL. 

555 

556 :param markup: A string of markup. 

557 :return: Whether or not the markup resembled a URL 

558 closely enough to justify issuing a warning. 

559 """ 

560 problem: bool = False 

561 if isinstance(markup, bytes): 

562 problem = ( 

563 any(markup.startswith(prefix) for prefix in (b"http:", b"https:")) 

564 and b" " not in markup 

565 ) 

566 elif isinstance(markup, str): 

567 problem = ( 

568 any(markup.startswith(prefix) for prefix in ("http:", "https:")) 

569 and " " not in markup 

570 ) 

571 else: 

572 return False 

573 

574 if not problem: 

575 return False 

576 warnings.warn( 

577 MarkupResemblesLocatorWarning.URL_MESSAGE % dict(what="URL"), 

578 MarkupResemblesLocatorWarning, 

579 stacklevel=3, 

580 ) 

581 return True 

582 

583 @classmethod 

584 def _markup_resembles_filename(cls, markup: _RawMarkup) -> bool: 

585 """Error-handling method to issue a warning if incoming markup 

586 resembles a filename. 

587 

588 :param markup: A string of markup. 

589 :return: Whether or not the markup resembled a filename 

590 closely enough to justify issuing a warning. 

591 """ 

592 markup_b: bytes 

593 

594 # We're only checking ASCII characters, so rather than write 

595 # the same tests twice, convert Unicode to a bytestring and 

596 # operate on the bytestring. 

597 if isinstance(markup, str): 

598 markup_b = markup.encode("utf8") 

599 else: 

600 markup_b = markup 

601 

602 # Step 1: does it end with a common textual file extension? 

603 filelike = False 

604 lower = markup_b.lower() 

605 extensions = [b".html", b".htm", b".xml", b".xhtml", b".txt"] 

606 if any(lower.endswith(ext) for ext in extensions): 

607 filelike = True 

608 if not filelike: 

609 return False 

610 

611 # Step 2: it _might_ be a file, but there are a few things 

612 # we can look for that aren't very common in filenames. 

613 

614 # Characters that have special meaning to Unix shells. (< was 

615 # excluded before this method was called.) 

616 # 

617 # Many of these are also reserved characters that cannot 

618 # appear in Windows filenames. 

619 for byte in markup_b: 

620 if byte in b"?*#&;>$|": 

621 return False 

622 

623 # Two consecutive forward slashes (as seen in a URL) or two 

624 # consecutive spaces (as seen in fixed-width data). 

625 # 

626 # (Paths to Windows network shares contain consecutive 

627 # backslashes, so checking that doesn't seem as helpful.) 

628 if b"//" in markup_b: 

629 return False 

630 if b" " in markup_b: 

631 return False 

632 

633 # A colon in any position other than position 1 (e.g. after a 

634 # Windows drive letter). 

635 if markup_b.startswith(b":"): 

636 return False 

637 colon_i = markup_b.rfind(b":") 

638 if colon_i not in (-1, 1): 

639 return False 

640 

641 # Step 3: If it survived all of those checks, it's similar 

642 # enough to a file to justify issuing a warning. 

643 warnings.warn( 

644 MarkupResemblesLocatorWarning.FILENAME_MESSAGE % dict(what="filename"), 

645 MarkupResemblesLocatorWarning, 

646 stacklevel=3, 

647 ) 

648 return True 

649 

650 def _feed(self) -> None: 

651 """Internal method that parses previously set markup, creating a large 

652 number of Tag and NavigableString objects. 

653 """ 

654 # Convert the document to Unicode. 

655 self.builder.reset() 

656 

657 if self.markup is not None: 

658 self.builder.feed(self.markup) 

659 # Close out any unfinished strings and close all the open tags. 

660 self.endData() 

661 while ( 

662 self.currentTag is not None and self.currentTag.name != self.ROOT_TAG_NAME 

663 ): 

664 self.popTag() 

665 

666 def reset(self) -> None: 

667 """Reset this object to a state as though it had never parsed any 

668 markup. 

669 """ 

670 Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) 

671 self.hidden = True 

672 self.builder.reset() 

673 self.current_data = [] 

674 self.currentTag = None 

675 self.tagStack = [] 

676 self.open_tag_counter = Counter() 

677 self.preserve_whitespace_tag_stack = [] 

678 self.string_container_stack = [] 

679 self._most_recent_element = None 

680 self.pushTag(self) 

681 

682 def new_tag( 

683 self, 

684 name: str, 

685 namespace: Optional[str] = None, 

686 nsprefix: Optional[str] = None, 

687 attrs: Optional[_RawAttributeValues] = None, 

688 sourceline: Optional[int] = None, 

689 sourcepos: Optional[int] = None, 

690 string: Optional[str] = None, 

691 **kwattrs: _RawAttributeValue, 

692 ) -> Tag: 

693 """Create a new Tag associated with this BeautifulSoup object. 

694 

695 :param name: The name of the new Tag. 

696 :param namespace: The URI of the new Tag's XML namespace, if any. 

697 :param prefix: The prefix for the new Tag's XML namespace, if any. 

698 :param attrs: A dictionary of this Tag's attribute values; can 

699 be used instead of ``kwattrs`` for attributes like 'class' 

700 that are reserved words in Python. 

701 :param sourceline: The line number where this tag was 

702 (purportedly) found in its source document. 

703 :param sourcepos: The character position within ``sourceline`` where this 

704 tag was (purportedly) found. 

705 :param string: String content for the new Tag, if any. 

706 :param kwattrs: Keyword arguments for the new Tag's attribute values. 

707 

708 """ 

709 attr_container = self.builder.attribute_dict_class(**kwattrs) 

710 if attrs is not None: 

711 attr_container.update(attrs) 

712 tag_class = self.element_classes.get(Tag, Tag) 

713 

714 # Assume that this is either Tag or a subclass of Tag. If not, 

715 # the user brought type-unsafety upon themselves. 

716 tag_class = cast(Type[Tag], tag_class) 

717 tag = tag_class( 

718 None, 

719 self.builder, 

720 name, 

721 namespace, 

722 nsprefix, 

723 attr_container, 

724 sourceline=sourceline, 

725 sourcepos=sourcepos, 

726 ) 

727 

728 if string is not None: 

729 tag.string = string 

730 return tag 

731 

732 def string_container( 

733 self, base_class: Optional[Type[NavigableString]] = None 

734 ) -> Type[NavigableString]: 

735 """Find the class that should be instantiated to hold a given kind of 

736 string. 

737 

738 This may be a built-in Beautiful Soup class or a custom class passed 

739 in to the BeautifulSoup constructor. 

740 """ 

741 container = base_class or NavigableString 

742 

743 # The user may want us to use some other class (hopefully a 

744 # custom subclass) instead of the one we'd use normally. 

745 container = cast( 

746 Type[NavigableString], self.element_classes.get(container, container) 

747 ) 

748 

749 # On top of that, we may be inside a tag that needs a special 

750 # container class. 

751 if self.string_container_stack and container is NavigableString: 

752 container = self.builder.string_containers.get( 

753 self.string_container_stack[-1].name, container 

754 ) 

755 return container 

756 

757 def new_string( 

758 self, s: str, subclass: Optional[Type[NavigableString]] = None 

759 ) -> NavigableString: 

760 """Create a new `NavigableString` associated with this `BeautifulSoup` 

761 object. 

762 

763 :param s: The string content of the `NavigableString` 

764 :param subclass: The subclass of `NavigableString`, if any, to 

765 use. If a document is being processed, an appropriate 

766 subclass for the current location in the document will 

767 be determined automatically. 

768 """ 

769 container = self.string_container(subclass) 

770 return container(s) 

771 

772 def insert_before(self, *args: _InsertableElement) -> List[PageElement]: 

773 """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement 

774 it because there is nothing before or after it in the parse tree. 

775 """ 

776 raise NotImplementedError( 

777 "BeautifulSoup objects don't support insert_before()." 

778 ) 

779 

780 def insert_after(self, *args: _InsertableElement) -> List[PageElement]: 

781 """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement 

782 it because there is nothing before or after it in the parse tree. 

783 """ 

784 raise NotImplementedError("BeautifulSoup objects don't support insert_after().") 

785 

786 def popTag(self) -> Optional[Tag]: 

787 """Internal method called by _popToTag when a tag is closed. 

788 

789 :meta private: 

790 """ 

791 if not self.tagStack: 

792 # Nothing to pop. This shouldn't happen. 

793 return None 

794 tag = self.tagStack.pop() 

795 if tag.name in self.open_tag_counter: 

796 self.open_tag_counter[tag.name] -= 1 

797 if ( 

798 self.preserve_whitespace_tag_stack 

799 and tag == self.preserve_whitespace_tag_stack[-1] 

800 ): 

801 self.preserve_whitespace_tag_stack.pop() 

802 if self.string_container_stack and tag == self.string_container_stack[-1]: 

803 self.string_container_stack.pop() 

804 # print("Pop", tag.name) 

805 if self.tagStack: 

806 self.currentTag = self.tagStack[-1] 

807 return self.currentTag 

808 

809 def pushTag(self, tag: Tag) -> None: 

810 """Internal method called by handle_starttag when a tag is opened. 

811 

812 :meta private: 

813 """ 

814 # print("Push", tag.name) 

815 if self.currentTag is not None: 

816 self.currentTag.contents.append(tag) 

817 self.tagStack.append(tag) 

818 self.currentTag = self.tagStack[-1] 

819 if tag.name != self.ROOT_TAG_NAME: 

820 self.open_tag_counter[tag.name] += 1 

821 if tag.name in self.builder.preserve_whitespace_tags: 

822 self.preserve_whitespace_tag_stack.append(tag) 

823 if tag.name in self.builder.string_containers: 

824 self.string_container_stack.append(tag) 

825 

826 def endData(self, containerClass: Optional[Type[NavigableString]] = None) -> None: 

827 """Method called by the TreeBuilder when the end of a data segment 

828 occurs. 

829 

830 :param containerClass: The class to use when incorporating the 

831 data segment into the parse tree. 

832 

833 :meta private: 

834 """ 

835 if self.current_data: 

836 current_data = "".join(self.current_data) 

837 # If whitespace is not preserved, and this string contains 

838 # nothing but ASCII spaces, replace it with a single space 

839 # or newline. 

840 if not self.preserve_whitespace_tag_stack: 

841 strippable = True 

842 for i in current_data: 

843 if i not in self.ASCII_SPACES: 

844 strippable = False 

845 break 

846 if strippable: 

847 if "\n" in current_data: 

848 current_data = "\n" 

849 else: 

850 current_data = " " 

851 

852 # Reset the data collector. 

853 self.current_data = [] 

854 

855 # Should we add this string to the tree at all? 

856 if ( 

857 self.parse_only 

858 and len(self.tagStack) <= 1 

859 and (not self.parse_only.allow_string_creation(current_data)) 

860 ): 

861 return 

862 

863 containerClass = self.string_container(containerClass) 

864 o = containerClass(current_data) 

865 self.object_was_parsed(o) 

866 

867 def object_was_parsed( 

868 self, 

869 o: PageElement, 

870 parent: Optional[Tag] = None, 

871 most_recent_element: Optional[PageElement] = None, 

872 ) -> None: 

873 """Method called by the TreeBuilder to integrate an object into the 

874 parse tree. 

875 

876 :meta private: 

877 """ 

878 if parent is None: 

879 parent = self.currentTag 

880 assert parent is not None 

881 previous_element: Optional[PageElement] 

882 if most_recent_element is not None: 

883 previous_element = most_recent_element 

884 else: 

885 previous_element = self._most_recent_element 

886 

887 next_element = previous_sibling = next_sibling = None 

888 if isinstance(o, Tag): 

889 next_element = o.next_element 

890 next_sibling = o.next_sibling 

891 previous_sibling = o.previous_sibling 

892 if previous_element is None: 

893 previous_element = o.previous_element 

894 

895 fix = parent.next_element is not None 

896 

897 o.setup(parent, previous_element, next_element, previous_sibling, next_sibling) 

898 

899 self._most_recent_element = o 

900 parent.contents.append(o) 

901 

902 # Check if we are inserting into an already parsed node. 

903 if fix: 

904 self._linkage_fixer(parent) 

905 

906 def _linkage_fixer(self, el: Tag) -> None: 

907 """Make sure linkage of this fragment is sound.""" 

908 

909 first = el.contents[0] 

910 child = el.contents[-1] 

911 descendant: PageElement = child 

912 

913 if child is first and el.parent is not None: 

914 # Parent should be linked to first child 

915 el.next_element = child 

916 # We are no longer linked to whatever this element is 

917 prev_el = child.previous_element 

918 if prev_el is not None and prev_el is not el: 

919 prev_el.next_element = None 

920 # First child should be linked to the parent, and no previous siblings. 

921 child.previous_element = el 

922 child.previous_sibling = None 

923 

924 # We have no sibling as we've been appended as the last. 

925 child.next_sibling = None 

926 

927 # This index is a tag, dig deeper for a "last descendant" 

928 if isinstance(child, Tag) and child.contents: 

929 # _last_decendant is typed as returning Optional[PageElement], 

930 # but the value can't be None here, because el is a Tag 

931 # which we know has contents. 

932 descendant = cast(PageElement, child._last_descendant(False)) 

933 

934 # As the final step, link last descendant. It should be linked 

935 # to the parent's next sibling (if found), else walk up the chain 

936 # and find a parent with a sibling. It should have no next sibling. 

937 descendant.next_element = None 

938 descendant.next_sibling = None 

939 

940 target: Optional[Tag] = el 

941 while True: 

942 if target is None: 

943 break 

944 elif target.next_sibling is not None: 

945 descendant.next_element = target.next_sibling 

946 target.next_sibling.previous_element = child 

947 break 

948 target = target.parent 

949 

950 def _popToTag( 

951 self, name: str, nsprefix: Optional[str] = None, inclusivePop: bool = True 

952 ) -> Optional[Tag]: 

953 """Pops the tag stack up to and including the most recent 

954 instance of the given tag. 

955 

956 If there are no open tags with the given name, nothing will be 

957 popped. 

958 

959 :param name: Pop up to the most recent tag with this name. 

960 :param nsprefix: The namespace prefix that goes with `name`. 

961 :param inclusivePop: It this is false, pops the tag stack up 

962 to but *not* including the most recent instqance of the 

963 given tag.