Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/bs4/element.py: 55%

1739 """An `HTML comment <https://dev.w3.org/html5/spec-LC/syntax.html#comments>`_ or `XML comment <https://www.w3.org/TR/REC-xml/#sec-comments>`_."""

1740

1741 PREFIX: str = "<!--"

1742 SUFFIX: str = "-->"

1743

1744

1745class Declaration(PreformattedString):

1746 """An `XML declaration <https://www.w3.org/TR/REC-xml/#sec-prolog-dtd>`_."""

1747

1748 PREFIX: str = "<?"

1749 SUFFIX: str = "?>"

1750

1751

1752class Doctype(PreformattedString):

1753 """A `document type declaration <https://www.w3.org/TR/REC-xml/#dt-doctype>`_."""

1754

1755 @classmethod

1756 def for_name_and_ids(

1757 cls, name: str, pub_id: Optional[str], system_id: Optional[str]

1758 ) -> Doctype:

1759 """Generate an appropriate document type declaration for a given

1760 public ID and system ID.

1761

1762 :param name: The name of the document's root element, e.g. 'html'.

1763 :param pub_id: The Formal Public Identifier for this document type,

1764 e.g. '-//W3C//DTD XHTML 1.1//EN'

1765 :param system_id: The system identifier for this document type,

1766 e.g. 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'

1767 """

1768 return Doctype(cls._string_for_name_and_ids(name, pub_id, system_id))

1769

1770 @classmethod

1771 def _string_for_name_and_ids(

1772 cls, name: str, pub_id: Optional[str], system_id: Optional[str]

1773 ) -> str:

1774 """Generate a string to be used as the basis of a Doctype object.

1775

1776 This is a separate method from for_name_and_ids() because the lxml

1777 TreeBuilder needs to call it.

1778 """

1779 value = name or ""

1780 if pub_id is not None:

1781 value += ' PUBLIC "%s"' % pub_id

1782 if system_id is not None:

1783 value += ' "%s"' % system_id

1784 elif system_id is not None:

1785 value += ' SYSTEM "%s"' % system_id

1786 return value

1787

1788 PREFIX: str = "<!DOCTYPE "

1789 SUFFIX: str = ">\n"

1790

1791

1792class Stylesheet(NavigableString):

1793 """A `NavigableString` representing the contents of a `<style> HTML

1794 tag <https://dev.w3.org/html5/spec-LC/Overview.html#the-style-element>`_

1795 (probably CSS).

1796

1797 Used to distinguish embedded stylesheets from textual content.

1798 """

1799

1800

1801class Script(NavigableString):

1802 """A `NavigableString` representing the contents of a `<script>

1803 HTML tag

1804 <https://dev.w3.org/html5/spec-LC/Overview.html#the-script-element>`_

1805 (probably Javascript).

1806

1807 Used to distinguish executable code from textual content.

1808 """

1809

1810

1811class TemplateString(NavigableString):

1812 """A `NavigableString` representing a string found inside an `HTML

1813 <template> tag <https://html.spec.whatwg.org/multipage/scripting.html#the-template-element>`_

1814 embedded in a larger document.

1815

1816 Used to distinguish such strings from the main body of the document.

1817 """

1818

1819

1820class RubyTextString(NavigableString):

1821 """A NavigableString representing the contents of an `<rt> HTML

1822 tag <https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rt-element>`_.

1823

1824 Can be used to distinguish such strings from the strings they're

1825 annotating.

1826 """

1827

1828

1829class RubyParenthesisString(NavigableString):

1830 """A NavigableString representing the contents of an `<rp> HTML

1831 tag <https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rp-element>`_.

1832 """

1833

1834

1835class Tag(PageElement):

1836 """An HTML or XML tag that is part of a parse tree, along with its

1837 attributes, contents, and relationships to other parts of the tree.

1838

1839 When Beautiful Soup parses the markup ``<b>penguin</b>``, it will

1840 create a `Tag` object representing the ``<b>`` tag. You can

1841 instantiate `Tag` objects directly, but it's not necessary unless

1842 you're adding entirely new markup to a parsed document. Most of

1843 the constructor arguments are intended for use by the `TreeBuilder`

1844 that's parsing a document.

1845

1846 :param parser: A `BeautifulSoup` object representing the parse tree this

1847 `Tag` will be part of.

1848 :param builder: The `TreeBuilder` being used to build the tree.

1849 :param name: The name of the tag.

1850 :param namespace: The URI of this tag's XML namespace, if any.

1851 :param prefix: The prefix for this tag's XML namespace, if any.

1852 :param attrs: A dictionary of attribute values.

1853 :param parent: The `Tag` to use as the parent of this `Tag`. May be

1854 the `BeautifulSoup` object itself.

1855 :param previous: The `PageElement` that was parsed immediately before

1856 parsing this tag.

1857 :param is_xml: If True, this is an XML tag. Otherwise, this is an

1858 HTML tag.

1859 :param sourceline: The line number where this tag was found in its

1860 source document.

1861 :param sourcepos: The character position within ``sourceline`` where this

1862 tag was found.

1863 :param can_be_empty_element: If True, this tag should be

1864 represented as <tag/>. If False, this tag should be represented

1865 as <tag></tag>.

1866 :param cdata_list_attributes: A dictionary of attributes whose values should

1867 be parsed as lists of strings if they ever show up on this tag.

1868 :param preserve_whitespace_tags: Names of tags whose contents

1869 should have their whitespace preserved if they are encountered inside

1870 this tag.

1871 :param interesting_string_types: When iterating over this tag's

1872 string contents in methods like `Tag.strings` or

1873 `PageElement.get_text`, these are the types of strings that are

1874 interesting enough to be considered. By default,

1875 `NavigableString` (normal strings) and `CData` (CDATA

1876 sections) are the only interesting string subtypes.

1877 :param namespaces: A dictionary mapping currently active

1878 namespace prefixes to URIs, as of the point in the parsing process when

1879 this tag was encountered. This can be used later to

1880 construct CSS selectors.

1881

1882 """

1883

1884 def __init__(

1885 self,

1886 parser: Optional[BeautifulSoup] = None,

1887 builder: Optional[TreeBuilder] = None,

1888 name: Optional[str] = None,

1889 namespace: Optional[str] = None,

1890 prefix: Optional[str] = None,

1891 attrs: Optional[_RawOrProcessedAttributeValues] = None,

1892 parent: Optional[Union[BeautifulSoup, Tag]] = None,

1893 previous: _AtMostOneElement = None,

1894 is_xml: Optional[bool] = None,

1895 sourceline: Optional[int] = None,

1896 sourcepos: Optional[int] = None,

1897 can_be_empty_element: Optional[bool] = None,

1898 cdata_list_attributes: Optional[Dict[str, Set[str]]] = None,

1899 preserve_whitespace_tags: Optional[Set[str]] = None,

1900 interesting_string_types: Optional[Set[Type[NavigableString]]] = None,

1901 namespaces: Optional[Dict[str, str]] = None,

1902 # NOTE: Any new arguments here need to be mirrored in

1903 # Tag.copy_self, and potentially BeautifulSoup.new_tag

1904 # as well.

1905 ):

1906 if parser is None:

1907 self.parser_class = None

1908 else:

1909 # We don't actually store the parser object: that lets extracted

1910 # chunks be garbage-collected.

1911 self.parser_class = parser.__class__

1912 if name is None:

1913 raise ValueError("No value provided for new tag's name.")

1914 self.name = name

1915 self.namespace = namespace

1916 self._namespaces = namespaces or {}

1917 self.prefix = prefix

1918 if (not builder or builder.store_line_numbers) and (

1919 sourceline is not None or sourcepos is not None

1920 ):

1921 self.sourceline = sourceline

1922 self.sourcepos = sourcepos

1923 else:

1924 self.sourceline = sourceline

1925 self.sourcepos = sourcepos

1926

1927 attr_dict_class: type[AttributeDict]

1928 attribute_value_list_class: type[AttributeValueList]

1929 if builder is None:

1930 if is_xml:

1931 attr_dict_class = XMLAttributeDict

1932 else:

1933 attr_dict_class = HTMLAttributeDict

1934 attribute_value_list_class = AttributeValueList

1935 else:

1936 attr_dict_class = builder.attribute_dict_class

1937 attribute_value_list_class = builder.attribute_value_list_class

1938 self.attribute_value_list_class = attribute_value_list_class

1939

1940 if attrs is None:

1941 self.attrs = attr_dict_class()

1942 else:

1943 if builder is not None and builder.cdata_list_attributes:

1944 self.attrs = builder._replace_cdata_list_attribute_values(

1945 self.name, attrs

1946 )

1947 else:

1948 self.attrs = attr_dict_class()

1949 # Make sure that the values of any multi-valued

1950 # attributes (e.g. when a Tag is copied) are stored in

1951 # new lists.

1952 for k, v in attrs.items():

1953 if isinstance(v, list):

1954 v = v.__class__(v)

1955 self.attrs[k] = v

1956

1957 # If possible, determine ahead of time whether this tag is an

1958 # XML tag.

1959 if builder:

1960 self.known_xml = builder.is_xml

1961 else:

1962 self.known_xml = is_xml

1963 self.contents: List[PageElement] = []

1964 self.setup(parent, previous)

1965 self.hidden = False

1966

1967 if builder is None:

1968 # In the absence of a TreeBuilder, use whatever values were

1969 # passed in here. They're probably None, unless this is a copy of some

1970 # other tag.

1971 self.can_be_empty_element = can_be_empty_element

1972 self.cdata_list_attributes = cdata_list_attributes

1973 self.preserve_whitespace_tags = preserve_whitespace_tags

1974 self.interesting_string_types = interesting_string_types

1975 else:

1976 # Set up any substitutions for this tag, such as the charset in a META tag.

1977 self.attribute_value_list_class = builder.attribute_value_list_class

1978 builder.set_up_substitutions(self)

1979

1980 # Ask the TreeBuilder whether this tag might be an empty-element tag.

1981 self.can_be_empty_element = builder.can_be_empty_element(name)

1982

1983 # Keep track of the list of attributes of this tag that

1984 # might need to be treated as a list.

1985 #

1986 # For performance reasons, we store the whole data structure

1987 # rather than asking the question of every tag. Asking would

1988 # require building a new data structure every time, and

1989 # (unlike can_be_empty_element), we almost never need

1990 # to check this.

1991 self.cdata_list_attributes = builder.cdata_list_attributes

1992

1993 # Keep track of the names that might cause this tag to be treated as a

1994 # whitespace-preserved tag.

1995 self.preserve_whitespace_tags = builder.preserve_whitespace_tags

1996

1997 if self.name in builder.string_containers:

1998 # This sort of tag uses a special string container

1999 # subclass for most of its strings. We need to be able

2000 # to look up the proper container subclass.

2001 self.interesting_string_types = {builder.string_containers[self.name]}

2002 else:

2003 self.interesting_string_types = self.MAIN_CONTENT_STRING_TYPES

2004

2005 parser_class: Optional[type[BeautifulSoup]]

2006 name: str

2007 namespace: Optional[str]

2008 prefix: Optional[str]

2009 attrs: _AttributeValues

2010 sourceline: Optional[int]

2011 sourcepos: Optional[int]

2012 known_xml: Optional[bool]

2013 contents: List[PageElement]

2014 hidden: bool

2015 interesting_string_types: Optional[Set[Type[NavigableString]]]

2016

2017 can_be_empty_element: Optional[bool]

2018 cdata_list_attributes: Optional[Dict[str, Set[str]]]

2019 preserve_whitespace_tags: Optional[Set[str]]

2020

2021 #: :meta private:

2022 parserClass = _deprecated_alias("parserClass", "parser_class", "4.0.0")

2023

2024 def __deepcopy__(self, memo: Dict[Any, Any], recursive: bool = True) -> Self:

2025 """A deepcopy of a Tag is a new Tag, unconnected to the parse tree.

2026 Its contents are a copy of the old Tag's contents.

2027 """

2028 clone = self.copy_self()

2029

2030 if recursive:

2031 # Clone this tag's descendants recursively, but without

2032 # making any recursive function calls.

2033 tag_stack: List[Tag] = [clone]

2034 for event, element in self._event_stream(self.descendants):

2035 if event is Tag.END_ELEMENT_EVENT:

2036 # Stop appending incoming Tags to the Tag that was

2037 # just closed.

2038 tag_stack.pop()

2039 else:

2040 descendant_clone = element.__deepcopy__(memo, recursive=False)

2041 # Add to its parent's .contents

2042 tag_stack[-1].append(descendant_clone)

2043

2044 if event is Tag.START_ELEMENT_EVENT:

2045 # Add the Tag itself to the stack so that its

2046 # children will be .appended to it.

2047 tag_stack.append(cast(Tag, descendant_clone))

2048 return clone

2049

2050 def copy_self(self) -> Self:

2051 """Create a new Tag just like this one, but with no

2052 contents and unattached to any parse tree.

2053

2054 This is the first step in the deepcopy process, but you can

2055 call it on its own to create a copy of a Tag without copying its

2056 contents.

2057 """

2058 clone = type(self)(

2059 None,

2060 None,

2061 self.name,

2062 self.namespace,

2063 self.prefix,

2064 self.attrs,

2065 is_xml=self._is_xml,

2066 sourceline=self.sourceline,

2067 sourcepos=self.sourcepos,

2068 can_be_empty_element=self.can_be_empty_element,

2069 cdata_list_attributes=self.cdata_list_attributes,

2070 preserve_whitespace_tags=self.preserve_whitespace_tags,

2071 interesting_string_types=self.interesting_string_types,

2072 namespaces=self._namespaces,

2073 )

2074 for attr in ("can_be_empty_element", "hidden"):

2075 setattr(clone, attr, getattr(self, attr))

2076 return clone

2077

2078 @property

2079 def is_empty_element(self) -> bool:

2080 """Is this tag an empty-element tag? (aka a self-closing tag)

2081

2082 A tag that has contents is never an empty-element tag.

2083

2084 A tag that has no contents may or may not be an empty-element

2085 tag. It depends on the `TreeBuilder` used to create the

2086 tag. If the builder has a designated list of empty-element

2087 tags, then only a tag whose name shows up in that list is

2088 considered an empty-element tag. This is usually the case

2089 for HTML documents.

2090

2091 If the builder has no designated list of empty-element, then

2092 any tag with no contents is an empty-element tag. This is usually

2093 the case for XML documents.

2094 """

2095 return len(self.contents) == 0 and self.can_be_empty_element is True

2096

2097 @_deprecated("is_empty_element", "4.0.0")

2098 def isSelfClosing(self) -> bool:

2099 ": :meta private:"

2100 return self.is_empty_element

2101

2102 @property

2103 def string(self) -> Optional[str]:

2104 """Convenience property to get the single string within this

2105 `Tag`, assuming there is just one.

2106

2107 :return: If this `Tag` has a single child that's a

2108 `NavigableString`, the return value is that string. If this

2109 element has one child `Tag`, the return value is that child's

2110 `Tag.string`, recursively. If this `Tag` has no children,

2111 or has more than one child, the return value is ``None``.

2112

2113 If this property is unexpectedly returning ``None`` for you,

2114 it's probably because your `Tag` has more than one thing

2115 inside it.

2116 """

2117 if len(self.contents) != 1:

2118 return None

2119 child = self.contents[0]

2120 if isinstance(child, NavigableString):

2121 return child

2122 elif isinstance(child, Tag):

2123 return child.string

2124 return None

2125

2126 @string.setter

2127 def string(self, string: str) -> None:

2128 """Replace the `Tag.contents` of this `Tag` with a single string."""

2129 self.clear()

2130 if isinstance(string, NavigableString):

2131 new_class = string.__class__

2132 else:

2133 new_class = NavigableString

2134 self.append(new_class(string))

2135

2136 #: :meta private:

2137 MAIN_CONTENT_STRING_TYPES = {NavigableString, CData}

2138

2139 def _all_strings(

2140 self, strip: bool = False, types: _OneOrMoreStringTypes = PageElement.default

2141 ) -> Iterator[str]:

2142 """Yield all strings of certain classes, possibly stripping them.

2143

2144 :param strip: If True, all strings will be stripped before being

2145 yielded.

2146

2147 :param types: A tuple of NavigableString subclasses. Any strings of

2148 a subclass not found in this list will be ignored. By

2149 default, the subclasses considered are the ones found in

2150 self.interesting_string_types. If that's not specified,

2151 only NavigableString and CData objects will be

2152 considered. That means no comments, processing

2153 instructions, etc.

2154 """

2155 if types is self.default:

2156 if self.interesting_string_types is None:

2157 types = self.MAIN_CONTENT_STRING_TYPES

2158 else:

2159 types = self.interesting_string_types

2160

2161 for descendant in self.descendants:

2162 if not isinstance(descendant, NavigableString):

2163 continue

2164 descendant_type = type(descendant)

2165 if isinstance(types, type):

2166 if descendant_type is not types:

2167 # We're not interested in strings of this type.

2168 continue

2169 elif types is not None and descendant_type not in types:

2170 # We're not interested in strings of this type.

2171 continue

2172 if strip:

2173 stripped = descendant.strip()

2174 if len(stripped) == 0:

2175 continue

2176 yield stripped

2177 else:

2178 yield descendant

2179

2180 strings = property(_all_strings)

2181

2182 def insert(self, position: int, *new_children: _InsertableElement) -> List[PageElement]:

2183 """Insert one or more new PageElements as a child of this `Tag`.

2184

2185 This works similarly to :py:meth:`list.insert`, except you can insert

2186 multiple elements at once.

2187

2188 :param position: The numeric position that should be occupied

2189 in this Tag's `Tag.children` by the first new `PageElement`.

2190

2191 :param new_children: The PageElements to insert.

2192

2193 :return The newly inserted PageElements.

2194 """

2195 inserted: List[PageElement] = []

2196 for new_child in new_children:

2197 inserted.extend(self._insert(position, new_child))

2198 position += 1

2199 return inserted

2200

2201 def _insert(self, position: int, new_child: _InsertableElement) -> List[PageElement]:

2202 if new_child is None:

2203 raise ValueError("Cannot insert None into a tag.")

2204 if new_child is self:

2205 raise ValueError("Cannot insert a tag into itself.")

2206 if isinstance(new_child, str) and not isinstance(new_child, NavigableString):

2207 new_child = NavigableString(new_child)

2208

2209 from bs4 import BeautifulSoup

2210 if isinstance(new_child, BeautifulSoup):

2211 # We don't want to end up with a situation where one BeautifulSoup

2212 # object contains another. Insert the BeautifulSoup's children and

2213 # return them.

2214 return self.insert(position, *list(new_child.contents))

2215 position = min(position, len(self.contents))

2216 if hasattr(new_child, "parent") and new_child.parent is not None:

2217 # We're 'inserting' an element that's already one

2218 # of this object's children.

2219 if new_child.parent is self:

2220 current_index = self.index(new_child)

2221 if current_index < position:

2222 # We're moving this element further down the list

2223 # of this object's children. That means that when

2224 # we extract this element, our target index will

2225 # jump down one.

2226 position -= 1

2227 elif current_index == position:

2228 # We're 'inserting' an element into its current location.

2229 # This is a no-op.

2230 return [new_child]

2231 new_child.extract()

2232

2233 new_child.parent = self

2234 previous_child = None

2235 if position == 0:

2236 new_child.previous_sibling = None

2237 new_child.previous_element = self

2238 else:

2239 previous_child = self.contents[position - 1]

2240 new_child.previous_sibling = previous_child

2241 new_child.previous_sibling.next_sibling = new_child

2242 new_child.previous_element = previous_child._last_descendant(False)

2243 if new_child.previous_element is not None:

2244 new_child.previous_element.next_element = new_child

2245

2246 new_childs_last_element = new_child._last_descendant(

2247 is_initialized=False, accept_self=True

2248 )

2249 # new_childs_last_element can't be None because we passed

2250 # accept_self=True into _last_descendant. Worst case,

2251 # new_childs_last_element will be new_child itself. Making

2252 # this cast removes several mypy complaints later on as we

2253 # manipulate new_childs_last_element.

2254 new_childs_last_element = cast(PageElement, new_childs_last_element)

2255

2256 if position >= len(self.contents):

2257 new_child.next_sibling = None

2258

2259 parent: Optional[Tag] = self

2260 parents_next_sibling = None

2261 while parents_next_sibling is None and parent is not None:

2262 parents_next_sibling = parent.next_sibling

2263 parent = parent.parent

2264 if parents_next_sibling is not None:

2265 # We found the element that comes next in the document.

2266 break

2267 if parents_next_sibling is not None:

2268 new_childs_last_element.next_element = parents_next_sibling

2269 else:

2270 # The last element of this tag is the last element in

2271 # the document.

2272 new_childs_last_element.next_element = None

2273 else:

2274 next_child = self.contents[position]

2275 new_child.next_sibling = next_child

2276 if new_child.next_sibling is not None:

2277 new_child.next_sibling.previous_sibling = new_child

2278 new_childs_last_element.next_element = next_child

2279

2280 if new_childs_last_element.next_element is not None:

2281 new_childs_last_element.next_element.previous_element = (

2282 new_childs_last_element

2283 )

2284 self.contents.insert(position, new_child)

2285

2286 return [new_child]

2287

2288 def unwrap(self) -> Self:

2289 """Replace this `PageElement` with its contents.

2290

2291 :return: This object, no longer part of the tree.

2292 """

2293 my_parent = self.parent

2294 if my_parent is None:

2295 raise ValueError(

2296 "Cannot replace an element with its contents when that "

2297 "element is not part of a tree."

2298 )

2299 my_index = my_parent.index(self)

2300 self.extract(_self_index=my_index)

2301 for child in reversed(self.contents[:]):

2302 my_parent.insert(my_index, child)

2303 return self

2304

2305 replace_with_children = unwrap

2306

2307 @_deprecated("unwrap", "4.0.0")

2308 def replaceWithChildren(self) -> _OneElement:

2309 ": :meta private:"

2310 return self.unwrap()

2311

2312 def append(self, tag: _InsertableElement) -> PageElement|List[PageElement]:

2313 """Appends the given `PageElement` to the contents of this `Tag`.

2314

2315 :param tag: A PageElement. If this is another BeautifulSoup

2316 object, all of its contents will be inserted into this

2317 `Tag`, since one BeautifulSoup object can't contain another

2318 one.

2319

2320 :return: The object that was just appended, or (if `tag` was a BeautifulSoup

2321 object) all such objects.

2322 """

2323 inserted = self.insert(len(self.contents), tag)

2324 if isinstance(tag, Tag) and tag.name == "[document]": # TODO: can't reference BeautifulSoup class in this module

2325 return inserted

2326 else:

2327 return inserted[0]

2328

2329 def extend(self, tags: Union[Iterable[_InsertableElement], Tag]) -> List[PageElement]:

2330 """Appends one or more objects to the contents of this

2331 `Tag`.

2332

2333 :param tags: If a list of `PageElement` objects is provided,

2334 they will be appended to this tag's contents, one at a time.

2335 If a single `Tag` is provided, its `Tag.contents` will be

2336 used to extend this object's `Tag.contents`.

2337

2338 :return The list of PageElements that were appended.

2339 """

2340 tag_list: Iterable[_InsertableElement]

2341

2342 if isinstance(tags, Tag):

2343 tag_list = list(tags.contents)

2344 elif isinstance(tags, (PageElement, str)):

2345 # The caller should really be using append() instead,

2346 # but we can make it work.

2347 warnings.warn(

2348 "A single non-Tag item was passed into Tag.extend. Use Tag.append instead.",

2349 UserWarning,

2350 stacklevel=2,

2351 )

2352 if isinstance(tags, str) and not isinstance(tags, PageElement):

2353 tags = NavigableString(tags)

2354 tag_list = [tags]

2355 elif isinstance(tags, Iterable):

2356 # Moving items around the tree may change their position in

2357 # the original list. Make a list that won't change.

2358 tag_list = list(tags)

2359

2360 results: List[PageElement] = []

2361 for tag in tag_list:

2362 appended = self.append(tag)

2363 if isinstance(appended, list):

2364 # This can happen if you pass in a mixture of Tag and BeautifulSoup objects.

2365 results.extend(appended)

2366 else:

2367 results.append(appended)

2368

2369 return results

2370

2371 def clear(self, decompose: bool = False) -> None:

2372 """Destroy all children of this `Tag` by calling

2373 `PageElement.extract` on them.

2374

2375 :param decompose: If this is True, `PageElement.decompose` (a

2376 more destructive method) will be called instead of

2377 `PageElement.extract`.

2378 """

2379 for element in self.contents[:]:

2380 if decompose:

2381 element.decompose()

2382 else:

2383 element.extract()

2384

2385 def smooth(self) -> None:

2386 """Smooth out the children of this `Tag` by consolidating consecutive

2387 strings.

2388

2389 If you perform a lot of operations that modify the tree,

2390 calling this method afterwards can make pretty-printed output

2391 look more natural.

2392 """

2393 # Mark the first position of every pair of children that need

2394 # to be consolidated. Do this rather than making a copy of

2395 # self.contents, since in most cases very few strings will be

2396 # affected.

2397 marked = []

2398 for i, a in enumerate(self.contents):

2399 if isinstance(a, Tag):

2400 # Recursively smooth children.

2401 a.smooth()

2402 if i == len(self.contents) - 1:

2403 # This is the last item in .contents, and it's not a

2404 # tag. There's no chance it needs any work.

2405 continue

2406 b = self.contents[i + 1]

2407 if (

2408 isinstance(a, NavigableString)

2409 and isinstance(b, NavigableString)

2410 and not isinstance(a, PreformattedString)

2411 and not isinstance(b, PreformattedString)

2412 ):

2413 marked.append(i)

2414

2415 # Go over the marked positions in reverse order, so that

2416 # removing items from .contents won't affect the remaining

2417 # positions.

2418 for i in reversed(marked):

2419 a = cast(NavigableString, self.contents[i])

2420 b = cast(NavigableString, self.contents[i + 1])

2421 b.extract()

2422 n = NavigableString(a + b)

2423 a.replace_with(n)

2424

2425 def index(self, element: PageElement) -> int:

2426 """Find the index of a child of this `Tag` (by identity, not value).

2427

2428 Doing this by identity avoids issues when a `Tag` contains two

2429 children that have string equality.

2430

2431 :param element: Look for this `PageElement` in this object's contents.

2432 """

2433 for i, child in enumerate(self.contents):

2434 if child is element:

2435 return i

2436 raise ValueError("Tag.index: element not in tag")

2437

2438 @overload

2439 def get(self, key: str, default: _AttributeValue) -> _AttributeValue:

2440 ...

2441 @overload

2442 def get(self, key: str, default:None=None) -> Optional[_AttributeValue]:

2443 ...

2444 def get(

2445 self, key: str, default: Optional[_AttributeValue] = None

2446 ) -> Optional[_AttributeValue]:

2447 """Returns the value of the 'key' attribute for the tag, or

2448 the value given for 'default' if it doesn't have that

2449 attribute.

2450

2451 :param key: The attribute to look for.

2452 :param default: Use this value if the attribute is not present

2453 on this `Tag`.

2454 """

2455 return self.attrs.get(key, default)

2456

2457 def get_attribute_list(

2458 self, key: str, default: Optional[AttributeValueList] = None

2459 ) -> AttributeValueList:

2460 """The same as get(), but always returns a (possibly empty) list.

2461

2462 :param key: The attribute to look for.

2463 :param default: Use this value if the attribute is not present

2464 on this `Tag`.

2465 :return: A list of strings, usually empty or containing only a single

2466 value.

2467 """

2468 list_value: AttributeValueList

2469 value = self.get(key, default)

2470 if value is None:

2471 list_value = self.attribute_value_list_class()

2472 elif isinstance(value, list):

2473 list_value = value

2474 else:

2475 if not isinstance(value, str):

2476 value = cast(str, value)

2477 list_value = self.attribute_value_list_class([value])

2478 return list_value

2479

2480 def has_attr(self, key: str) -> bool:

2481 """Does this `Tag` have an attribute with the given name?"""

2482 return key in self.attrs

2483

2484 def __hash__(self) -> int:

2485 return str(self).__hash__()

2486

2487 def __getitem__(self, key: str) -> _AttributeValue:

2488 """tag[key] returns the value of the 'key' attribute for the Tag,

2489 and throws an exception if it's not there."""

2490 return self.attrs[key]

2491

2492 def __iter__(self) -> Iterator[PageElement]:

2493 "Iterating over a Tag iterates over its contents."

2494 return iter(self.contents)

2495

2496 def __len__(self) -> int:

2497 "The length of a Tag is the length of its list of contents."

2498 return len(self.contents)

2499

2500 def __contains__(self, x: Any) -> bool:

2501 return x in self.contents

2502

2503 def __bool__(self) -> bool:

2504 "A tag is non-None even if it has no contents."

2505 return True

2506

2507 def __setitem__(self, key: str, value: _AttributeValue) -> None:

2508 """Setting tag[key] sets the value of the 'key' attribute for the

2509 tag."""

2510 self.attrs[key] = value

2511

2512 def __delitem__(self, key: str) -> None:

2513 "Deleting tag[key] deletes all 'key' attributes for the tag."

2514 self.attrs.pop(key, None)

2515

2516 # Since Tag.__call__ is effectively the same as PageElement.find_all, see find_all for notes

2517 # on these overloads.

2518

2519 @overload

2520 def __call__(

2521 self,

2522 name: None = None,

2523 attrs: None = None,

2524 recursive: bool = True,

2525 *,

2526 string: _StrainableString,

2527 limit: Optional[int] = None,

2528 **kwargs: _StrainableAttribute,

2529 ) -> _SomeNavigableStrings:

2530 ...

2531

2532 @overload

2533 def __call__(

2534 self,

2535 name: None = None,

2536 attrs: None = None,

2537 recursive: bool = True,

2538 string: None = None,

2539 limit: Optional[int] = None,

2540 **kwargs: _StrainableAttribute,

2541 ) -> _SomeTags:

2542 ...

2543

2544 @overload

2545 def __call__(

2546 self,

2547 name: None,

2548 attrs: _StrainableAttributes,

2549 recursive: bool = True,

2550 string: None = None,

2551 limit: Optional[int] = None,

2552 **kwargs: _StrainableAttribute,

2553 ) -> _SomeTags:

2554 ...

2555

2556 @overload

2557 def __call__(

2558 self,

2559 name: _FindMethodName,

2560 attrs: Optional[_StrainableAttributes] = None,

2561 recursive: bool = True,

2562 string: Optional[_StrainableString] = None,

2563 limit: Optional[int] = None,

2564 **kwargs: _StrainableAttribute,

2565 ) -> _SomeTags:

2566 ...

2567

2568 def __call__(

2569 self,

2570 name: _OptionalFindMethodName = None,

2571 attrs: Optional[_StrainableAttributes] = None,

2572 recursive: bool = True,

2573 string: Optional[_StrainableString] = None,

2574 limit: Optional[int] = None,

2575 **kwargs: _StrainableAttribute,

2576 ) -> Union[_SomeTags,_SomeNavigableStrings,_QueryResults]:

2577 """Calling a Tag like a function is the same as calling its

2578 find_all() method.

2579

2580 Eg. tag('a') returns a list of all the A tags found within this tag.

2581 """

2582 return self._find_all(name, attrs, string, limit, self._generator_for_recursive(recursive), **kwargs)

2583

2584 def __getattr__(self, subtag: str) -> Optional[Tag]:

2585 """Calling tag.subtag is the same as calling tag.find(name="subtag")"""

2586 # print("Getattr %s.%s" % (self.__class__, tag))

2587 result: _AtMostOneElement

2588 if len(subtag) > 3 and subtag.endswith("Tag"):

2589 # BS3: soup.aTag -> "soup.find("a")

2590 tag_name = subtag[:-3]

2591 warnings.warn(

2592 '.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")'

2593 % dict(name=tag_name),

2594 DeprecationWarning,

2595 stacklevel=2,

2596 )

2597 result = self.find(tag_name)

2598 # We special case contents to avoid recursion.

2599 elif not subtag.startswith("__") and not subtag == "contents":

2600 result = self.find(subtag)

2601 else:

2602 raise AttributeError(

2603 "'%s' object has no attribute '%s'" % (self.__class__, subtag)

2604 )

2605 return result

2606

2607 def __eq__(self, other: Any) -> bool:

2608 """Returns true iff this Tag has the same name, the same attributes,

2609 and the same contents (recursively) as `other`."""

2610 if self is other:

2611 return True

2612 if not isinstance(other, Tag):

2613 return False

2614 if (

2615 not hasattr(other, "name")

2616 or not hasattr(other, "attrs")

2617 or not hasattr(other, "contents")

2618 or self.name != other.name

2619 or self.attrs != other.attrs

2620 or len(self) != len(other)

2621 ):

2622 return False

2623 for i, my_child in enumerate(self.contents):

2624 if my_child != other.contents[i]:

2625 return False

2626 return True

2627

2628 def __ne__(self, other: Any) -> bool:

2629 """Returns true iff this Tag is not identical to `other`,

2630 as defined in __eq__."""

2631 return not self == other

2632

2633 def __repr__(self) -> str:

2634 """Renders this `Tag` as a string."""

2635 return self.decode()

2636

2637 __str__ = __unicode__ = __repr__

2638

2639 def encode(

2640 self,

2641 encoding: _Encoding = DEFAULT_OUTPUT_ENCODING,

2642 indent_level: Optional[int] = None,

2643 formatter: _FormatterOrName = "minimal",

2644 errors: str = "xmlcharrefreplace",

2645 ) -> bytes:

2646 """Render this `Tag` and its contents as a bytestring.

2647

2648 :param encoding: The encoding to use when converting to

2649 a bytestring. This may also affect the text of the document,

2650 specifically any encoding declarations within the document.

2651 :param indent_level: Each line of the rendering will be

2652 indented this many levels. (The ``formatter`` decides what a

2653 'level' means, in terms of spaces or other characters

2654 output.) This is used internally in recursive calls while

2655 pretty-printing.

2656 :param formatter: Either a `Formatter` object, or a string naming one of

2657 the standard formatters.

2658 :param errors: An error handling strategy such as

2659 'xmlcharrefreplace'. This value is passed along into

2660 :py:meth:`str.encode` and its value should be one of the `error

2661 handling constants defined by Python's codecs module

2662 <https://docs.python.org/3/library/codecs.html#error-handlers>`_.

2663 """

2664 # Turn the data structure into Unicode, then encode the

2665 # Unicode.

2666 u = self.decode(indent_level, encoding, formatter)

2667 return u.encode(encoding, errors)

2668

2669 def decode(

2670 self,

2671 indent_level: Optional[int] = None,

2672 eventual_encoding: _Encoding = DEFAULT_OUTPUT_ENCODING,

2673 formatter: _FormatterOrName = "minimal",

2674 iterator: Optional[Iterator[PageElement]] = None,

2675 ) -> str:

2676 """Render this `Tag` and its contents as a Unicode string.

2677

2678 :param indent_level: Each line of the rendering will be

2679 indented this many levels. (The ``formatter`` decides what a

2680 'level' means, in terms of spaces or other characters

2681 output.) This is used internally in recursive calls while

2682 pretty-printing.

2683 :param encoding: The encoding you intend to use when

2684 converting the string to a bytestring. decode() is *not*

2685 responsible for performing that encoding. This information

2686 is needed so that a real encoding can be substituted in if

2687 the document contains an encoding declaration (e.g. in a

2688 <meta> tag).

2689 :param formatter: Either a `Formatter` object, or a string

2690 naming one of the standard formatters.

2691 :param iterator: The iterator to use when navigating over the

2692 parse tree. This is only used by `Tag.decode_contents` and

2693 you probably won't need to use it.

2694 """

2695 pieces = []

2696 # First off, turn a non-Formatter `formatter` into a Formatter

2697 # object. This will stop the lookup from happening over and

2698 # over again.

2699 if not isinstance(formatter, Formatter):

2700 formatter = self.formatter_for_name(formatter)

2701

2702 if indent_level is True:

2703 indent_level = 0

2704

2705 # The currently active tag that put us into string literal

2706 # mode. Until this element is closed, children will be treated

2707 # as string literals and not pretty-printed. String literal

2708 # mode is turned on immediately after this tag begins, and

2709 # turned off immediately before it's closed. This means there

2710 # will be whitespace before and after the tag itself.

2711 string_literal_tag = None

2712

2713 for event, element in self._event_stream(iterator):

2714 if event in (Tag.START_ELEMENT_EVENT, Tag.EMPTY_ELEMENT_EVENT):

2715 element = cast(Tag, element)

2716 piece = element._format_tag(eventual_encoding, formatter, opening=True)

2717 elif event is Tag.END_ELEMENT_EVENT:

2718 element = cast(Tag, element)

2719 piece = element._format_tag(eventual_encoding, formatter, opening=False)

2720 if indent_level is not None:

2721 indent_level -= 1

2722 else:

2723 element = cast(NavigableString, element)

2724 piece = element.output_ready(formatter)

2725

2726 # Now we need to apply the 'prettiness' -- extra

2727 # whitespace before and/or after this tag. This can get

2728 # complicated because certain tags, like <pre> and

2729 # <script>, can't be prettified, since adding whitespace would

2730 # change the meaning of the content.

2731

2732 # The default behavior is to add whitespace before and

2733 # after an element when string literal mode is off, and to

2734 # leave things as they are when string literal mode is on.

2735 if string_literal_tag:

2736 indent_before = indent_after = False

2737 else:

2738 indent_before = indent_after = True

2739

2740 # The only time the behavior is more complex than that is

2741 # when we encounter an opening or closing tag that might

2742 # put us into or out of string literal mode.

2743 if (

2744 event is Tag.START_ELEMENT_EVENT

2745 and not string_literal_tag

2746 and not cast(Tag, element)._should_pretty_print()

2747 ):

2748 # We are about to enter string literal mode. Add

2749 # whitespace before this tag, but not after. We

2750 # will stay in string literal mode until this tag

2751 # is closed.

2752 indent_before = True

2753 indent_after = False

2754 string_literal_tag = element

2755 elif event is Tag.END_ELEMENT_EVENT and element is string_literal_tag:

2756 # We are about to exit string literal mode by closing

2757 # the tag that sent us into that mode. Add whitespace

2758 # after this tag, but not before.

2759 indent_before = False

2760 indent_after = True

2761 string_literal_tag = None

2762

2763 # Now we know whether to add whitespace before and/or

2764 # after this element.

2765 if indent_level is not None:

2766 if indent_before or indent_after:

2767 if isinstance(element, NavigableString):

2768 piece = piece.strip()

2769 if piece:

2770 piece = self._indent_string(

2771 piece, indent_level, formatter, indent_before, indent_after

2772 )

2773 if event == Tag.START_ELEMENT_EVENT:

2774 indent_level += 1

2775 pieces.append(piece)

2776 return "".join(pieces)

2777

2778 class _TreeTraversalEvent(object):

2779 """An internal class representing an event in the process

2780 of traversing a parse tree.

2781

2782 :meta private:

2783 """

2784

2785 # Stand-ins for the different events yielded by _event_stream

2786 START_ELEMENT_EVENT = _TreeTraversalEvent() #: :meta private:

2787 END_ELEMENT_EVENT = _TreeTraversalEvent() #: :meta private:

2788 EMPTY_ELEMENT_EVENT = _TreeTraversalEvent() #: :meta private:

2789 STRING_ELEMENT_EVENT = _TreeTraversalEvent() #: :meta private:

2790

2791 def _event_stream(

2792 self, iterator: Optional[Iterator[PageElement]] = None

2793 ) -> Iterator[Tuple[_TreeTraversalEvent, PageElement]]:

2794 """Yield a sequence of events that can be used to reconstruct the DOM

2795 for this element.

2796

2797 This lets us recreate the nested structure of this element

2798 (e.g. when formatting it as a string) without using recursive

2799 method calls.

2800

2801 This is similar in concept to the SAX API, but it's a simpler

2802 interface designed for internal use. The events are different

2803 from SAX and the arguments associated with the events are Tags

2804 and other Beautiful Soup objects.

2805

2806 :param iterator: An alternate iterator to use when traversing

2807 the tree.

2808 """

2809 tag_stack: List[Tag] = []

2810

2811 iterator = iterator or self.self_and_descendants

2812

2813 for c in iterator:

2814 # If the parent of the element we're about to yield is not

2815 # the tag currently on the stack, it means that the tag on

2816 # the stack closed before this element appeared.

2817 while tag_stack and c.parent is not tag_stack[-1]:

2818 now_closed_tag = tag_stack.pop()

2819 yield Tag.END_ELEMENT_EVENT, now_closed_tag

2820

2821 if isinstance(c, Tag):

2822 if c.is_empty_element:

2823 yield Tag.EMPTY_ELEMENT_EVENT, c

2824 else:

2825 yield Tag.START_ELEMENT_EVENT, c

2826 tag_stack.append(c)

2827 continue

2828 else:

2829 yield Tag.STRING_ELEMENT_EVENT, c

2830

2831 while tag_stack:

2832 now_closed_tag = tag_stack.pop()

2833 yield Tag.END_ELEMENT_EVENT, now_closed_tag

2834

2835 def _indent_string(

2836 self,

2837 s: str,

2838 indent_level: int,

2839 formatter: Formatter,

2840 indent_before: bool,

2841 indent_after: bool,

2842 ) -> str:

2843 """Add indentation whitespace before and/or after a string.

2844

2845 :param s: The string to amend with whitespace.

2846 :param indent_level: The indentation level; affects how much

2847 whitespace goes before the string.

2848 :param indent_before: Whether or not to add whitespace

2849 before the string.

2850 :param indent_after: Whether or not to add whitespace

2851 (a newline) after the string.

2852 """

2853 space_before = ""

2854 if indent_before and indent_level:

2855 space_before = formatter.indent * indent_level

2856

2857 space_after = ""

2858 if indent_after:

2859 space_after = "\n"

2860

2861 return space_before + s + space_after

2862

2863 def _format_tag(

2864 self, eventual_encoding: str, formatter: Formatter, opening: bool

2865 ) -> str:

2866 if self.hidden:

2867 # A hidden tag is invisible, although its contents

2868 # are visible.

2869 return ""

2870

2871 # A tag starts with the < character (see below).

2872

2873 # Then the / character, if this is a closing tag.

2874 closing_slash = ""

2875 if not opening:

2876 closing_slash = "/"

2877

2878 # Then an optional namespace prefix.

2879 prefix = ""

2880 if self.prefix:

2881 prefix = self.prefix + ":"

2882

2883 # Then a list of attribute values, if this is an opening tag.

2884 attribute_string = ""

2885 if opening:

2886 attributes = formatter.attributes(self)

2887 attrs = []

2888 for key, val in attributes:

2889 if val is None:

2890 decoded = key

2891 else:

2892 if isinstance(val, list) or isinstance(val, tuple):

2893 val = " ".join(val)

2894 elif not isinstance(val, str):

2895 val = str(val)

2896 elif (

2897 isinstance(val, AttributeValueWithCharsetSubstitution)

2898 and eventual_encoding is not None

2899 ):

2900 val = val.substitute_encoding(eventual_encoding)

2901

2902 text = formatter.attribute_value(val)

2903 decoded = str(key) + "=" + formatter.quoted_attribute_value(text)

2904 attrs.append(decoded)

2905 if attrs:

2906 attribute_string = " " + " ".join(attrs)

2907

2908 # Then an optional closing slash (for a void element in an

2909 # XML document).

2910 void_element_closing_slash = ""

2911 if self.is_empty_element:

2912 void_element_closing_slash = formatter.void_element_close_prefix or ""

2913

2914 # Put it all together.

2915 return (

2916 "<"

2917 + closing_slash

2918 + prefix

2919 + self.name

2920 + attribute_string

2921 + void_element_closing_slash

2922 + ">"

2923 )

2924

2925 def _should_pretty_print(self, indent_level: int = 1) -> bool:

2926 """Should this tag be pretty-printed?

2927

2928 Most of them should, but some (such as <pre> in HTML

2929 documents) should not.

2930 """

2931 return indent_level is not None and (

2932 not self.preserve_whitespace_tags

2933 or self.name not in self.preserve_whitespace_tags

2934 )

2935

2936 @overload

2937 def prettify(

2938 self,

2939 encoding: None = None,

2940 formatter: _FormatterOrName = "minimal",

2941 ) -> str:

2942 ...

2943

2944 @overload

2945 def prettify(

2946 self,

2947 encoding: _Encoding,

2948 formatter: _FormatterOrName = "minimal",

2949 ) -> bytes:

2950 ...

2951

2952 def prettify(

2953 self,

2954 encoding: Optional[_Encoding] = None,

2955 formatter: _FormatterOrName = "minimal",

2956 ) -> Union[str, bytes]:

2957 """Pretty-print this `Tag` as a string or bytestring.

2958

2959 :param encoding: The encoding of the bytestring, or None if you want Unicode.

2960 :param formatter: A Formatter object, or a string naming one of

2961 the standard formatters.

2962 :return: A string (if no ``encoding`` is provided) or a bytestring

2963 (otherwise).

2964 """

2965 if encoding is None:

2966 return self.decode(indent_level=0, formatter=formatter)

2967 else:

2968 return self.encode(encoding=encoding, indent_level=0, formatter=formatter)

2969

2970 def decode_contents(

2971 self,

2972 indent_level: Optional[int] = None,

2973 eventual_encoding: _Encoding = DEFAULT_OUTPUT_ENCODING,

2974 formatter: _FormatterOrName = "minimal",

2975 ) -> str:

2976 """Renders the contents of this tag as a Unicode string.

2977

2978 :param indent_level: Each line of the rendering will be

2979 indented this many levels. (The formatter decides what a

2980 'level' means in terms of spaces or other characters

2981 output.) Used internally in recursive calls while

2982 pretty-printing.

2983

2984 :param eventual_encoding: The tag is destined to be

2985 encoded into this encoding. decode_contents() is *not*

2986 responsible for performing that encoding. This information

2987 is needed so that a real encoding can be substituted in if

2988 the document contains an encoding declaration (e.g. in a

2989 <meta> tag).

2990

2991 :param formatter: A `Formatter` object, or a string naming one of

2992 the standard Formatters.

2993 """

2994 return self.decode(

2995 indent_level, eventual_encoding, formatter, iterator=self.descendants

2996 )

2997

2998 def encode_contents(

2999 self,

3000 indent_level: Optional[int] = None,

3001 encoding: _Encoding = DEFAULT_OUTPUT_ENCODING,

3002 formatter: _FormatterOrName = "minimal",

3003 ) -> bytes:

3004 """Renders the contents of this PageElement as a bytestring.

3005

3006 :param indent_level: Each line of the rendering will be

3007 indented this many levels. (The ``formatter`` decides what a

3008 'level' means, in terms of spaces or other characters

3009 output.) This is used internally in recursive calls while

3010 pretty-printing.

3011 :param formatter: Either a `Formatter` object, or a string naming one of

3012 the standard formatters.

3013 :param encoding: The bytestring will be in this encoding.

3014 """

3015 contents = self.decode_contents(indent_level, encoding, formatter)

3016 return contents.encode(encoding)

3017

3018 @_deprecated("encode_contents", "4.0.0")

3019 def renderContents(

3020 self,

3021 encoding: _Encoding = DEFAULT_OUTPUT_ENCODING,

3022 prettyPrint: bool = False,

3023 indentLevel: Optional[int] = 0,

3024 ) -> bytes:

3025 """Deprecated method for BS3 compatibility.

3026

3027 :meta private:

3028 """

3029 if not prettyPrint:

3030 indentLevel = None

3031 return self.encode_contents(indent_level=indentLevel, encoding=encoding)

3032

3033 # Soup methods

3034 #

3035

3036 # People who call these methods in a type-safe environment

3037 # basically want to know whether the call is going to return

3038 # NavigableStrings or Tags. It's always one or the other, never

3039 # both, but spelling it out requires a number of overloads for

3040 # each method.

3041 #

3042 # If I had it to do over again I'd design this API differently (it

3043 # would look more like ElementFilter), but that's life.

3044 #

3045 # The overloads all look for a clue in the input which restricts

3046 # the method to returning either only strings or only tags. Only

3047 # the most common cases are covered.

3048

3049 # e.g. find(string="foo")

3050 # -> string information but no tag information

3051 # -> string

3052 @overload

3053 def find(

3054 self,

3055 name: None = None,

3056 attrs: None = None,

3057 recursive: bool = True,

3058 *,

3059 string: _StrainableString,

3060 **kwargs: _StrainableAttribute,

3061 ) -> _AtMostOneNavigableString:

3062 ...

3063

3064 # e.g. find() -> default behavior -> tag

3065 # find(attr="value") -> only tags have attrs -> tag

3066 @overload

3067 def find(

3068 self,

3069 name: None = None,

3070 attrs: None = None,

3071 recursive: bool = True,

3072 string: None = None,

3073 **kwargs: _StrainableAttribute,

3074 ) -> _AtMostOneTag:

3075 ...

3076

3077 # e.g. find(attrs=dict(attr="value"))

3078 # -> only tags have attrs

3079 # -> tag

3080 @overload

3081 def find(

3082 self,

3083 name: None,

3084 attrs: _StrainableAttributes,

3085 recursive: bool = True,

3086 string: Optional[_StrainableString] = None,

3087 **kwargs: _StrainableAttribute,

3088 ) -> _AtMostOneTag:

3089 ...

3090

3091 # e.g. find(name="a")) -> only tags have names -> tag

3092 #

3093 # The confusing and controversial case of find(name="a", string="foo")

3094 # also hits this overload.

3095 @overload

3096 def find(

3097 self,

3098 name: _FindMethodName,

3099 attrs: Optional[_StrainableAttributes] = None,

3100 recursive: bool = True,

3101 string: Optional[_StrainableString] = None,

3102 **kwargs: _StrainableAttribute,

3103 ) -> _AtMostOneTag:

3104 ...

3105

3106 # Some lesser-used cases are not covered by the overrides. Those

3107 # cases will hit this method directly and return a very general

3108 # type which will need to be cast after the call.

3109 def find(

3110 self,

3111 name: _OptionalFindMethodName = None,

3112 attrs: Optional[_StrainableAttributes] = None,

3113 recursive: bool = True,

3114 string: Optional[_StrainableString] = None,

3115 **kwargs: _StrainableAttribute,

3116 ) -> Union[_AtMostOneTag,_AtMostOneNavigableString,_AtMostOneElement]:

3117 """Look in the children of this PageElement and find the first

3118 PageElement that matches the given criteria.

3119

3120 All find_* methods take a common set of arguments. See the online

3121 documentation for detailed explanations.

3122

3123 :param name: A filter on tag name.

3124 :param attrs: Additional filters on attribute values.

3125 :param recursive: If this is True, find() will perform a

3126 recursive search of this Tag's children. Otherwise,

3127 only the direct children will be considered.

3128 :param string: A filter on the `Tag.string` attribute.

3129 :kwargs: Additional filters on attribute values.

3130 """

3131 tags = self._find_all(name, attrs, string, 1, self._generator_for_recursive(recursive), **kwargs)

3132 if tags:

3133 return tags[0]

3134 return None

3135

3136 findChild = _deprecated_function_alias("findChild", "find", "3.0.0")

3137

3138 # e.g. find_all(string="foo")

3139 # -> string information but no tag information

3140 # -> strings

3141 #

3142 # Also covers unlikely cases like find_all(name=None, string="foo")

3143 #

3144 # "To mark parameters as keyword-only, indicating the parameters

3145 # must be passed by keyword argument, place an * in the arguments

3146 # list just before the first keyword-only parameter."

3147 #

3148 # --https://peps.python.org/pep-0570/#keyword-only-arguments

3149 @overload

3150 def find_all(

3151 self,

3152 name: None = None,

3153 attrs: None = None,

3154 recursive: bool = True,

3155 *,

3156 string: _StrainableString,

3157 limit: Optional[int] = None,

3158 **kwargs: _StrainableAttribute,

3159 ) -> _SomeNavigableStrings:

3160 ...

3161

3162 # e.g. find_all() -> default behavior -> tags

3163 # find_all(attr="value") -> only tags have attrs -> tags

3164 @overload

3165 def find_all(

3166 self,

3167 name: None = None,

3168 attrs: None = None,

3169 recursive: bool = True,

3170 string: None = None,

3171 limit: Optional[int] = None,

3172 **kwargs: _StrainableAttribute,

3173 ) -> _SomeTags:

3174 ...

3175

3176 # e.g. find_all(attrs=dict(attr="value"))

3177 # -> only tags have attrs

3178 # -> tags

3179 @overload

3180 def find_all(

3181 self,

3182 name: None,

3183 attrs: _StrainableAttributes,

3184 recursive: bool = True,

3185 string: Optional[_StrainableString] = None,

3186 limit: Optional[int] = None,

3187 **kwargs: _StrainableAttribute,

3188 ) -> _SomeTags:

3189 ...

3190

3191 # e.g. find_all(name="a")) -> only tags have names -> tags

3192 #

3193 # The confusing and controversial case of find_all(name="a", string="foo")

3194 # also hits this overload.

3195 @overload

3196 def find_all(

3197 self,

3198 name: _FindMethodName,

3199 attrs: Optional[_StrainableAttributes] = None,

3200 recursive: bool = True,

3201 string: Optional[_StrainableString] = None,

3202 limit: Optional[int] = None,

3203 **kwargs: _StrainableAttribute,

3204 ) -> _SomeTags:

3205 ...

3206

3207 # Without the clues above, we don't know whether the method will

3208 # return strings or tags. However every common case will trigger one

3209 # of the overloads and give us the clue we need.

3210 def find_all(

3211 self,

3212 name: _OptionalFindMethodName = None,

3213 attrs: Optional[_StrainableAttributes] = None,

3214 recursive: bool = True,

3215 string: Optional[_StrainableString] = None,

3216 limit: Optional[int] = None,

3217 **kwargs: _StrainableAttribute,

3218 ) -> Union[_SomeTags,_SomeNavigableStrings]:

3219 """Look in the children of this `PageElement` and find all

3220 `PageElement` objects that match the given criteria.

3221

3222 All find_* methods take a common set of arguments. See the online

3223 documentation for detailed explanations.

3224

3225 :param name: A filter on tag name.

3226 :param attrs: Additional filters on attribute values.

3227 :param recursive: If this is True, find_all() will perform a

3228 recursive search of this PageElement's children. Otherwise,

3229 only the direct children will be considered.

3230 :param limit: Stop looking after finding this many results.

3231 :kwargs: Additional filters on attribute values.

3232 """

3233 generator = self._generator_for_recursive(recursive)

3234

3235 if string is not None and (name is not None or attrs is not None or kwargs):

3236 # TODO: Using the @overload decorator to express the three ways you

3237 # could get into this path is way too much code for a rarely(?) used

3238 # feature.

3239 return cast(ResultSet[Tag],

3240 self._find_all(name, attrs, string, limit, generator,

3241 **kwargs))

3242

3243 if string is None:

3244 # If string is None, we're searching for tags.

3245 return cast(ResultSet[Tag], self._find_all(

3246 name, attrs, None, limit, generator, **kwargs

3247 ))

3248

3249 # Otherwise, we're searching for strings.

3250 return cast(ResultSet[NavigableString], self._find_all(

3251 None, None, string, limit, generator, **kwargs

3252 ))

3253

3254 findAll = _deprecated_function_alias("findAll", "find_all", "4.0.0")

3255 findChildren = _deprecated_function_alias("findChildren", "find_all", "3.0.0")

3256

3257 # Generator methods

3258 @property

3259 def children(self) -> Iterator[PageElement]:

3260 """Iterate over all direct children of this `PageElement`."""

3261 return (x for x in self.contents)

3262

3263 @property

3264 def self_and_descendants(self) -> Iterator[PageElement]:

3265 """Iterate over this `Tag` and its children in a

3266 breadth-first sequence.

3267 """

3268 return self._self_and(self.descendants)

3269

3270 @property

3271 def descendants(self) -> Iterator[PageElement]:

3272 """Iterate over all children of this `Tag` in a

3273 breadth-first sequence.

3274 """

3275 if not len(self.contents):

3276 return

3277 # _last_descendant() can't return None here because

3278 # accept_self is True. Worst case, last_descendant will end up

3279 # as self.

3280 last_descendant = cast(PageElement, self._last_descendant(accept_self=True))

3281 stopNode = last_descendant.next_element

3282 current: _AtMostOneElement = self.contents[0]

3283 while current is not stopNode and current is not None:

3284 successor = current.next_element

3285 yield current

3286 current = successor

3287

3288 def _generator_for_recursive(self, recursive:bool) -> Iterator[PageElement]:

3289 """Helper method to process the boolean `recursive` argument

3290 for find* methods.

3291

3292 :return: the appropriate generator

3293 """

3294 if recursive:

3295 return self.descendants

3296 return self.children

3297

3298 # CSS selector code

3299 def select_one(

3300 self, selector: str, namespaces: Optional[Dict[str, str]] = None, **kwargs: Any

3301 ) -> Optional[Tag]:

3302 """Perform a CSS selection operation on the current element.

3303

3304 :param selector: A CSS selector.

3305

3306 :param namespaces: A dictionary mapping namespace prefixes

3307 used in the CSS selector to namespace URIs. By default,

3308 Beautiful Soup will use the prefixes it encountered while

3309 parsing the document.

3310

3311 :param kwargs: Keyword arguments to be passed into Soup Sieve's

3312 soupsieve.select() method.

3313 """

3314 return self.css.select_one(selector, namespaces, **kwargs)

3315

3316 def select(

3317 self,

3318 selector: str,

3319 namespaces: Optional[Dict[str, str]] = None,

3320 limit: int = 0,

3321 **kwargs: Any,

3322 ) -> ResultSet[Tag]:

3323 """Perform a CSS selection operation on the current element.

3324

3325 This uses the SoupSieve library.

3326

3327 :param selector: A string containing a CSS selector.

3328

3329 :param namespaces: A dictionary mapping namespace prefixes

3330 used in the CSS selector to namespace URIs. By default,

3331 Beautiful Soup will use the prefixes it encountered while

3332 parsing the document.

3333

3334 :param limit: After finding this number of results, stop looking.

3335

3336 :param kwargs: Keyword arguments to be passed into SoupSieve's

3337 soupsieve.select() method.

3338 """

3339 return self.css.select(selector, namespaces, limit, **kwargs)

3340

3341 @property

3342 def css(self) -> CSS:

3343 """Return an interface to the CSS selector API."""

3344 return CSS(self)

3345

3346 # Old names for backwards compatibility

3347 @_deprecated("children", "4.0.0")

3348 def childGenerator(self) -> Iterator[PageElement]:

3349 """Deprecated generator.

3350

3351 :meta private:

3352 """

3353 return self.children

3354

3355 @_deprecated("descendants", "4.0.0")

3356 def recursiveChildGenerator(self) -> Iterator[PageElement]:

3357 """Deprecated generator.

3358

3359 :meta private:

3360 """

3361 return self.descendants

3362

3363 @_deprecated("has_attr", "4.0.0")

3364 def has_key(self, key: str) -> bool:

3365 """Deprecated method. This was kind of misleading because has_key()

3366 (attributes) was different from __in__ (contents).

3367

3368 has_key() is gone in Python 3, anyway.

3369

3370 :meta private:

3371 """

3372 return self.has_attr(key)

3373

3374

3375_PageElementT = TypeVar("_PageElementT", bound=PageElement)

3376

3377class ResultSet(List[_PageElementT], Generic[_PageElementT]):

3378 """A ResultSet is a list of `PageElement` objects, gathered as the result

3379 of matching an :py:class:`ElementFilter` against a parse tree. Basically, a list of

3380 search results.

3381 """

3382

3383 source: Optional[ElementFilter]

3384

3385 def __init__(

3386 self, source: Optional[ElementFilter], result: Iterable[_PageElementT] = ()

3387 ) -> None:

3388 super().__init__(result)

3389 self.source = source

3390

3391 def __getattr__(self, key: str) -> None:

3392 """Raise a helpful exception to explain a common code fix."""

3393 raise AttributeError(

3394 f"""ResultSet object has no attribute "{key}". You're probably treating a list of elements like a single element. Did you call find_all() when you meant to call find()?"""

3395 )

3396

3397# Now that all the classes used by SoupStrainer have been defined,

3398# import SoupStrainer itself into this module to preserve the

3399# backwards compatibility of anyone who imports

3400# bs4.element.SoupStrainer.

3401from bs4.filter import SoupStrainer # noqa: E402