Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tornado/escape.py: 35%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Copyright 2009 Facebook
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
16"""Escaping/unescaping methods for HTML, JSON, URLs, and others.
18Also includes a few other miscellaneous string manipulation functions that
19have crept in over time.
21Many functions in this module have near-equivalents in the standard library
22(the differences mainly relate to handling of bytes and unicode strings,
23and were more relevant in Python 2). In new code, the standard library
24functions are encouraged instead of this module where applicable. See the
25docstrings on each function for details.
26"""
28import html
29import json
30import re
31import typing
32import urllib.parse
33from collections.abc import Callable
34from typing import Any
36from tornado.util import unicode_type
39def xhtml_escape(value: str | bytes) -> str:
40 """Escapes a string so it is valid within HTML or XML.
42 Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``.
43 When used in attribute values the escaped strings must be enclosed
44 in quotes.
46 Equivalent to `html.escape` except that this function always returns
47 type `str` while `html.escape` returns `bytes` if its input is `bytes`.
49 .. versionchanged:: 3.2
51 Added the single quote to the list of escaped characters.
53 .. versionchanged:: 6.4
55 Now simply wraps `html.escape`. This is equivalent to the old behavior
56 except that single quotes are now escaped as ``'`` instead of
57 ``'`` and performance may be different.
58 """
59 return html.escape(to_unicode(value))
62def xhtml_unescape(value: str | bytes) -> str:
63 """Un-escapes an XML-escaped string.
65 Equivalent to `html.unescape` except that this function always returns
66 type `str` while `html.unescape` returns `bytes` if its input is `bytes`.
68 .. versionchanged:: 6.4
70 Now simply wraps `html.unescape`. This changes behavior for some inputs
71 as required by the HTML 5 specification
72 https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
74 Some invalid inputs such as surrogates now raise an error, and numeric
75 references to certain ISO-8859-1 characters are now handled correctly.
76 """
77 return html.unescape(to_unicode(value))
80# The fact that json_encode wraps json.dumps is an implementation detail.
81# Please see https://github.com/tornadoweb/tornado/pull/706
82# before sending a pull request that adds **kwargs to this function.
83def json_encode(value: Any) -> str:
84 """JSON-encodes the given Python object.
86 Equivalent to `json.dumps` with the additional guarantee that the output
87 will never contain the character sequence ``</`` which can be problematic
88 when JSON is embedded in an HTML ``<script>`` tag.
89 """
90 # JSON permits but does not require forward slashes to be escaped.
91 # This is useful when json data is emitted in a <script> tag
92 # in HTML, as it prevents </script> tags from prematurely terminating
93 # the JavaScript. Some json libraries do this escaping by default,
94 # although python's standard library does not, so we do it here.
95 # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped
96 return json.dumps(value).replace("</", "<\\/")
99def json_decode(value: str | bytes) -> Any:
100 """Returns Python objects for the given JSON string.
102 Supports both `str` and `bytes` inputs. Equivalent to `json.loads`.
103 """
104 return json.loads(value)
107def squeeze(value: str) -> str:
108 """Replace all sequences of whitespace chars with a single space."""
109 return re.sub(r"[\x00-\x20]+", " ", value).strip()
112def url_escape(value: str | bytes, plus: bool = True) -> str:
113 """Returns a URL-encoded version of the given value.
115 Equivalent to either `urllib.parse.quote_plus` or `urllib.parse.quote` depending on the ``plus``
116 argument.
118 If ``plus`` is true (the default), spaces will be represented as ``+`` and slashes will be
119 represented as ``%2F``. This is appropriate for query strings. If ``plus`` is false, spaces
120 will be represented as ``%20`` and slashes are left as-is. This is appropriate for the path
121 component of a URL. Note that the default of ``plus=True`` is effectively the
122 reverse of Python's urllib module.
124 .. versionadded:: 3.1
125 The ``plus`` argument
126 """
127 quote = urllib.parse.quote_plus if plus else urllib.parse.quote
128 return quote(value)
131@typing.overload
132def url_unescape(value: str | bytes, encoding: None, plus: bool = True) -> bytes:
133 pass
136@typing.overload
137def url_unescape(value: str | bytes, encoding: str = "utf-8", plus: bool = True) -> str:
138 pass
141def url_unescape(
142 value: str | bytes, encoding: str | None = "utf-8", plus: bool = True
143) -> str | bytes:
144 """Decodes the given value from a URL.
146 The argument may be either a byte or unicode string.
148 If encoding is None, the result will be a byte string and this function is equivalent to
149 `urllib.parse.unquote_to_bytes` if ``plus=False``. Otherwise, the result is a unicode string in
150 the specified encoding and this function is equivalent to either `urllib.parse.unquote_plus` or
151 `urllib.parse.unquote` except that this function also accepts `bytes` as input.
153 If ``plus`` is true (the default), plus signs will be interpreted as spaces (literal plus signs
154 must be represented as "%2B"). This is appropriate for query strings and form-encoded values
155 but not for the path component of a URL. Note that this default is the reverse of Python's
156 urllib module.
158 .. versionadded:: 3.1
159 The ``plus`` argument
160 """
161 if encoding is None:
162 if plus:
163 # unquote_to_bytes doesn't have a _plus variant
164 value = to_basestring(value).replace("+", " ")
165 return urllib.parse.unquote_to_bytes(value)
166 else:
167 unquote = urllib.parse.unquote_plus if plus else urllib.parse.unquote
168 return unquote(to_basestring(value), encoding=encoding)
171def parse_qs_bytes(
172 qs: str | bytes,
173 keep_blank_values: bool = False,
174 strict_parsing: bool = False,
175 *,
176 max_num_fields: int | None = None,
177) -> dict[str, list[bytes]]:
178 """Parses a query string like urlparse.parse_qs,
179 but takes bytes and returns the values as byte strings.
181 Keys still become type str (interpreted as latin1 in python3!)
182 because it's too painful to keep them as byte strings in
183 python3 and in practice they're nearly always ascii anyway.
185 .. versionadded:: 6.5.8
186 The ``max_num_fields`` argument. ValueError is raised if this limit is exceeded.
187 """
188 # This is gross, but python3 doesn't give us another way.
189 # Latin1 is the universal donor of character encodings.
190 if isinstance(qs, bytes):
191 qs = qs.decode("latin1")
192 result = urllib.parse.parse_qs(
193 qs,
194 keep_blank_values,
195 strict_parsing,
196 encoding="latin1",
197 errors="strict",
198 max_num_fields=max_num_fields,
199 )
200 encoded = {}
201 for k, v in result.items():
202 encoded[k] = [i.encode("latin1") for i in v]
203 return encoded
206_UTF8_TYPES = (bytes, type(None))
209@typing.overload
210def utf8(value: bytes) -> bytes:
211 pass
214@typing.overload
215def utf8(value: str) -> bytes:
216 pass
219@typing.overload
220def utf8(value: None) -> None:
221 pass
224def utf8(value: None | str | bytes) -> bytes | None:
225 """Converts a string argument to a byte string.
227 If the argument is already a byte string or None, it is returned unchanged.
228 Otherwise it must be a unicode string and is encoded as utf8.
229 """
230 if isinstance(value, _UTF8_TYPES):
231 return value
232 if not isinstance(value, unicode_type):
233 raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
234 return value.encode("utf-8")
237_TO_UNICODE_TYPES = (unicode_type, type(None))
240@typing.overload
241def to_unicode(value: str) -> str:
242 pass
245@typing.overload
246def to_unicode(value: bytes) -> str:
247 pass
250@typing.overload
251def to_unicode(value: None) -> None:
252 pass
255def to_unicode(value: None | str | bytes) -> str | None:
256 """Converts a string argument to a unicode string.
258 If the argument is already a unicode string or None, it is returned
259 unchanged. Otherwise it must be a byte string and is decoded as utf8.
260 """
261 if isinstance(value, _TO_UNICODE_TYPES):
262 return value
263 if not isinstance(value, bytes):
264 raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
265 return value.decode("utf-8")
268# to_unicode was previously named _unicode not because it was private,
269# but to avoid conflicts with the built-in unicode() function/type
270_unicode = to_unicode
272# When dealing with the standard library across python 2 and 3 it is
273# sometimes useful to have a direct conversion to the native string type
274native_str = to_unicode
275to_basestring = to_unicode
278def recursive_unicode(obj: Any) -> Any:
279 """Walks a simple data structure, converting byte strings to unicode.
281 Supports lists, tuples, and dictionaries.
282 """
283 if isinstance(obj, dict):
284 return {recursive_unicode(k): recursive_unicode(v) for (k, v) in obj.items()}
285 elif isinstance(obj, list):
286 return list(recursive_unicode(i) for i in obj)
287 elif isinstance(obj, tuple):
288 return tuple(recursive_unicode(i) for i in obj)
289 elif isinstance(obj, bytes):
290 return to_unicode(obj)
291 else:
292 return obj
295# I originally used the regex from
296# http://daringfireball.net/2010/07/improved_regex_for_matching_urls
297# but it gets all exponential on certain patterns (such as too many trailing
298# dots), causing the regex matcher to never return.
299# This regex should avoid those problems.
300# Use to_unicode instead of tornado.util.u - we don't want backslashes getting
301# processed as escapes.
302_URL_RE = re.compile(
303 to_unicode(
304 r"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""" # noqa: E501
305 )
306)
309def linkify(
310 text: str | bytes,
311 shorten: bool = False,
312 extra_params: str | Callable[[str], str] = "",
313 require_protocol: bool = False,
314 permitted_protocols: list[str] = ["http", "https"],
315) -> str:
316 """Converts plain text into HTML with links.
318 For example: ``linkify("Hello http://tornadoweb.org!")`` would return
319 ``Hello <a href="http://tornadoweb.org">http://tornadoweb.org</a>!``
321 Parameters:
323 * ``shorten``: Long urls will be shortened for display.
325 * ``extra_params``: Extra text to include in the link tag, or a callable
326 taking the link as an argument and returning the extra text
327 e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``,
328 or::
330 def extra_params_cb(url):
331 if url.startswith("http://example.com"):
332 return 'class="internal"'
333 else:
334 return 'class="external" rel="nofollow"'
335 linkify(text, extra_params=extra_params_cb)
337 * ``require_protocol``: Only linkify urls which include a protocol. If
338 this is False, urls such as www.facebook.com will also be linkified.
340 * ``permitted_protocols``: List (or set) of protocols which should be
341 linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp",
342 "mailto"])``. It is very unsafe to include protocols such as
343 ``javascript``.
344 """
345 if extra_params and not callable(extra_params):
346 extra_params = " " + extra_params.strip()
348 def make_link(m: typing.Match) -> str:
349 url = m.group(1)
350 proto = m.group(2)
351 if require_protocol and not proto:
352 return url # not protocol, no linkify
354 if proto and proto not in permitted_protocols:
355 return url # bad protocol, no linkify
357 href = m.group(1)
358 if not proto:
359 href = "http://" + href # no proto specified, use http
361 if callable(extra_params):
362 params = " " + extra_params(href).strip()
363 else:
364 params = extra_params
366 # clip long urls. max_len is just an approximation
367 max_len = 30
368 if shorten and len(url) > max_len:
369 before_clip = url
370 if proto:
371 proto_len = len(proto) + 1 + len(m.group(3) or "") # +1 for :
372 else:
373 proto_len = 0
375 parts = url[proto_len:].split("/")
376 if len(parts) > 1:
377 # Grab the whole host part plus the first bit of the path
378 # The path is usually not that interesting once shortened
379 # (no more slug, etc), so it really just provides a little
380 # extra indication of shortening.
381 url = (
382 url[:proto_len]
383 + parts[0]
384 + "/"
385 + parts[1][:8].split("?")[0].split(".")[0]
386 )
388 if len(url) > max_len * 1.5: # still too long
389 url = url[:max_len]
391 if url != before_clip:
392 amp = url.rfind("&")
393 # avoid splitting html char entities
394 if amp > max_len - 5:
395 url = url[:amp]
396 url += "..."
398 if len(url) >= len(before_clip):
399 url = before_clip
400 else:
401 # full url is visible on mouse-over (for those who don't
402 # have a status bar, such as Safari by default)
403 params += ' title="%s"' % href
405 return f'<a href="{href}"{params}>{url}</a>'
407 # First HTML-escape so that our strings are all safe.
408 # The regex is modified to avoid character entities other than & so
409 # that we won't pick up ", etc.
410 text = _unicode(xhtml_escape(text))
411 return _URL_RE.sub(make_link, text)