Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tornado/escape.py: 32%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Copyright 2009 Facebook
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
16"""Escaping/unescaping methods for HTML, JSON, URLs, and others.
18Also includes a few other miscellaneous string manipulation functions that
19have crept in over time.
21Many functions in this module have near-equivalents in the standard library
22(the differences mainly relate to handling of bytes and unicode strings,
23and were more relevant in Python 2). In new code, the standard library
24functions are encouraged instead of this module where applicable. See the
25docstrings on each function for details.
26"""
28import binascii
29import html
30import json
31import re
32import typing
33import urllib.parse
34from collections.abc import Callable
35from typing import Any
37from tornado.util import unicode_type
40def xhtml_escape(value: str | bytes) -> str:
41 """Escapes a string so it is valid within HTML or XML.
43 Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``.
44 When used in attribute values the escaped strings must be enclosed
45 in quotes.
47 Equivalent to `html.escape` except that this function always returns
48 type `str` while `html.escape` returns `bytes` if its input is `bytes`.
50 .. versionchanged:: 3.2
52 Added the single quote to the list of escaped characters.
54 .. versionchanged:: 6.4
56 Now simply wraps `html.escape`. This is equivalent to the old behavior
57 except that single quotes are now escaped as ``'`` instead of
58 ``'`` and performance may be different.
59 """
60 return html.escape(to_unicode(value))
63def xhtml_unescape(value: str | bytes) -> str:
64 """Un-escapes an XML-escaped string.
66 Equivalent to `html.unescape` except that this function always returns
67 type `str` while `html.unescape` returns `bytes` if its input is `bytes`.
69 .. versionchanged:: 6.4
71 Now simply wraps `html.unescape`. This changes behavior for some inputs
72 as required by the HTML 5 specification
73 https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
75 Some invalid inputs such as surrogates now raise an error, and numeric
76 references to certain ISO-8859-1 characters are now handled correctly.
77 """
78 return html.unescape(to_unicode(value))
81# The fact that json_encode wraps json.dumps is an implementation detail.
82# Please see https://github.com/tornadoweb/tornado/pull/706
83# before sending a pull request that adds **kwargs to this function.
84def json_encode(value: Any) -> str:
85 """JSON-encodes the given Python object.
87 Equivalent to `json.dumps` with the additional guarantee that the output
88 will never contain the character sequence ``</`` which can be problematic
89 when JSON is embedded in an HTML ``<script>`` tag.
90 """
91 # JSON permits but does not require forward slashes to be escaped.
92 # This is useful when json data is emitted in a <script> tag
93 # in HTML, as it prevents </script> tags from prematurely terminating
94 # the JavaScript. Some json libraries do this escaping by default,
95 # although python's standard library does not, so we do it here.
96 # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped
97 return json.dumps(value).replace("</", "<\\/")
100def json_decode(value: str | bytes) -> Any:
101 """Returns Python objects for the given JSON string.
103 Supports both `str` and `bytes` inputs. Equivalent to `json.loads`.
104 """
105 return json.loads(value)
108def squeeze(value: str) -> str:
109 """Replace all sequences of whitespace chars with a single space."""
110 return re.sub(r"[\x00-\x20]+", " ", value).strip()
113def url_escape(value: str | bytes, plus: bool = True) -> str:
114 """Returns a URL-encoded version of the given value.
116 Equivalent to either `urllib.parse.quote_plus` or `urllib.parse.quote` depending on the ``plus``
117 argument.
119 If ``plus`` is true (the default), spaces will be represented as ``+`` and slashes will be
120 represented as ``%2F``. This is appropriate for query strings. If ``plus`` is false, spaces
121 will be represented as ``%20`` and slashes are left as-is. This is appropriate for the path
122 component of a URL. Note that the default of ``plus=True`` is effectively the
123 reverse of Python's urllib module.
125 .. versionadded:: 3.1
126 The ``plus`` argument
127 """
128 quote = urllib.parse.quote_plus if plus else urllib.parse.quote
129 return quote(value)
132@typing.overload
133def url_unescape(value: str | bytes, encoding: None, plus: bool = True) -> bytes:
134 pass
137@typing.overload
138def url_unescape(value: str | bytes, encoding: str = "utf-8", plus: bool = True) -> str:
139 pass
142def url_unescape(
143 value: str | bytes, encoding: str | None = "utf-8", plus: bool = True
144) -> str | bytes:
145 """Decodes the given value from a URL.
147 The argument may be either a byte or unicode string.
149 If encoding is None, the result will be a byte string and this function is equivalent to
150 `urllib.parse.unquote_to_bytes` if ``plus=False``. Otherwise, the result is a unicode string in
151 the specified encoding and this function is equivalent to either `urllib.parse.unquote_plus` or
152 `urllib.parse.unquote` except that this function also accepts `bytes` as input.
154 If ``plus`` is true (the default), plus signs will be interpreted as spaces (literal plus signs
155 must be represented as "%2B"). This is appropriate for query strings and form-encoded values
156 but not for the path component of a URL. Note that this default is the reverse of Python's
157 urllib module.
159 .. versionadded:: 3.1
160 The ``plus`` argument
161 """
162 if encoding is None:
163 if plus:
164 # unquote_to_bytes doesn't have a _plus variant
165 value = to_basestring(value).replace("+", " ")
166 return urllib.parse.unquote_to_bytes(value)
167 else:
168 unquote = urllib.parse.unquote_plus if plus else urllib.parse.unquote
169 return unquote(to_basestring(value), encoding=encoding)
172# Swaps % and = so that binascii.a2b_qp, which decodes =XX in C, can decode
173# percent escapes.
174_PERCENT_EQUALS_SWAP = bytes.maketrans(b"%=", b"=%")
177def _unquote_to_bytes(data: bytes) -> bytes:
178 """Equivalent to `urllib.parse.unquote_to_bytes`, but fast.
180 The standard library's version (as of Python 3.14) creates an object for
181 every ``%`` in its input, which makes large escaped bodies cost many
182 times their size in memory and microseconds per byte. This does the
183 work in a few passes over the data in C.
184 """
185 if b"%" not in data:
186 return data
187 # After the swap, escapes are =XX and literal = signs are %, which
188 # a2b_qp passes through. a2b_qp also passes through = that is not
189 # followed by two hex digits (an invalid escape), and the final swap
190 # turns those back into %.
191 data = data.translate(_PERCENT_EQUALS_SWAP)
192 # Escapes that decode to % or = would be swapped too, so exchange them
193 # first. Each token is three bytes with = only at the start, so these
194 # replacements can't create or break other tokens; =3d is free to use
195 # as a placeholder once it has been normalized away.
196 data = (
197 data.replace(b"=3d", b"=3D")
198 .replace(b"=25", b"=3d")
199 .replace(b"=3D", b"=25")
200 .replace(b"=3d", b"=3D")
201 )
202 # a2b_qp decodes == as =, consuming both. The first = of a pair is
203 # always an invalid escape, so make it explicit. Two passes are needed
204 # because replace() doesn't see the pair formed where two matches meet.
205 data = data.replace(b"==", b"=3D=").replace(b"==", b"=3D=")
206 # a2b_qp treats = before a line break or at the end as a soft line break
207 # and drops it (after =\r, everything up to the next \n). Those are
208 # invalid escapes too.
209 data = data.replace(b"=\n", b"=3D\n").replace(b"=\r", b"=3D\r")
210 if data.endswith(b"="):
211 data += b"3D"
212 return binascii.a2b_qp(data).translate(_PERCENT_EQUALS_SWAP)
215def parse_qs_bytes(
216 qs: str | bytes,
217 keep_blank_values: bool = False,
218 strict_parsing: bool = False,
219 *,
220 max_num_fields: int | None = None,
221) -> dict[str, list[bytes]]:
222 """Parses a query string like urlparse.parse_qs,
223 but takes bytes and returns the values as byte strings.
225 Keys still become type str (interpreted as latin1 in python3!)
226 because it's too painful to keep them as byte strings in
227 python3 and in practice they're nearly always ascii anyway.
229 .. versionadded:: 6.5.8
230 The ``max_num_fields`` argument. ValueError is raised if this limit is exceeded.
231 """
232 # This follows urllib.parse.parse_qsl, with a faster unquote.
233 if isinstance(qs, str):
234 qs = qs.encode("latin1")
235 result: dict[str, list[bytes]] = {}
236 if not qs:
237 return result
238 if max_num_fields is not None and 1 + qs.count(b"&") > max_num_fields:
239 raise ValueError("Max number of fields exceeded")
240 for field in qs.split(b"&"):
241 if not field and not strict_parsing:
242 continue
243 name, has_eq, value = field.partition(b"=")
244 if not has_eq and strict_parsing:
245 raise ValueError("bad query field: %r" % (field,))
246 if value or keep_blank_values:
247 key = _unquote_to_bytes(name.replace(b"+", b" ")).decode("latin1")
248 result.setdefault(key, []).append(
249 _unquote_to_bytes(value.replace(b"+", b" "))
250 )
251 return result
254_UTF8_TYPES = (bytes, type(None))
257@typing.overload
258def utf8(value: bytes) -> bytes:
259 pass
262@typing.overload
263def utf8(value: str) -> bytes:
264 pass
267@typing.overload
268def utf8(value: None) -> None:
269 pass
272def utf8(value: None | str | bytes) -> bytes | None:
273 """Converts a string argument to a byte string.
275 If the argument is already a byte string or None, it is returned unchanged.
276 Otherwise it must be a unicode string and is encoded as utf8.
277 """
278 if isinstance(value, _UTF8_TYPES):
279 return value
280 if not isinstance(value, unicode_type):
281 raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
282 return value.encode("utf-8")
285_TO_UNICODE_TYPES = (unicode_type, type(None))
288@typing.overload
289def to_unicode(value: str) -> str:
290 pass
293@typing.overload
294def to_unicode(value: bytes) -> str:
295 pass
298@typing.overload
299def to_unicode(value: None) -> None:
300 pass
303def to_unicode(value: None | str | bytes) -> str | None:
304 """Converts a string argument to a unicode string.
306 If the argument is already a unicode string or None, it is returned
307 unchanged. Otherwise it must be a byte string and is decoded as utf8.
308 """
309 if isinstance(value, _TO_UNICODE_TYPES):
310 return value
311 if not isinstance(value, bytes):
312 raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
313 return value.decode("utf-8")
316# to_unicode was previously named _unicode not because it was private,
317# but to avoid conflicts with the built-in unicode() function/type
318_unicode = to_unicode
320# When dealing with the standard library across python 2 and 3 it is
321# sometimes useful to have a direct conversion to the native string type
322native_str = to_unicode
323to_basestring = to_unicode
326def recursive_unicode(obj: Any) -> Any:
327 """Walks a simple data structure, converting byte strings to unicode.
329 Supports lists, tuples, and dictionaries.
330 """
331 if isinstance(obj, dict):
332 return {recursive_unicode(k): recursive_unicode(v) for (k, v) in obj.items()}
333 elif isinstance(obj, list):
334 return list(recursive_unicode(i) for i in obj)
335 elif isinstance(obj, tuple):
336 return tuple(recursive_unicode(i) for i in obj)
337 elif isinstance(obj, bytes):
338 return to_unicode(obj)
339 else:
340 return obj
343# I originally used the regex from
344# http://daringfireball.net/2010/07/improved_regex_for_matching_urls
345# but it gets all exponential on certain patterns (such as too many trailing
346# dots), causing the regex matcher to never return.
347# This regex should avoid those problems.
348# Use to_unicode instead of tornado.util.u - we don't want backslashes getting
349# processed as escapes.
350_URL_RE = re.compile(
351 to_unicode(
352 r"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""" # noqa: E501
353 )
354)
357def linkify(
358 text: str | bytes,
359 shorten: bool = False,
360 extra_params: str | Callable[[str], str] = "",
361 require_protocol: bool = False,
362 permitted_protocols: list[str] = ["http", "https"],
363) -> str:
364 """Converts plain text into HTML with links.
366 For example: ``linkify("Hello http://tornadoweb.org!")`` would return
367 ``Hello <a href="http://tornadoweb.org">http://tornadoweb.org</a>!``
369 Parameters:
371 * ``shorten``: Long urls will be shortened for display.
373 * ``extra_params``: Extra text to include in the link tag, or a callable
374 taking the link as an argument and returning the extra text
375 e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``,
376 or::
378 def extra_params_cb(url):
379 if url.startswith("http://example.com"):
380 return 'class="internal"'
381 else:
382 return 'class="external" rel="nofollow"'
383 linkify(text, extra_params=extra_params_cb)
385 * ``require_protocol``: Only linkify urls which include a protocol. If
386 this is False, urls such as www.facebook.com will also be linkified.
388 * ``permitted_protocols``: List (or set) of protocols which should be
389 linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp",
390 "mailto"])``. It is very unsafe to include protocols such as
391 ``javascript``.
392 """
393 if extra_params and not callable(extra_params):
394 extra_params = " " + extra_params.strip()
396 def make_link(m: typing.Match) -> str:
397 url = m.group(1)
398 proto = m.group(2)
399 if require_protocol and not proto:
400 return url # not protocol, no linkify
402 if proto and proto not in permitted_protocols:
403 return url # bad protocol, no linkify
405 href = m.group(1)
406 if not proto:
407 href = "http://" + href # no proto specified, use http
409 if callable(extra_params):
410 params = " " + extra_params(href).strip()
411 else:
412 params = extra_params
414 # clip long urls. max_len is just an approximation
415 max_len = 30
416 if shorten and len(url) > max_len:
417 before_clip = url
418 if proto:
419 proto_len = len(proto) + 1 + len(m.group(3) or "") # +1 for :
420 else:
421 proto_len = 0
423 parts = url[proto_len:].split("/")
424 if len(parts) > 1:
425 # Grab the whole host part plus the first bit of the path
426 # The path is usually not that interesting once shortened
427 # (no more slug, etc), so it really just provides a little
428 # extra indication of shortening.
429 url = (
430 url[:proto_len]
431 + parts[0]
432 + "/"
433 + parts[1][:8].split("?")[0].split(".")[0]
434 )
436 if len(url) > max_len * 1.5: # still too long
437 url = url[:max_len]
439 if url != before_clip:
440 amp = url.rfind("&")
441 # avoid splitting html char entities
442 if amp > max_len - 5:
443 url = url[:amp]
444 url += "..."
446 if len(url) >= len(before_clip):
447 url = before_clip
448 else:
449 # full url is visible on mouse-over (for those who don't
450 # have a status bar, such as Safari by default)
451 params += ' title="%s"' % href
453 return f'<a href="{href}"{params}>{url}</a>'
455 # First HTML-escape so that our strings are all safe.
456 # The regex is modified to avoid character entities other than & so
457 # that we won't pick up ", etc.
458 text = _unicode(xhtml_escape(text))
459 return _URL_RE.sub(make_link, text)