Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import codecs
2import datetime
3import locale
4from decimal import Decimal
5from types import NoneType
6from urllib.parse import quote
8from django.utils.functional import Promise
11class DjangoUnicodeDecodeError(UnicodeDecodeError):
12 def __str__(self):
13 return "%s. You passed in %r (%s)" % (
14 super().__str__(),
15 self.object,
16 type(self.object),
17 )
20def smart_str(s, encoding="utf-8", strings_only=False, errors="strict"):
21 """
22 Return a string representing 's'. Treat bytestrings using the 'encoding'
23 codec.
25 If strings_only is True, don't convert (some) non-string-like objects.
26 """
27 if isinstance(s, Promise):
28 # The input is the result of a gettext_lazy() call.
29 return s
30 return force_str(s, encoding, strings_only, errors)
33_PROTECTED_TYPES = (
34 NoneType,
35 int,
36 float,
37 Decimal,
38 datetime.datetime,
39 datetime.date,
40 datetime.time,
41)
44def is_protected_type(obj):
45 """Determine if the object instance is of a protected type.
47 Objects of protected types are preserved as-is when passed to
48 force_str(strings_only=True).
49 """
50 return isinstance(obj, _PROTECTED_TYPES)
53def force_str(s, encoding="utf-8", strings_only=False, errors="strict"):
54 """
55 Similar to smart_str(), except that lazy instances are resolved to
56 strings, rather than kept as lazy objects.
58 If strings_only is True, don't convert (some) non-string-like objects.
59 """
60 # Handle the common case first for performance reasons.
61 if issubclass(type(s), str):
62 return s
63 if strings_only and is_protected_type(s):
64 return s
65 try:
66 if isinstance(s, bytes):
67 s = str(s, encoding, errors)
68 else:
69 s = str(s)
70 except UnicodeDecodeError as e:
71 raise DjangoUnicodeDecodeError(*e.args) from None
72 return s
75def smart_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
76 """
77 Return a bytestring version of 's', encoded as specified in 'encoding'.
79 If strings_only is True, don't convert (some) non-string-like objects.
80 """
81 if isinstance(s, Promise):
82 # The input is the result of a gettext_lazy() call.
83 return s
84 return force_bytes(s, encoding, strings_only, errors)
87def force_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
88 """
89 Similar to smart_bytes, except that lazy instances are resolved to
90 strings, rather than kept as lazy objects.
92 If strings_only is True, don't convert (some) non-string-like objects.
93 """
94 # Handle the common case first for performance reasons.
95 if isinstance(s, bytes):
96 if encoding == "utf-8":
97 return s
98 else:
99 return s.decode("utf-8", errors).encode(encoding, errors)
100 if strings_only and is_protected_type(s):
101 return s
102 if isinstance(s, memoryview):
103 return bytes(s)
104 return str(s).encode(encoding, errors)
107def iri_to_uri(iri):
108 """
109 Convert an Internationalized Resource Identifier (IRI) portion to a URI
110 portion that is suitable for inclusion in a URL.
112 This is the algorithm from RFC 3987 Section 3.1, slightly simplified since
113 the input is assumed to be a string rather than an arbitrary byte stream.
115 Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or
116 b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded
117 result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/').
118 """
119 # The list of safe characters here is constructed from the "reserved" and
120 # "unreserved" characters specified in RFC 3986 Sections 2.2 and 2.3:
121 # reserved = gen-delims / sub-delims
122 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
123 # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
124 # / "*" / "+" / "," / ";" / "="
125 # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
126 # Of the unreserved characters, urllib.parse.quote() already considers all
127 # but the ~ safe.
128 # The % character is also added to the list of safe characters here, as the
129 # end of RFC 3987 Section 3.1 specifically mentions that % must not be
130 # converted.
131 if iri is None:
132 return iri
133 elif isinstance(iri, Promise):
134 iri = str(iri)
135 return quote(iri, safe="/#%[]=:;$&()+,!?*@'~")
138# List of byte values that uri_to_iri() decodes from percent encoding.
139# First, the unreserved characters from RFC 3986:
140_ascii_ranges = [[45, 46, 95, 126], range(65, 91), range(97, 123)]
141_hextobyte = {
142 (fmt % char).encode(): bytes((char,))
143 for ascii_range in _ascii_ranges
144 for char in ascii_range
145 for fmt in ["%02x", "%02X"]
146}
147# And then everything above 128, because bytes ≥ 128 are part of multibyte
148# Unicode characters.
149_hexdig = "0123456789ABCDEFabcdef"
150_hextobyte.update(
151 {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig[8:] for b in _hexdig}
152)
155def uri_to_iri(uri):
156 """
157 Convert a Uniform Resource Identifier(URI) into an Internationalized
158 Resource Identifier(IRI).
160 This is the algorithm from RFC 3987 Section 3.2, excluding step 4.
162 Take an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and return
163 a string containing the encoded result (e.g. '/I%20♥%20Django/').
164 """
165 if uri is None:
166 return uri
167 uri = force_bytes(uri)
168 # Fast selective unquote: First, split on '%' and then starting with the
169 # second block, decode the first 2 bytes if they represent a hex code to
170 # decode. The rest of the block is the part after '%AB', not containing
171 # any '%'. Add that to the output without further processing.
172 bits = uri.split(b"%")
173 if len(bits) == 1:
174 iri = uri
175 else:
176 parts = [bits[0]]
177 append = parts.append
178 hextobyte = _hextobyte
179 for item in bits[1:]:
180 hex = item[:2]
181 if hex in hextobyte:
182 append(hextobyte[item[:2]])
183 append(item[2:])
184 else:
185 append(b"%")
186 append(item)
187 iri = b"".join(parts)
188 return repercent_broken_unicode(iri).decode()
191def escape_uri_path(path):
192 """
193 Escape the unsafe characters from the path portion of a Uniform Resource
194 Identifier (URI).
195 """
196 # These are the "reserved" and "unreserved" characters specified in RFC
197 # 3986 Sections 2.2 and 2.3:
198 # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
199 # unreserved = alphanum | mark
200 # mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
201 # The list of safe characters here is constructed subtracting ";", "=",
202 # and "?" according to RFC 3986 Section 3.3.
203 # The reason for not subtracting and escaping "/" is that we are escaping
204 # the entire path, not a path segment.
205 return quote(path, safe="/:@&+$,-_.!~*'()")
208def punycode(domain):
209 """Return the Punycode of the given domain if it's non-ASCII."""
210 return domain.encode("idna").decode("ascii")
213def repercent_broken_unicode(path):
214 """
215 As per RFC 3987 Section 3.2, step three of converting a URI into an IRI,
216 repercent-encode any octet produced that is not part of a strictly legal
217 UTF-8 octet sequence.
218 """
219 changed_parts = []
220 while True:
221 try:
222 path.decode()
223 except UnicodeDecodeError as e:
224 # CVE-2019-14235: A recursion shouldn't be used since the exception
225 # handling uses massive amounts of memory
226 repercent = quote(path[e.start : e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
227 changed_parts.append(path[: e.start] + repercent.encode())
228 path = path[e.end :]
229 else:
230 return b"".join(changed_parts) + path
233def filepath_to_uri(path):
234 """Convert a file system path to a URI portion that is suitable for
235 inclusion in a URL.
237 Encode certain chars that would normally be recognized as special chars
238 for URIs. Do not encode the ' character, as it is a valid character
239 within URIs. See the encodeURIComponent() JavaScript function for details.
240 """
241 if path is None:
242 return path
243 # I know about `os.sep` and `os.altsep` but I want to leave
244 # some flexibility for hardcoding separators.
245 return quote(str(path).replace("\\", "/"), safe="/~!*()'")
248def get_system_encoding():
249 """
250 The encoding for the character type functions. Fallback to 'ascii' if the
251 #encoding is unsupported by Python or could not be determined. See tickets
252 #10335 and #5846.
253 """
254 try:
255 encoding = locale.getlocale()[1] or "ascii"
256 codecs.lookup(encoding)
257 except Exception:
258 encoding = "ascii"
259 return encoding
262DEFAULT_LOCALE_ENCODING = get_system_encoding()