Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/mdurl/_parse.py: 99%
140 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:37 +0000
1# Copyright Joyent, Inc. and other Node contributors.
2#
3# Permission is hereby granted, free of charge, to any person obtaining a
4# copy of this software and associated documentation files (the
5# "Software"), to deal in the Software without restriction, including
6# without limitation the rights to use, copy, modify, merge, publish,
7# distribute, sublicense, and/or sell copies of the Software, and to permit
8# persons to whom the Software is furnished to do so, subject to the
9# following conditions:
10#
11# The above copyright notice and this permission notice shall be included
12# in all copies or substantial portions of the Software.
13#
14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20# USE OR OTHER DEALINGS IN THE SOFTWARE.
23# Changes from joyent/node:
24#
25# 1. No leading slash in paths,
26# e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/`
27#
28# 2. Backslashes are not replaced with slashes,
29# so `http:\\example.org\` is treated like a relative path
30#
31# 3. Trailing colon is treated like a part of the path,
32# i.e. in `http://example.org:foo` pathname is `:foo`
33#
34# 4. Nothing is URL-encoded in the resulting object,
35# (in joyent/node some chars in auth and paths are encoded)
36#
37# 5. `url.parse()` does not have `parseQueryString` argument
38#
39# 6. Removed extraneous result properties: `host`, `path`, `query`, etc.,
40# which can be constructed using other parts of the url.
42from __future__ import annotations
44from collections import defaultdict
45import re
47from mdurl._url import URL
49# Reference: RFC 3986, RFC 1808, RFC 2396
51# define these here so at least they only have to be
52# compiled once on the first module load.
53PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE)
54PORT_PATTERN = re.compile(r":[0-9]*$")
56# Special case for a simple path URL
57SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$")
59# RFC 2396: characters reserved for delimiting URLs.
60# We actually just auto-escape these.
61DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t")
63# RFC 2396: characters not allowed for various reasons.
64UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS
66# Allowed by RFCs, but cause of XSS attacks. Always escape these.
67AUTO_ESCAPE = ("'",) + UNWISE
68# Characters that are never ever allowed in a hostname.
69# Note that any invalid chars are also handled, but these
70# are the ones that are *expected* to be seen, so we fast-path
71# them.
72NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE
73HOST_ENDING_CHARS = ("/", "?", "#")
74HOSTNAME_MAX_LEN = 255
75HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$")
76HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$")
77# protocols that can allow "unsafe" and "unwise" chars.
79# protocols that never have a hostname.
80HOSTLESS_PROTOCOL = defaultdict(
81 bool,
82 {
83 "javascript": True,
84 "javascript:": True,
85 },
86)
87# protocols that always contain a // bit.
88SLASHED_PROTOCOL = defaultdict(
89 bool,
90 {
91 "http": True,
92 "https": True,
93 "ftp": True,
94 "gopher": True,
95 "file": True,
96 "http:": True,
97 "https:": True,
98 "ftp:": True,
99 "gopher:": True,
100 "file:": True,
101 },
102)
105class MutableURL:
106 def __init__(self) -> None:
107 self.protocol: str | None = None
108 self.slashes: bool = False
109 self.auth: str | None = None
110 self.port: str | None = None
111 self.hostname: str | None = None
112 self.hash: str | None = None
113 self.search: str | None = None
114 self.pathname: str | None = None
116 def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL":
117 lower_proto = ""
118 slashes = False
119 rest = url
121 # trim before proceeding.
122 # This is to support parse stuff like " http://foo.com \n"
123 rest = rest.strip()
125 if not slashes_denote_host and len(url.split("#")) == 1:
126 # Try fast path regexp
127 simple_path = SIMPLE_PATH_PATTERN.match(rest)
128 if simple_path:
129 self.pathname = simple_path.group(1)
130 if simple_path.group(2):
131 self.search = simple_path.group(2)
132 return self
134 proto = ""
135 proto_match = PROTOCOL_PATTERN.match(rest)
136 if proto_match:
137 proto = proto_match.group()
138 lower_proto = proto.lower()
139 self.protocol = proto
140 rest = rest[len(proto) :]
142 # figure out if it's got a host
143 # user@server is *always* interpreted as a hostname, and url
144 # resolution will treat //foo/bar as host=foo,path=bar because that's
145 # how the browser resolves relative URLs.
146 if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest):
147 slashes = rest.startswith("//")
148 if slashes and not (proto and HOSTLESS_PROTOCOL[proto]):
149 rest = rest[2:]
150 self.slashes = True
152 if not HOSTLESS_PROTOCOL[proto] and (
153 slashes or (proto and not SLASHED_PROTOCOL[proto])
154 ):
155 # there's a hostname.
156 # the first instance of /, ?, ;, or # ends the host.
157 #
158 # If there is an @ in the hostname, then non-host chars *are* allowed
159 # to the left of the last @ sign, unless some host-ending character
160 # comes *before* the @-sign.
161 # URLs are obnoxious.
162 #
163 # ex:
164 # http://a@b@c/ => user:a@b host:c
165 # http://a@b?@c => user:a host:c path:/?@c
167 # v0.12 TODO(isaacs): This is not quite how Chrome does things.
168 # Review our test case against browsers more comprehensively.
170 # find the first instance of any hostEndingChars
171 host_end = -1
172 for i in range(len(HOST_ENDING_CHARS)):
173 hec = rest.find(HOST_ENDING_CHARS[i])
174 if hec != -1 and (host_end == -1 or hec < host_end):
175 host_end = hec
177 # at this point, either we have an explicit point where the
178 # auth portion cannot go past, or the last @ char is the decider.
179 if host_end == -1:
180 # atSign can be anywhere.
181 at_sign = rest.rfind("@")
182 else:
183 # atSign must be in auth portion.
184 # http://a@b/c@d => host:b auth:a path:/c@d
185 at_sign = rest.rfind("@", 0, host_end + 1)
187 # Now we have a portion which is definitely the auth.
188 # Pull that off.
189 if at_sign != -1:
190 auth = rest[:at_sign]
191 rest = rest[at_sign + 1 :]
192 self.auth = auth
194 # the host is the remaining to the left of the first non-host char
195 host_end = -1
196 for i in range(len(NON_HOST_CHARS)):
197 hec = rest.find(NON_HOST_CHARS[i])
198 if hec != -1 and (host_end == -1 or hec < host_end):
199 host_end = hec
200 # if we still have not hit it, then the entire thing is a host.
201 if host_end == -1:
202 host_end = len(rest)
204 if host_end > 0 and rest[host_end - 1] == ":":
205 host_end -= 1
206 host = rest[:host_end]
207 rest = rest[host_end:]
209 # pull out port.
210 self.parse_host(host)
212 # we've indicated that there is a hostname,
213 # so even if it's empty, it has to be present.
214 self.hostname = self.hostname or ""
216 # if hostname begins with [ and ends with ]
217 # assume that it's an IPv6 address.
218 ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith(
219 "]"
220 )
222 # validate a little.
223 if not ipv6_hostname:
224 hostparts = self.hostname.split(".")
225 l = len(hostparts) # noqa: E741
226 i = 0
227 while i < l:
228 part = hostparts[i]
229 if not part:
230 i += 1 # emulate statement3 in JS for loop
231 continue
232 if not HOSTNAME_PART_PATTERN.search(part):
233 newpart = ""
234 k = len(part)
235 j = 0
236 while j < k:
237 if ord(part[j]) > 127:
238 # we replace non-ASCII char with a temporary placeholder
239 # we need this to make sure size of hostname is not
240 # broken by replacing non-ASCII by nothing
241 newpart += "x"
242 else:
243 newpart += part[j]
244 j += 1 # emulate statement3 in JS for loop
246 # we test again with ASCII char only
247 if not HOSTNAME_PART_PATTERN.search(newpart):
248 valid_parts = hostparts[:i]
249 not_host = hostparts[i + 1 :]
250 bit = HOSTNAME_PART_START.search(part)
251 if bit:
252 valid_parts.append(bit.group(1))
253 not_host.insert(0, bit.group(2))
254 if not_host:
255 rest = ".".join(not_host) + rest
256 self.hostname = ".".join(valid_parts)
257 break
258 i += 1 # emulate statement3 in JS for loop
260 if len(self.hostname) > HOSTNAME_MAX_LEN:
261 self.hostname = ""
263 # strip [ and ] from the hostname
264 # the host field still retains them, though
265 if ipv6_hostname:
266 self.hostname = self.hostname[1:-1]
268 # chop off from the tail first.
269 hash = rest.find("#") # noqa: A001
270 if hash != -1:
271 # got a fragment string.
272 self.hash = rest[hash:]
273 rest = rest[:hash]
274 qm = rest.find("?")
275 if qm != -1:
276 self.search = rest[qm:]
277 rest = rest[:qm]
278 if rest:
279 self.pathname = rest
280 if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname:
281 self.pathname = ""
283 return self
285 def parse_host(self, host: str) -> None:
286 port_match = PORT_PATTERN.search(host)
287 if port_match:
288 port = port_match.group()
289 if port != ":":
290 self.port = port[1:]
291 host = host[: -len(port)]
292 if host:
293 self.hostname = host
296def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL:
297 if isinstance(url, URL):
298 return url
299 u = MutableURL()
300 u.parse(url, slashes_denote_host)
301 return URL(
302 u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname
303 )