Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/mdurl/_parse.py: 95%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright Joyent, Inc. and other Node contributors.
2#
3# Permission is hereby granted, free of charge, to any person obtaining a
4# copy of this software and associated documentation files (the
5# "Software"), to deal in the Software without restriction, including
6# without limitation the rights to use, copy, modify, merge, publish,
7# distribute, sublicense, and/or sell copies of the Software, and to permit
8# persons to whom the Software is furnished to do so, subject to the
9# following conditions:
10#
11# The above copyright notice and this permission notice shall be included
12# in all copies or substantial portions of the Software.
13#
14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20# USE OR OTHER DEALINGS IN THE SOFTWARE.
23# Changes from joyent/node:
24#
25# 1. No leading slash in paths,
26# e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/`
27#
28# 2. Backslashes are not replaced with slashes,
29# so `http:\\example.org\` is treated like a relative path
30#
31# 3. Trailing colon is treated like a part of the path,
32# i.e. in `http://example.org:foo` pathname is `:foo`
33#
34# 4. Nothing is URL-encoded in the resulting object,
35# (in joyent/node some chars in auth and paths are encoded)
36#
37# 5. `url.parse()` does not have `parseQueryString` argument
38#
39# 6. Removed extraneous result properties: `host`, `path`, `query`, etc.,
40# which can be constructed using other parts of the url.
42from __future__ import annotations
44from collections import defaultdict
45import re
47from mdurl._url import URL
49# Reference: RFC 3986, RFC 1808, RFC 2396
51# define these here so at least they only have to be
52# compiled once on the first module load.
53PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE)
54PORT_PATTERN = re.compile(r":[0-9]*$")
56# Special case for a simple path URL
57SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$")
59# RFC 2396: characters reserved for delimiting URLs.
60# We actually just auto-escape these.
61DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t")
63# RFC 2396: characters not allowed for various reasons.
64UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS
66# Allowed by RFCs, but cause of XSS attacks. Always escape these.
67AUTO_ESCAPE = ("'",) + UNWISE
68# Characters that are never ever allowed in a hostname.
69# Note that any invalid chars are also handled, but these
70# are the ones that are *expected* to be seen, so we fast-path
71# them.
72NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE
73HOST_ENDING_CHARS = ("/", "?", "#")
74HOSTNAME_MAX_LEN = 255
75HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$")
76HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$")
77# protocols that can allow "unsafe" and "unwise" chars.
79# protocols that never have a hostname.
80HOSTLESS_PROTOCOL = defaultdict(
81 bool,
82 {
83 "javascript": True,
84 "javascript:": True,
85 },
86)
87# protocols that always contain a // bit.
88SLASHED_PROTOCOL = defaultdict(
89 bool,
90 {
91 "http": True,
92 "https": True,
93 "ftp": True,
94 "gopher": True,
95 "file": True,
96 "http:": True,
97 "https:": True,
98 "ftp:": True,
99 "gopher:": True,
100 "file:": True,
101 },
102)
105class MutableURL:
106 def __init__(self) -> None:
107 self.protocol: str | None = None
108 self.slashes: bool = False
109 self.auth: str | None = None
110 self.port: str | None = None
111 self.hostname: str | None = None
112 self.hash: str | None = None
113 self.search: str | None = None
114 self.pathname: str | None = None
116 def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL":
117 lower_proto = ""
118 slashes = False
119 rest = url
121 # trim before proceeding.
122 # This is to support parse stuff like " http://foo.com \n"
123 rest = rest.strip()
125 if not slashes_denote_host and len(url.split("#")) == 1:
126 # Try fast path regexp
127 simple_path = SIMPLE_PATH_PATTERN.match(rest)
128 if simple_path:
129 self.pathname = simple_path.group(1)
130 if simple_path.group(2):
131 self.search = simple_path.group(2)
132 return self
134 proto = ""
135 proto_match = PROTOCOL_PATTERN.match(rest)
136 if proto_match:
137 proto = proto_match.group()
138 lower_proto = proto.lower()
139 self.protocol = proto
140 rest = rest[len(proto) :]
142 # figure out if it's got a host
143 # user@server is *always* interpreted as a hostname, and url
144 # resolution will treat //foo/bar as host=foo,path=bar because that's
145 # how the browser resolves relative URLs.
146 if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest):
147 slashes = rest.startswith("//")
148 if slashes and not (proto and HOSTLESS_PROTOCOL[proto]):
149 rest = rest[2:]
150 self.slashes = True
152 if not HOSTLESS_PROTOCOL[proto] and (
153 slashes or (proto and not SLASHED_PROTOCOL[proto])
154 ):
156 # there's a hostname.
157 # the first instance of /, ?, ;, or # ends the host.
158 #
159 # If there is an @ in the hostname, then non-host chars *are* allowed
160 # to the left of the last @ sign, unless some host-ending character
161 # comes *before* the @-sign.
162 # URLs are obnoxious.
163 #
164 # ex:
165 # http://a@b@c/ => user:a@b host:c
166 # http://a@b?@c => user:a host:c path:/?@c
168 # v0.12 TODO(isaacs): This is not quite how Chrome does things.
169 # Review our test case against browsers more comprehensively.
171 # find the first instance of any hostEndingChars
172 host_end = -1
173 for i in range(len(HOST_ENDING_CHARS)):
174 hec = rest.find(HOST_ENDING_CHARS[i])
175 if hec != -1 and (host_end == -1 or hec < host_end):
176 host_end = hec
178 # at this point, either we have an explicit point where the
179 # auth portion cannot go past, or the last @ char is the decider.
180 if host_end == -1:
181 # atSign can be anywhere.
182 at_sign = rest.rfind("@")
183 else:
184 # atSign must be in auth portion.
185 # http://a@b/c@d => host:b auth:a path:/c@d
186 at_sign = rest.rfind("@", 0, host_end + 1)
188 # Now we have a portion which is definitely the auth.
189 # Pull that off.
190 if at_sign != -1:
191 auth = rest[:at_sign]
192 rest = rest[at_sign + 1 :]
193 self.auth = auth
195 # the host is the remaining to the left of the first non-host char
196 host_end = -1
197 for i in range(len(NON_HOST_CHARS)):
198 hec = rest.find(NON_HOST_CHARS[i])
199 if hec != -1 and (host_end == -1 or hec < host_end):
200 host_end = hec
201 # if we still have not hit it, then the entire thing is a host.
202 if host_end == -1:
203 host_end = len(rest)
205 if host_end > 0 and rest[host_end - 1] == ":":
206 host_end -= 1
207 host = rest[:host_end]
208 rest = rest[host_end:]
210 # pull out port.
211 self.parse_host(host)
213 # we've indicated that there is a hostname,
214 # so even if it's empty, it has to be present.
215 self.hostname = self.hostname or ""
217 # if hostname begins with [ and ends with ]
218 # assume that it's an IPv6 address.
219 ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith(
220 "]"
221 )
223 # validate a little.
224 if not ipv6_hostname:
225 hostparts = self.hostname.split(".")
226 l = len(hostparts) # noqa: E741
227 i = 0
228 while i < l:
229 part = hostparts[i]
230 if not part:
231 i += 1 # emulate statement3 in JS for loop
232 continue
233 if not HOSTNAME_PART_PATTERN.search(part):
234 newpart = ""
235 k = len(part)
236 j = 0
237 while j < k:
238 if ord(part[j]) > 127:
239 # we replace non-ASCII char with a temporary placeholder
240 # we need this to make sure size of hostname is not
241 # broken by replacing non-ASCII by nothing
242 newpart += "x"
243 else:
244 newpart += part[j]
245 j += 1 # emulate statement3 in JS for loop
247 # we test again with ASCII char only
248 if not HOSTNAME_PART_PATTERN.search(newpart):
249 valid_parts = hostparts[:i]
250 not_host = hostparts[i + 1 :]
251 bit = HOSTNAME_PART_START.search(part)
252 if bit:
253 valid_parts.append(bit.group(1))
254 not_host.insert(0, bit.group(2))
255 if not_host:
256 rest = ".".join(not_host) + rest
257 self.hostname = ".".join(valid_parts)
258 break
259 i += 1 # emulate statement3 in JS for loop
261 if len(self.hostname) > HOSTNAME_MAX_LEN:
262 self.hostname = ""
264 # strip [ and ] from the hostname
265 # the host field still retains them, though
266 if ipv6_hostname:
267 self.hostname = self.hostname[1:-1]
269 # chop off from the tail first.
270 hash = rest.find("#") # noqa: A001
271 if hash != -1:
272 # got a fragment string.
273 self.hash = rest[hash:]
274 rest = rest[:hash]
275 qm = rest.find("?")
276 if qm != -1:
277 self.search = rest[qm:]
278 rest = rest[:qm]
279 if rest:
280 self.pathname = rest
281 if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname:
282 self.pathname = ""
284 return self
286 def parse_host(self, host: str) -> None:
287 port_match = PORT_PATTERN.search(host)
288 if port_match:
289 port = port_match.group()
290 if port != ":":
291 self.port = port[1:]
292 host = host[: -len(port)]
293 if host:
294 self.hostname = host
297def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL:
298 if isinstance(url, URL):
299 return url
300 u = MutableURL()
301 u.parse(url, slashes_denote_host)
302 return URL(
303 u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname
304 )