Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/yarl/

1"""URL parsing utilities."""

3import codecs

4import re

5import unicodedata

6from functools import lru_cache

7from urllib.parse import parse_qsl, scheme_chars

9from ._quoters import QUOTER, UNQUOTER_PLUS

11# Leading and trailing C0 control and space to be stripped per WHATWG spec.

12# == "".join([chr(i) for i in range(0, 0x20 + 1)])

13WHATWG_C0_CONTROL_OR_SPACE = (

14 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10"

15 "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f "

16)

18# Unsafe bytes to be removed per WHATWG spec

19UNSAFE_URL_BYTES_TO_REMOVE = ["\t", "\r", "\n"]

20# The WHATWG "special" schemes: their URLs always have an authority, so in

21# WHATWG mode "//" is written out even when the authority is missing.

22SPECIAL_SCHEMES = frozenset({"http", "https", "ws", "wss", "ftp", "file"})

24# IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ), RFC 3986

25# section 3.2.2; the "v" is case-insensitive.

26IP_FUTURE_RE = re.compile(r"[vV][0-9A-Fa-f]+\.[A-Za-z0-9\-._~!$&'()*+,;=:]+")

28# Bits of the "empty" mask: the component is present but empty, as in

29# "sc://", "http://h/?" or "http://h/#". An absent and an empty component

30# are different per RFC 3986 section 5.3, but both are stored as "".

31EMPTY_AUTHORITY = 1

32EMPTY_QUERY = 2

33EMPTY_FRAGMENT = 4

35SplitURLType = tuple[str, str, str, str, str]

37# The ZoneID of RFC 6874; in a URL it follows the address as "%25" <ZoneID>.

38# RFC 6874 requires at least one character, yarl also accepts an empty zone

39# identifier (#998).

40ZONE_ID_RE = re.compile(r"(?:[0-9A-Za-z._~-]|%[0-9A-Fa-f]{2})*")

43def check_zone_id(ip_literal: str) -> None:

44 """Reject an IPv6 zone identifier not written as RFC 6874 says.

46 The zone identifier follows the address as "%25" and the ZoneID, whose

47 characters other than unreserved ones are percent-encoded. A bare "%",

48 as in "fe80::1%eth0" (the user interface form of RFC 9844), or another

49 percent-encoded octet, as in "::%31", is not URI syntax.

50 """

51 pct = ip_literal.index("%")

52 if ip_literal[pct : pct + 3] != "%25" or not ZONE_ID_RE.fullmatch(

53 ip_literal, pct + 3

54 ):

55 raise ValueError(f"Invalid IPv6 zone identifier in {ip_literal!r}")

58def split_url(url: str) -> tuple[str, str, str, str, str, int]:

59 """Split URL into parts and the mask of present but empty parts."""

60 # Adapted from urllib.parse.urlsplit

61 # Strip leading and trailing C0 control or space in both modes, as the

62 # WHATWG basic URL parser does; RFC 3986 appendix C asks the same of

63 # software that accepts user-typed URIs.

64 url = url.strip(WHATWG_C0_CONTROL_OR_SPACE)

65 for b in UNSAFE_URL_BYTES_TO_REMOVE:

66 if b in url:

67 url = url.replace(b, "")

69 scheme = netloc = query = fragment = ""

70 empty = 0

71 i = url.find(":")

72 if i > 0 and url[0] in scheme_chars:

73 for c in url[1:i]:

74 if c not in scheme_chars:

75 break

76 else:

77 scheme, url = url[:i].lower(), url[i + 1 :]

78 has_hash = "#" in url

79 has_question_mark = "?" in url

80 if url[:2] == "//":

81 empty = EMPTY_AUTHORITY

82 delim = len(url) # position of end of domain part of url, default is end

83 if has_hash and has_question_mark:

84 delim_chars = "/?#"

85 elif has_question_mark:

86 delim_chars = "/?"

87 elif has_hash:

88 delim_chars = "/#"

89 else:

90 delim_chars = "/"

91 for c in delim_chars: # look for delimiters; the order is NOT important

92 wdelim = url.find(c, 2) # find first of this delim

93 if wdelim >= 0 and wdelim < delim: # if found

94 delim = wdelim # use earliest delim position

95 netloc = url[2:delim]

96 url = url[delim:]

97 # Backslash is not valid in the authority component per RFC 3986.

98 # WHATWG parsers treat \ as a path separator for special schemes, so

99 # accepting it in the authority can cause host parsing ambiguity.

100 # Other schemes keep it, percent-encoded, in the userinfo, which goes

101 # up to the last "@" as in split_netloc().

102 if "\\" in netloc and (

103 not scheme or scheme in SPECIAL_SCHEMES or "\\" in netloc.rpartition("@")[2]

104 ):

105 raise ValueError(

106 "Invalid URL: backslash ('\\') is not allowed in the authority "

107 "component per RFC 3986."

108 )

109 has_left_bracket = "[" in netloc

110 has_right_bracket = "]" in netloc

111 if (has_left_bracket or has_right_bracket) and "@" in netloc:

112 # Brackets in the userinfo are no IP-literal, WHATWG percent-encodes

113 # them.

114 hostinfo = netloc.rpartition("@")[2]

115 has_left_bracket = "[" in hostinfo

116 has_right_bracket = "]" in hostinfo

117 if (has_left_bracket and not has_right_bracket) or (

118 has_right_bracket and not has_left_bracket

119 ):

120 raise ValueError("Invalid IPv6 URL")

121 if has_left_bracket:

122 # Per RFC 3986, brackets are only valid at the START of the host

123 # for IP-literal addresses. Text before '[' (e.g. '127.0.0.1[::1]')

124 # is invalid and must be rejected to prevent SSRF bypasses. The

125 # count checks reject URLs with more than one bracket pair in the

126 # host subcomponent (e.g. 'http://[:localhost[]].google:80'),

127 # which would otherwise resolve to an unintended host.

128 hostinfo = netloc.rpartition("@")[2]

129 if hostinfo[0] != "[" or hostinfo.count("[") > 1 or hostinfo.count("]") > 1:

130 raise ValueError("Invalid IPv6 URL")

131 bracketed_host, _, after_bracket = hostinfo[1:].partition("]")

132 # Per RFC 3986 §3.2.2, after the closing ']' of an IP-literal

133 # only ":" <port> or end-of-authority is valid. Any other text

134 # (e.g. '[::1]allowed.example:1') must be rejected to prevent

135 # host-confusion where the suffix is silently dropped.

136 if after_bracket and after_bracket[0] != ":":

137 raise ValueError("Invalid IPv6 URL")

138 # Valid bracketed hosts are defined in

139 # https://www.rfc-editor.org/rfc/rfc3986#page-49

140 # https://url.spec.whatwg.org/

141 if bracketed_host and bracketed_host[0] in "vV":

142 if not IP_FUTURE_RE.fullmatch(bracketed_host):

143 raise ValueError("IPvFuture address is invalid")

144 elif ":" not in bracketed_host:

145 raise ValueError("The IPv6 content between brackets is not valid")

146 elif "%" in bracketed_host:

147 check_zone_id(bracketed_host)

148 if has_hash:

149 url, _, fragment = url.partition("#")

150 empty |= EMPTY_FRAGMENT

151 if has_question_mark and "?" in url:

152 url, _, query = url.partition("?")

153 empty |= EMPTY_QUERY

154 if netloc:

155 if not netloc.isascii():

156 _check_netloc(netloc)

157 empty &= ~EMPTY_AUTHORITY

158 if query:

159 empty &= ~EMPTY_QUERY

160 if fragment:

161 empty &= ~EMPTY_FRAGMENT

162 return scheme, netloc, url, query, fragment, empty

163

164

165def _check_netloc(netloc: str) -> None:

166 # Adapted from urllib.parse._checknetloc

167 # looking for characters like \u2100 that expand to 'a/c'

168 # IDNA uses NFKC equivalence, so normalize for this check

169

170 # ignore characters already included

171 # but not the surrounding text

172 n = netloc.replace("@", "").replace(":", "").replace("#", "").replace("?", "")

173 normalized_netloc = unicodedata.normalize("NFKC", n)

174 if n == normalized_netloc:

175 return

176 # Note that there are no unicode decompositions for the character '@' so

177 # its currently impossible to have test coverage for this branch, however if the

178 # one should be added in the future we want to make sure its still checked.

179 for c in "/?#@:%": # pragma: no branch

180 if c in normalized_netloc:

181 raise ValueError(

182 f"netloc '{netloc}' contains invalid "

183 "characters under NFKC normalization"

184 )

185

186

187@lru_cache # match the same size as urlsplit

188def split_netloc(

189 netloc: str,

190) -> tuple[str | None, str | None, str | None, int | None]:

191 """Split netloc into username, password, host and port.

192

193 An empty username is None, see split_netloc_rfc() for RFC 3986 mode.

194 """

195 if "@" not in netloc:

196 username: str | None = None

197 password: str | None = None

198 hostinfo = netloc

199 else:

200 userinfo, _, hostinfo = netloc.rpartition("@")

201 username, have_password, password = userinfo.partition(":")

202 if not have_password:

203 password = None

204

205 if "[" in hostinfo:

206 if hostinfo[0] != "[" or hostinfo.count("[") > 1 or hostinfo.count("]") > 1:

207 raise ValueError("Invalid IPv6 URL")

208 _, _, bracketed = hostinfo.partition("[")

209 hostname, _, port_str = bracketed.partition("]")

210 # Defense-in-depth: after ']' only ':port' or empty is valid.

211 # split_url() should have already rejected invalid suffixes,

212 # but guard here too for callers that use split_netloc() directly.

213 if port_str and port_str[0] != ":":

214 raise ValueError("Invalid IPv6 URL")

215 _, _, port_str = port_str.partition(":")

216 if hostname[:1] in ("v", "V"):

217 # An IPvFuture address keeps its brackets: without them it would

218 # read as a reg-name. IPv6 addresses never start with "v".

219 hostname = f"[{hostname}]"

220 else:

221 hostname, _, port_str = hostinfo.partition(":")

222

223 if not port_str:

224 return username or None, password, hostname or None, None

225

226 # RFC 3986 section 3.2.3 defines the port as *DIGIT, i.e. ASCII digits

227 # only. int() is more permissive and would accept a leading '+',

228 # surrounding whitespace, underscore digit separators and non-ASCII

229 # decimal digits, so reject those before converting.

230 if not (port_str.isascii() and port_str.isdigit()):

231 raise ValueError("Invalid URL: port can't be converted to integer")

232 port = int(port_str)

233 if not (0 <= port <= 65535):

234 raise ValueError("Port out of range 0-65535")

235 return username or None, password, hostname or None, port

236

237

238@lru_cache

239def split_netloc_rfc(

240 netloc: str,

241) -> tuple[str | None, str | None, str | None, int | None]:

242 """Split netloc into username, password, host and port in RFC 3986 mode.

243

244 As split_netloc(), but an empty username is "" when the netloc has

245 userinfo, as RFC 3986 keeps an empty userinfo: "@h" and ":p@h" have the

246 username "". A separate cache keeps the single string key, which a mode

247 argument would turn into a tuple.

248 """

249 username, password, host, port = split_netloc(netloc)

250 if username is None and "@" in netloc:

251 username = ""

252 return username, password, host, port

253

254

255# "//" behind any number of "/." segments, see needs_dot_prefix().

256_DOT_PREFIXED_AUTHORITY_RE = re.compile(r"(?:/\.)*//")

257

258

259def needs_dot_prefix(path: str) -> bool:

260 """Tell if an authority-less path needs "/." in front when written out.

261

262 A path starting with "//" would read as an authority, so str() writes

263 "/." before it and the parsers drop that "/." again. To keep a literal

264 "/." segment in such a place, a path that is "//..." behind any number

265 of "/." segments gets the prefix too, e.g. "/.//a" is written "/././/a".

266 """

267 return _DOT_PREFIXED_AUTHORITY_RE.match(path) is not None

268

269

270def has_dot_prefix(path: str) -> bool:

271 """Tell if an authority-less path starts with the "/." str() adds."""

272 return path[:3] == "/./" and _DOT_PREFIXED_AUTHORITY_RE.match(path, 2) is not None

273

274

275def unsplit_result(

276 scheme: str, netloc: str, url: str, query: str, fragment: str

277) -> str:

278 """Unsplit a URL without any normalization."""

279 if netloc:

280 if url and url[:1] != "/":

281 url = f"{scheme}://{netloc}/{url}" if scheme else f"{scheme}:{url}"

282 else:

283 url = f"{scheme}://{netloc}{url}" if scheme else f"//{netloc}{url}"

284 else:

285 if url[:2] == "//" or (url[:3] == "/./" and needs_dot_prefix(url)):

286 # Without an authority a path cannot start with "//", which

287 # would read as one; "/." keeps it a path, as WHATWG does.

288 url = f"/.{url}"

289 if scheme:

290 url = f"{scheme}:{url}"

291 if query:

292 url = f"{url}?{query}"

293 return f"{url}#{fragment}" if fragment else url

294

295

296def unsplit_result_empty(

297 scheme: str, netloc: str, url: str, query: str, fragment: str, empty: int

298) -> str:

299 """Unsplit a URL that has present but empty components.

300

301 *empty* is a mask of EMPTY_AUTHORITY, EMPTY_QUERY and EMPTY_FRAGMENT

302 telling which empty components are present.

303 """

304 if netloc or empty & EMPTY_AUTHORITY:

305 if url and url[:1] != "/":

306 url = f"{scheme}://{netloc}/{url}" if scheme else f"{scheme}:{url}"

307 else:

308 url = f"{scheme}://{netloc}{url}" if scheme else f"//{netloc}{url}"

309 else:

310 if url[:2] == "//" or (url[:3] == "/./" and needs_dot_prefix(url)):

311 url = f"/.{url}"

312 if scheme:

313 url = f"{scheme}:{url}"

314 if query or empty & EMPTY_QUERY:

315 url = f"{url}?{query}"

316 return f"{url}#{fragment}" if fragment or empty & EMPTY_FRAGMENT else url

317

318

319def quote_userinfo(value: str, whatwg: bool) -> str:

320 """Encode a username or password that is not encoded.

321

322 WHATWG mode also encodes ";" and "=", which are in the userinfo

323 percent-encode set of the WHATWG URL Standard; RFC 3986 allows them.

324 """

325 value = QUOTER(value)

326 if whatwg and (";" in value or "=" in value):

327 value = value.replace(";", "%3B").replace("=", "%3D")

328 return value

329

330

331@lru_cache # match the same size as urlsplit

332def make_netloc(

333 user: str | None,

334 password: str | None,

335 host: str | None,

336 port: int | None,

337 encode: bool = False,

338 whatwg: bool = False,

339) -> str:

340 """Make netloc from parts.

341

342 The user and password are encoded if encode is True, see

343 quote_userinfo() for whatwg. A user that is

344 not None is written with "@" even when it is empty, as RFC 3986 mode

345 keeps an empty userinfo; WHATWG mode passes None for an empty user.

346

347 The host must already be encoded with _encode_host.

348 """

349 if host is None:

350 return ""

351 ret = host

352 if port is not None:

353 ret = f"{ret}:{port}"

354 if user is None and password is None:

355 return ret

356 if password is not None:

357 if not user:

358 user = ""

359 elif encode:

360 user = quote_userinfo(user, whatwg)

361 if encode:

362 password = quote_userinfo(password, whatwg)

363 user = f"{user}:{password}"

364 elif user and encode:

365 user = quote_userinfo(user, whatwg)

366 return f"{user}@{ret}"

367

368

369def query_to_pairs(

370 query_string: str, *, max_fields: int | None = None, encoding: str = "utf-8"

371) -> list[tuple[str, str]]:

372 """Parse a query string into a list of decoded name, value pairs.

373

374 The result is the same as

375 ``urllib.parse.parse_qsl(query_string, keep_blank_values=True,

376 encoding=encoding, max_num_fields=max_fields)``.

377

378 Raises :exc:`ValueError` if *max_fields* is not ``None`` and the

379 query string has more than *max_fields* fields.

380 """

381 if not query_string:

382 return []

383 if max_fields is not None and query_string.count("&") >= max_fields:

384 raise ValueError("Max number of fields exceeded")

385 pairs: list[tuple[str, str]] = []

386 if "%" not in query_string:

387 # Nothing to decode except '+', which is the same in every encoding

388 for name_value in query_string.replace("+", " ").split("&"):

389 if name_value:

390 name, _, value = name_value.partition("=")

391 pairs.append((name, value))

392 return pairs

393 if encoding != "utf-8" and codecs.lookup(encoding).name != "utf-8":

394 return parse_qsl(query_string, keep_blank_values=True, encoding=encoding)

395 for name_value in query_string.split("&"):

396 if name_value:

397 name, _, value = name_value.partition("=")

398 pairs.append((UNQUOTER_PLUS(name), UNQUOTER_PLUS(value)))

399 return pairs

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/yarl/_parse.py: 73%

206 statements