Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/url.py: 89%

1from __future__ import annotations

3import re

4import typing

5from functools import partial

6from urllib.parse import unquote as _unquote

8from ..exceptions import LocationParseError

9from .util import to_str

11# We only want to normalize urls with an HTTP(S) scheme.

12# urllib3 infers URLs without a scheme (None) to be http.

13_NORMALIZABLE_SCHEMES = ("http", "https", None)

15# Almost all of these patterns were derived from the

16# 'rfc3986' module: https://github.com/python-hyper/rfc3986

17_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")

18_HOST_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}|%")

19_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")

20_HOST_INVALID_CHAR_RE = re.compile(r"[\x00-\x20\x7f]")

21_URI_RE = re.compile(

22 r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"

23 r"(?://([^\\/?#]*))?"

24 r"([^?#]*)"

25 r"(?:\?([^#]*))?"

26 r"(?:#(.*))?$",

27 re.UNICODE | re.DOTALL,

28)

30_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"

31_HEX_PAT = "[0-9A-Fa-f]{1,4}"

32_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)

33_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}

34_variations = [

35 # 6( h16 ":" ) ls32

36 "(?:%(hex)s:){6}%(ls32)s",

37 # "::" 5( h16 ":" ) ls32

38 "::(?:%(hex)s:){5}%(ls32)s",

39 # [ h16 ] "::" 4( h16 ":" ) ls32

40 "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",

41 # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32

42 "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",

43 # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32

44 "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",

45 # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32

46 "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",

47 # [ *4( h16 ":" ) h16 ] "::" ls32

48 "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",

49 # [ *5( h16 ":" ) h16 ] "::" h16

50 "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",

51 # [ *6( h16 ":" ) h16 ] "::"

52 "(?:(?:%(hex)s:){0,6}%(hex)s)?::",

53]

55_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"

56_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"

57_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"

58_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"

59_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"

60_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")

62_IPV4_RE = re.compile(

63 r"^(?:0[xX][0-9a-fA-F]+|[0-9]+)(?:\.(?:0[xX][0-9a-fA-F]+|[0-9]+)){0,3}$"

64)

65_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")

66_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")

67_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")

68_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")

70_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (

71 _REG_NAME_PAT,

72 _IPV4_PAT,

73 _IPV6_ADDRZ_PAT,

74)

75_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)

77_UNRESERVED_CHARS = set(

78 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"

79)

80_SUB_DELIM_CHARS = set("!$&'()*+,;=")

81_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}

82_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}

83_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}

86class Url(

87 typing.NamedTuple(

88 "Url",

89 [

90 ("scheme", typing.Optional[str]),

91 ("auth", typing.Optional[str]),

92 ("host", typing.Optional[str]),

93 ("port", typing.Optional[int]),

94 ("path", typing.Optional[str]),

95 ("query", typing.Optional[str]),

96 ("fragment", typing.Optional[str]),

97 ],

98 )

99):

100 """

101 Data structure for representing an HTTP URL. Used as a return value for

102 :func:`parse_url`. Both the scheme and host are normalized as they are

103 both case-insensitive according to RFC 3986.

104

105 :param auth: User information as defined in RFC 3986 3.2.1. This

106 component is kept percent-encoded. Use :attr:`auth_decoded` or

107 :attr:`auth_decoded_joined` to get the decoded form.

108 """

109

110 def __new__( # type: ignore[no-untyped-def]

111 cls,

112 scheme: str | None = None,

113 auth: str | None = None,

114 host: str | None = None,

115 port: int | None = None,

116 path: str | None = None,

117 query: str | None = None,

118 fragment: str | None = None,

119 ):

120 if path and not path.startswith("/"):

121 path = "/" + path

122 if scheme is not None:

123 scheme = scheme.lower()

124 return super().__new__(cls, scheme, auth, host, port, path, query, fragment)

125

126 @property

127 def auth_decoded(self) -> tuple[None, None] | tuple[str, str | None]:

128 """

129 User information with %-escapes decoded as UTF-8, returned as a

130 ``(username, password)`` tuple.

131

132 Both values are ``None`` if ``auth`` is ``None``.

133 ``password`` is ``None`` if not present in the auth component.

134 """

135 if self.auth is None:

136 return None, None

137 username, sep, password = self.auth.partition(":")

138 return (

139 _unquote(username, encoding="utf-8"),

140 _unquote(password, encoding="utf-8") if sep else None,

141 )

142

143 @property

144 def auth_decoded_joined(self) -> str | None:

145 """

146 User information with %-escapes decoded as UTF-8, as a string

147 prepared for encoding into an 'authorization: basic ...' header.

148

149 This property does not choose the encoding used for an

150 'authorization: basic ...' header. Use the ``basic_auth_encoding`` or

151 ``proxy_basic_auth_encoding`` parameters of

152 :func:`urllib3.util.make_headers` when converting the returned string

153 into Basic authentication bytes.

154

155 This is a convenience property that joins the username and

156 password with a colon, if both are present.

157 If only the username is present, a trailing colon is still

158 appended.

159 If ``auth`` is ``None``, this returns ``None``.

160 """

161 username, password = self.auth_decoded

162 if username is None:

163 return None

164 return f"{username}:{password or ''}"

165

166 @property

167 def hostname(self) -> str | None:

168 """For backwards-compatibility with urlparse. We're nice like that."""

169 return self.host

170

171 @property

172 def request_uri(self) -> str:

173 """Absolute path including the query string."""

174 uri = self.path or "/"

175

176 if self.query is not None:

177 uri += "?" + self.query

178

179 return uri

180

181 @property

182 def authority(self) -> str | None:

183 """

184 Authority component as defined in RFC 3986 3.2.

185 This includes userinfo (auth), host and port.

186

187 i.e.

188 userinfo@host:port

189 """

190 userinfo = self.auth

191 netloc = self.netloc

192 if netloc is None or userinfo is None:

193 return netloc

194 else:

195 return f"{userinfo}@{netloc}"

196

197 @property

198 def netloc(self) -> str | None:

199 """

200 Network location including host and port.

201

202 If you need the equivalent of urllib.parse's ``netloc``,

203 use the ``authority`` property instead.

204 """

205 if self.host is None:

206 return None

207 if self.port is not None:

208 return f"{self.host}:{self.port}"

209 return self.host

210

211 @property

212 def url(self) -> str:

213 """

214 Convert self into a url

215

216 This function should more or less round-trip with :func:`.parse_url`. The

217 returned url may not be exactly the same as the url inputted to

218 :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls

219 with a blank port will have : removed).

220

221 Example:

222

223 .. code-block:: python

224

225 import urllib3

226

227 U = urllib3.util.parse_url("https://google.com/mail/")

228

229 print(U.url)

230 # "https://google.com/mail/"

231

232 print( urllib3.util.Url("https", "username:password",

233 "host.com", 80, "/path", "query", "fragment"

234 ).url

235 )

236 # "https://username:password@host.com:80/path?query#fragment"

237 """

238 scheme, auth, host, port, path, query, fragment = self

239 url = ""

240

241 # We use "is not None" we want things to happen with empty strings (or 0 port)

242 if scheme is not None:

243 url += scheme + "://"

244 if auth is not None:

245 url += auth + "@"

246 if host is not None:

247 url += host

248 if port is not None:

249 url += ":" + str(port)

250 if path is not None:

251 url += path

252 if query is not None:

253 url += "?" + query

254 if fragment is not None:

255 url += "#" + fragment

256

257 return url

258

259 def __str__(self) -> str:

260 return self.url

261

262

263@typing.overload

264def _encode_invalid_chars(

265 component: str, allowed_chars: typing.Container[str]

266) -> str: # Abstract

267 ...

268

269

270@typing.overload

271def _encode_invalid_chars(

272 component: None, allowed_chars: typing.Container[str]

273) -> None: # Abstract

274 ...

275

276

277def _encode_invalid_chars(

278 component: str | None, allowed_chars: typing.Container[str]

279) -> str | None:

280 """Percent-encodes a URI component without reapplying

281 onto an already percent-encoded component.

282 """

283 if component is None:

284 return component

285

286 component = to_str(component)

287

288 # Fast path: an ASCII component that consists only of allowed characters

289 # and contains no '%' needs neither encoding nor percent-normalization.

290 # Components containing '%' always take the general path below because

291 # how '%' is treated depends on whether the whole component is already

292 # percent-encoded.

293 if (

294 "%" not in component

295 and component.isascii()

296 and all(map(allowed_chars.__contains__, component))

297 ):

298 return component

299

300 # Normalize existing percent-encoded bytes.

301 # Try to see if the component we're encoding is already percent-encoded

302 # so we can skip all '%' characters but still encode all others.

303 component, percent_encodings = _PERCENT_RE.subn(

304 lambda match: match.group(0).upper(), component

305 )

306

307 uri_bytes = component.encode("utf-8", "surrogatepass")

308 is_percent_encoded = percent_encodings == uri_bytes.count(b"%")

309 encoded_component = bytearray()

310

311 # Iterating over bytes yields integers; 0x25 is '%'.

312 for byte_ord in uri_bytes:

313 if (is_percent_encoded and byte_ord == 0x25) or (

314 byte_ord < 128 and chr(byte_ord) in allowed_chars

315 ):

316 encoded_component.append(byte_ord)

317 else:

318 encoded_component += b"%%%02X" % byte_ord

319

320 return encoded_component.decode()

321

322

323def _remove_path_dot_segments(path: str) -> str:

324 # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code

325 segments = path.split("/") # Turn the path into a list of segments

326 output = [] # Initialize the variable to use to store output

327

328 for segment in segments:

329 # '.' is the current directory, so ignore it, it is superfluous

330 if segment == ".":

331 continue

332 # Anything other than '..', should be appended to the output

333 if segment != "..":

334 output.append(segment)

335 # In this case segment == '..', if we can, we should pop the last

336 # element

337 elif output:

338 output.pop()

339

340 # If the path starts with '/' and the output is empty or the first string

341 # is non-empty

342 if path.startswith("/") and (not output or output[0]):

343 output.insert(0, "")

344

345 # If the path starts with '/.' or '/..' ensure we add one more empty

346 # string to add a trailing '/'

347 if path.endswith(("/.", "/..")):

348 output.append("")

349

350 return "/".join(output)

351

352

353@typing.overload

354def _normalize_host(host: None, scheme: str | None) -> None: ...

355

356

357@typing.overload

358def _normalize_host(host: str, scheme: str | None) -> str: ...

359

360

361def _normalize_host(host: str | None, scheme: str | None) -> str | None:

362 if host:

363 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host)

364 if invalid_host_char:

365 raise LocationParseError(

366 f"Host {host!r} contains invalid character "

367 f"{invalid_host_char.group()!r}"

368 )

369 if scheme in _NORMALIZABLE_SCHEMES:

370 is_ipv6 = _IPV6_ADDRZ_RE.match(host)

371 if is_ipv6:

372 # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as

373 # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID

374 # separator as necessary to return a valid RFC 4007 scoped IP.

375 match = _ZONE_ID_RE.search(host)

376 if match:

377 start, end = match.span(1)

378 zone_id = host[start:end]

379

380 if zone_id.startswith("%25") and zone_id != "%25":

381 zone_id = zone_id[3:]

382 else:

383 zone_id = zone_id[1:]

384 zone_id = _PERCENT_RE.sub(

385 partial(_normalize_zone_id_percent_encoding, error_host=host),

386 zone_id,

387 )

388 zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)

389 return f"{host[:start].lower()}%{zone_id}{host[end:]}"

390 else:

391 return host.lower()

392 elif not _IPV4_RE.match(host):

393 if "%" in host:

394 address, separator, zone = host.partition("%")

395 if ":" in address and host.isascii() and _IPV6_RE.match(address):

396 # Native IPv6 zone names may be case-sensitive.

397 return f"{address.lower()}{separator}{zone}"

398 host = _HOST_PERCENT_RE.sub(_normalize_host_percent_encoding, host)

399 return to_str(

400 b".".join([_idna_encode(label) for label in host.split(".")]),

401 "ascii",

402 )

403 return host

404

405

406def _decode_percent_encoding(match: re.Match[str], *, error_host: str) -> str:

407 percent_encoded_octet = match.group(0)

408 decoded_octet = chr(int(percent_encoded_octet[1:], 16))

409 # Keep this narrower than _HOST_INVALID_CHAR_RE because "%20" is valid

410 # percent-encoding and remains encoded.

411 if decoded_octet < "\x20" or decoded_octet == "\x7f":

412 raise LocationParseError(

413 f"Host {error_host!r} contains invalid percent-encoded "

414 f"control character {percent_encoded_octet!r}"

415 )

416 return decoded_octet

417

418

419def _normalize_host_percent_encoding(match: re.Match[str]) -> str:

420 # Reject invalid percent encodings

421 if match.group(0) == "%":

422 raise LocationParseError(f"{match.string!r} is not a valid host")

423 decoded_octet = _decode_percent_encoding(match, error_host=match.string)

424 if decoded_octet in _UNRESERVED_CHARS:

425 return decoded_octet

426 return match.group(0).upper()

427

428

429def _normalize_zone_id_percent_encoding(

430 match: re.Match[str], *, error_host: str

431) -> str:

432 _decode_percent_encoding(match, error_host=error_host)

433 return match.group(0).upper()

434

435

436def _idna_encode(name: str) -> bytes:

437 if not name.isascii():

438 try:

439 import idna

440 except ImportError:

441 raise LocationParseError(

442 "Unable to parse URL without the 'idna' module"

443 ) from None

444

445 try:

446 return idna.encode(name.lower(), strict=True, std3_rules=True)

447 except idna.IDNAError:

448 raise LocationParseError(

449 f"Name '{name}' is not a valid IDNA label"

450 ) from None

451

452 return _PERCENT_RE.sub(lambda match: match.group(0).upper(), name.lower()).encode(

453 "ascii"

454 )

455

456

457def _encode_target(target: str) -> str:

458 """Percent-encodes a request target so that there are no invalid characters

459

460 Pre-condition for this function is that 'target' must start with '/'.

461 If that is the case then _TARGET_RE will always produce a match.

462 """

463 match = _TARGET_RE.match(target)

464 if not match: # Defensive:

465 raise LocationParseError(f"{target!r} is not a valid request URI")

466

467 path, query = match.groups()

468 encoded_target = _encode_invalid_chars(path, _PATH_CHARS)

469 if query is not None:

470 query = _encode_invalid_chars(query, _QUERY_CHARS)

471 encoded_target += "?" + query

472 return encoded_target

473

474

475def parse_url(url: str) -> Url:

476 """

477 Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is

478 performed to parse incomplete urls. Fields not provided will be None.

479 This parser is RFC 3986 and RFC 6874 compliant.

480

481 The parser logic and helper functions are based heavily on

482 work done in the ``rfc3986`` module.

483

484 :param str url: URL to parse into a :class:`.Url` namedtuple.

485

486 Partly backwards-compatible with :mod:`urllib.parse`.

487

488 Example:

489

490 .. code-block:: python

491

492 import urllib3

493

494 print( urllib3.util.parse_url('http://google.com/mail/'))

495 # Url(scheme='http', host='google.com', port=None, path='/mail/', ...)

496

497 print( urllib3.util.parse_url('google.com:80'))

498 # Url(scheme=None, host='google.com', port=80, path=None, ...)

499

500 print( urllib3.util.parse_url('/foo?bar'))

501 # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)

502 """

503 if not url:

504 # Empty

505 return Url()

506

507 if not _SCHEME_RE.search(url):

508 url = "//" + url

509

510 scheme: str | None

511 authority: str | None

512 auth: str | None

513 host: str | None

514 port: str | None

515 port_int: int | None

516 path: str | None

517 query: str | None

518 fragment: str | None

519

520 scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]

521 normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES

522

523 if scheme:

524 scheme = scheme.lower()

525

526 if authority:

527 auth, _, host_port = authority.rpartition("@")

528 auth = auth or None

529 host_port_match = _HOST_PORT_RE.fullmatch(host_port)

530 if not host_port_match:

531 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host_port)

532 if invalid_host_char:

533 raise LocationParseError(

534 f"Host {host_port!r} contains invalid character "

535 f"{invalid_host_char.group()!r}"

536 )

537 raise LocationParseError(f"{host_port!r} is not a valid host or port")

538 host, port = host_port_match.groups()

539 if auth and normalize_uri:

540 auth = _encode_invalid_chars(auth, _USERINFO_CHARS)

541 if port == "":

542 port = None

543 else:

544 auth, host, port = None, None, None

545

546 if port is not None:

547 port_int = int(port)

548 if not (0 <= port_int <= 65535):

549 raise LocationParseError(url)

550 else:

551 port_int = None

552

553 host = _normalize_host(host, scheme)

554

555 if normalize_uri and path:

556 path = _remove_path_dot_segments(path)

557 path = _encode_invalid_chars(path, _PATH_CHARS)

558 if normalize_uri and query:

559 query = _encode_invalid_chars(query, _QUERY_CHARS)

560 if normalize_uri and fragment:

561 fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)

562

563 # For the sake of backwards compatibility we put empty

564 # string values for path if there are any defined values

565 # beyond the path in the URL.

566 # TODO: Remove this when we break backwards compatibility.

567 if not path:

568 if query is not None or fragment is not None:

569 path = ""

570 else:

571 path = None

572

573 return Url(

574 scheme=scheme,

575 auth=auth,

576 host=host,

577 port=port_int,

578 path=path,

579 query=query,

580 fragment=fragment,

581 )