Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/url.py: 89%

1from __future__ import annotations

3import re

4import typing

5from functools import partial

6from urllib.parse import unquote as _unquote

8from ..exceptions import LocationParseError

9from .util import to_str

11# We only want to normalize urls with an HTTP(S) scheme.

12# urllib3 infers URLs without a scheme (None) to be http.

13_NORMALIZABLE_SCHEMES = ("http", "https", None)

15# Almost all of these patterns were derived from the

16# 'rfc3986' module: https://github.com/python-hyper/rfc3986

17_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")

18_HOST_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}|%")

19_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")

20_HOST_INVALID_CHAR_RE = re.compile(r"[\x00-\x20\x7f]")

21_URI_RE = re.compile(

22 r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"

23 r"(?://([^\\/?#]*))?"

24 r"([^?#]*)"

25 r"(?:\?([^#]*))?"

26 r"(?:#(.*))?$",

27 re.UNICODE | re.DOTALL,

28)

30_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"

31_HEX_PAT = "[0-9A-Fa-f]{1,4}"

32_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)

33_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}

34_variations = [

35 # 6( h16 ":" ) ls32

36 "(?:%(hex)s:){6}%(ls32)s",

37 # "::" 5( h16 ":" ) ls32

38 "::(?:%(hex)s:){5}%(ls32)s",

39 # [ h16 ] "::" 4( h16 ":" ) ls32

40 "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",

41 # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32

42 "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",

43 # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32

44 "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",

45 # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32

46 "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",

47 # [ *4( h16 ":" ) h16 ] "::" ls32

48 "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",

49 # [ *5( h16 ":" ) h16 ] "::" h16

50 "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",

51 # [ *6( h16 ":" ) h16 ] "::"

52 "(?:(?:%(hex)s:){0,6}%(hex)s)?::",

53]

55_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"

56_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"

57_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"

58_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"

59_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"

60_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")

62_IPV4_RE = re.compile(

63 r"^(?:0[xX][0-9a-fA-F]+|[0-9]+)(?:\.(?:0[xX][0-9a-fA-F]+|[0-9]+)){0,3}$"

64)

65_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")

66_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")

67_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")

68_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")

70_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (

71 _REG_NAME_PAT,

72 _IPV4_PAT,

73 _IPV6_ADDRZ_PAT,

74)

75_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)

77_UNRESERVED_CHARS = set(

78 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"

79)

80_SUB_DELIM_CHARS = set("!$&'()*+,;=")

81_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}

82_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}

83_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}

86class Url(

87 typing.NamedTuple(

88 "Url",

89 [

90 ("scheme", typing.Optional[str]),

91 ("auth", typing.Optional[str]),

92 ("host", typing.Optional[str]),

93 ("port", typing.Optional[int]),

94 ("path", typing.Optional[str]),

95 ("query", typing.Optional[str]),

96 ("fragment", typing.Optional[str]),

97 ],

98 )

99):

100 """

101 Data structure for representing an HTTP URL. Used as a return value for

102 :func:`parse_url`. Both the scheme and host are normalized as they are

103 both case-insensitive according to RFC 3986.

104

105 :param auth: User information as defined in RFC 3986 3.2.1. This

106 component is kept percent-encoded. Use :attr:`auth_decoded` or

107 :attr:`auth_decoded_joined` to get the decoded form.

108 """

109

110 def __new__( # type: ignore[no-untyped-def]

111 cls,

112 scheme: str | None = None,

113 auth: str | None = None,

114 host: str | None = None,

115 port: int | None = None,

116 path: str | None = None,

117 query: str | None = None,

118 fragment: str | None = None,

119 ):

120 if path and not path.startswith("/"):

121 path = "/" + path

122 if scheme is not None:

123 scheme = scheme.lower()

124 return super().__new__(cls, scheme, auth, host, port, path, query, fragment)

125

126 @property

127 def auth_decoded(self) -> tuple[None, None] | tuple[str, str | None]:

128 """

129 User information with %-escapes decoded as UTF-8, returned as a

130 ``(username, password)`` tuple.

131

132 Both values are ``None`` if ``auth`` is ``None``.

133 ``password`` is ``None`` if not present in the auth component.

134 """

135 if self.auth is None:

136 return None, None

137 username, sep, password = self.auth.partition(":")

138 return (

139 _unquote(username, encoding="utf-8"),

140 _unquote(password, encoding="utf-8") if sep else None,

141 )

142

143 @property

144 def auth_decoded_joined(self) -> str | None:

145 """

146 User information with %-escapes decoded as UTF-8, as a string

147 prepared for encoding into an 'authorization: basic ...' header.

148

149 This property does not choose the encoding used for an

150 'authorization: basic ...' header. Use the ``basic_auth_encoding`` or

151 ``proxy_basic_auth_encoding`` parameters of

152 :func:`urllib3.util.make_headers` when converting the returned string

153 into Basic authentication bytes.

154

155 This is a convenience property that joins the username and

156 password with a colon, if both are present.

157 If only the username is present, a trailing colon is still

158 appended.

159 If ``auth`` is ``None``, this returns ``None``.

160 """

161 username, password = self.auth_decoded

162 if username is None:

163 return None

164 return f"{username}:{password or ''}"

165

166 @property

167 def hostname(self) -> str | None:

168 """For backwards-compatibility with urlparse. We're nice like that."""

169 return self.host

170

171 @property

172 def request_uri(self) -> str:

173 """Absolute path including the query string."""

174 uri = self.path or "/"

175

176 if self.query is not None:

177 uri += "?" + self.query

178

179 return uri

180

181 @property

182 def authority(self) -> str | None:

183 """

184 Authority component as defined in RFC 3986 3.2.

185 This includes userinfo (auth), host and port.

186

187 i.e.

188 userinfo@host:port

189 """

190 userinfo = self.auth

191 netloc = self.netloc

192 if netloc is None or userinfo is None:

193 return netloc

194 else:

195 return f"{userinfo}@{netloc}"

196

197 @property

198 def netloc(self) -> str | None:

199 """

200 Network location including host and port.

201

202 If you need the equivalent of urllib.parse's ``netloc``,

203 use the ``authority`` property instead.

204 """

205 if self.host is None:

206 return None

207 if self.port is not None:

208 return f"{self.host}:{self.port}"

209 return self.host

210

211 @property

212 def url(self) -> str:

213 """

214 Convert self into a url

215

216 This function should more or less round-trip with :func:`.parse_url`. The

217 returned url may not be exactly the same as the url inputted to

218 :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls

219 with a blank port will have : removed).

220

221 Example:

222

223 .. code-block:: python

224

225 import urllib3

226

227 U = urllib3.util.parse_url("https://google.com/mail/")

228

229 print(U.url)

230 # "https://google.com/mail/"

231

232 print( urllib3.util.Url("https", "username:password",

233 "host.com", 80, "/path", "query", "fragment"

234 ).url

235 )

236 # "https://username:password@host.com:80/path?query#fragment"

237 """

238 scheme, auth, host, port, path, query, fragment = self

239 url = ""

240

241 # We use "is not None" we want things to happen with empty strings (or 0 port)

242 if scheme is not None:

243 url += scheme + "://"

244 if auth is not None:

245 url += auth + "@"

246 if host is not None:

247 url += host

248 if port is not None:

249 url += ":" + str(port)

250 if path is not None:

251 url += path

252 if query is not None:

253 url += "?" + query

254 if fragment is not None:

255 url += "#" + fragment

256

257 return url

258

259 def __str__(self) -> str:

260 return self.url

261

262

263@typing.overload

264def _encode_invalid_chars(

265 component: str, allowed_chars: typing.Container[str]

266) -> str: # Abstract

267 ...

268

269

270@typing.overload

271def _encode_invalid_chars(

272 component: None, allowed_chars: typing.Container[str]

273) -> None: # Abstract

274 ...

275

276

277def _encode_invalid_chars(

278 component: str | None, allowed_chars: typing.Container[str]

279) -> str | None:

280 """Percent-encodes a URI component without reapplying

281 onto an already percent-encoded component.

282 """

283 if component is None:

284 return component

285

286 component = to_str(component)

287

288 # Fast path: an ASCII component that consists only of allowed characters

289 # and contains no '%' needs neither encoding nor percent-normalization.

290 # Components containing '%' always take the general path below because

291 # how '%' is treated depends on whether the whole component is already

292 # percent-encoded.

293 if (

294 "%" not in component

295 and component.isascii()

296 and all(map(allowed_chars.__contains__, component))

297 ):

298 return component

299

300 # Normalize existing percent-encoded bytes.

301 # Try to see if the component we're encoding is already percent-encoded

302 # so we can skip all '%' characters but still encode all others.

303 component, percent_encodings = _PERCENT_RE.subn(

304 lambda match: match.group(0).upper(), component

305 )

306

307 uri_bytes = component.encode("utf-8", "surrogatepass")

308 is_percent_encoded = percent_encodings == uri_bytes.count(b"%")

309 encoded_component = bytearray()

310

311 # Iterating over bytes yields integers; 0x25 is '%'.

312 for byte_ord in uri_bytes:

313 if (is_percent_encoded and byte_ord == 0x25) or (

314 byte_ord < 128 and chr(byte_ord) in allowed_chars

315 ):

316 encoded_component.append(byte_ord)

317 else:

318 encoded_component += b"%%%02X" % byte_ord

319

320 return encoded_component.decode()

321

322

323def _remove_path_dot_segments(path: str) -> str:

324 # Fast path: without a '.' or '..' segment there is nothing to remove.

325 if "/." not in path and not path.startswith("."):

326 return path

327

328 # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code

329 segments = path.split("/") # Turn the path into a list of segments

330 output = [] # Initialize the variable to use to store output

331

332 for segment in segments:

333 # '.' is the current directory, so ignore it, it is superfluous

334 if segment == ".":

335 continue

336 # Anything other than '..', should be appended to the output

337 if segment != "..":

338 output.append(segment)

339 # In this case segment == '..', if we can, we should pop the last

340 # element

341 elif output:

342 output.pop()

343

344 # If the path starts with '/' and the output is empty or the first string

345 # is non-empty

346 if path.startswith("/") and (not output or output[0]):

347 output.insert(0, "")

348

349 # If the path starts with '/.' or '/..' ensure we add one more empty

350 # string to add a trailing '/'

351 if path.endswith(("/.", "/..")):

352 output.append("")

353

354 return "/".join(output)

355

356

357@typing.overload

358def _normalize_host(host: None, scheme: str | None) -> None: ...

359

360

361@typing.overload

362def _normalize_host(host: str, scheme: str | None) -> str: ...

363

364

365def _normalize_host(host: str | None, scheme: str | None) -> str | None:

366 if host:

367 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host)

368 if invalid_host_char:

369 raise LocationParseError(

370 f"Host {host!r} contains invalid character "

371 f"{invalid_host_char.group()!r}"

372 )

373 if scheme in _NORMALIZABLE_SCHEMES:

374 is_ipv6 = _IPV6_ADDRZ_RE.match(host)

375 if is_ipv6:

376 # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as

377 # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID

378 # separator as necessary to return a valid RFC 4007 scoped IP.

379 match = _ZONE_ID_RE.search(host)

380 if match:

381 start, end = match.span(1)

382 zone_id = host[start:end]

383

384 if zone_id.startswith("%25") and zone_id != "%25":

385 zone_id = zone_id[3:]

386 else:

387 zone_id = zone_id[1:]

388 zone_id = _PERCENT_RE.sub(

389 partial(_normalize_zone_id_percent_encoding, error_host=host),

390 zone_id,

391 )

392 zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)

393 return f"{host[:start].lower()}%{zone_id}{host[end:]}"

394 else:

395 return host.lower()

396 elif not _IPV4_RE.match(host):

397 # Fast path: for an ASCII host without percent-encoding,

398 # IDNA-encoding each label reduces to lower-casing it.

399 if "%" not in host and host.isascii():

400 return host.lower()

401 if "%" in host:

402 address, separator, zone = host.partition("%")

403 if ":" in address and host.isascii() and _IPV6_RE.match(address):

404 # Native IPv6 zone names may be case-sensitive.

405 return f"{address.lower()}{separator}{zone}"

406 host = _HOST_PERCENT_RE.sub(_normalize_host_percent_encoding, host)

407 return to_str(

408 b".".join([_idna_encode(label) for label in host.split(".")]),

409 "ascii",

410 )

411 return host

412

413

414def _decode_percent_encoding(match: re.Match[str], *, error_host: str) -> str:

415 percent_encoded_octet = match.group(0)

416 decoded_octet = chr(int(percent_encoded_octet[1:], 16))

417 # Keep this narrower than _HOST_INVALID_CHAR_RE because "%20" is valid

418 # percent-encoding and remains encoded.

419 if decoded_octet < "\x20" or decoded_octet == "\x7f":

420 raise LocationParseError(

421 f"Host {error_host!r} contains invalid percent-encoded "

422 f"control character {percent_encoded_octet!r}"

423 )

424 return decoded_octet

425

426

427def _normalize_host_percent_encoding(match: re.Match[str]) -> str:

428 # Reject invalid percent encodings

429 if match.group(0) == "%":

430 raise LocationParseError(f"{match.string!r} is not a valid host")

431 decoded_octet = _decode_percent_encoding(match, error_host=match.string)

432 if decoded_octet in _UNRESERVED_CHARS:

433 return decoded_octet

434 return match.group(0).upper()

435

436

437def _normalize_zone_id_percent_encoding(

438 match: re.Match[str], *, error_host: str

439) -> str:

440 _decode_percent_encoding(match, error_host=error_host)

441 return match.group(0).upper()

442

443

444def _idna_encode(name: str) -> bytes:

445 if not name.isascii():

446 try:

447 import idna

448 except ImportError:

449 raise LocationParseError(

450 "Unable to parse URL without the 'idna' module"

451 ) from None

452

453 try:

454 return idna.encode(name.lower(), strict=True, std3_rules=True)

455 except idna.IDNAError:

456 raise LocationParseError(

457 f"Name '{name}' is not a valid IDNA label"

458 ) from None

459

460 return _PERCENT_RE.sub(lambda match: match.group(0).upper(), name.lower()).encode(

461 "ascii"

462 )

463

464

465def _encode_target(target: str) -> str:

466 """Percent-encodes a request target so that there are no invalid characters

467

468 Pre-condition for this function is that 'target' must start with '/'.

469 If that is the case then _TARGET_RE will always produce a match.

470 """

471 match = _TARGET_RE.match(target)

472 if not match: # Defensive:

473 raise LocationParseError(f"{target!r} is not a valid request URI")

474

475 path, query = match.groups()

476 encoded_target = _encode_invalid_chars(path, _PATH_CHARS)

477 if query is not None:

478 query = _encode_invalid_chars(query, _QUERY_CHARS)

479 encoded_target += "?" + query

480 return encoded_target

481

482

483def parse_url(url: str) -> Url:

484 """

485 Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is

486 performed to parse incomplete urls. Fields not provided will be None.

487 This parser is RFC 3986 and RFC 6874 compliant.

488

489 The parser logic and helper functions are based heavily on

490 work done in the ``rfc3986`` module.

491

492 :param str url: URL to parse into a :class:`.Url` namedtuple.

493

494 Partly backwards-compatible with :mod:`urllib.parse`.

495

496 Example:

497

498 .. code-block:: python

499

500 import urllib3

501

502 print( urllib3.util.parse_url('http://google.com/mail/'))

503 # Url(scheme='http', host='google.com', port=None, path='/mail/', ...)

504

505 print( urllib3.util.parse_url('google.com:80'))

506 # Url(scheme=None, host='google.com', port=80, path=None, ...)

507

508 print( urllib3.util.parse_url('/foo?bar'))

509 # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)

510 """

511 if not url:

512 # Empty

513 return Url()

514

515 if not _SCHEME_RE.search(url):

516 url = "//" + url

517

518 scheme: str | None

519 authority: str | None

520 auth: str | None

521 host: str | None

522 port: str | None

523 port_int: int | None

524 path: str | None

525 query: str | None

526 fragment: str | None

527

528 scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]

529 normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES

530

531 if scheme:

532 scheme = scheme.lower()

533

534 if authority:

535 auth, _, host_port = authority.rpartition("@")

536 auth = auth or None

537 host_port_match = _HOST_PORT_RE.fullmatch(host_port)

538 if not host_port_match:

539 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host_port)

540 if invalid_host_char:

541 raise LocationParseError(

542 f"Host {host_port!r} contains invalid character "

543 f"{invalid_host_char.group()!r}"

544 )

545 raise LocationParseError(f"{host_port!r} is not a valid host or port")

546 host, port = host_port_match.groups()

547 if auth and normalize_uri:

548 auth = _encode_invalid_chars(auth, _USERINFO_CHARS)

549 if port == "":

550 port = None

551 else:

552 auth, host, port = None, None, None

553

554 if port is not None:

555 port_int = int(port)

556 if not (0 <= port_int <= 65535):

557 raise LocationParseError(url)

558 else:

559 port_int = None

560

561 host = _normalize_host(host, scheme)

562

563 if normalize_uri and path:

564 path = _remove_path_dot_segments(path)

565 path = _encode_invalid_chars(path, _PATH_CHARS)

566 if normalize_uri and query:

567 query = _encode_invalid_chars(query, _QUERY_CHARS)

568 if normalize_uri and fragment:

569 fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)

570

571 # For the sake of backwards compatibility we put empty

572 # string values for path if there are any defined values

573 # beyond the path in the URL.

574 # TODO: Remove this when we break backwards compatibility.

575 if not path:

576 if query is not None or fragment is not None:

577 path = ""

578 else:

579 path = None

580

581 return Url(

582 scheme=scheme,

583 auth=auth,

584 host=host,

585 port=port_int,

586 path=path,

587 query=query,

588 fragment=fragment,

589 )