Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/url.py: 89%

1from __future__ import annotations

3import re

4import typing

5from functools import partial

6from urllib.parse import unquote as _unquote

8from ..exceptions import LocationParseError

9from .util import to_str

11# We only want to normalize urls with an HTTP(S) scheme.

12# urllib3 infers URLs without a scheme (None) to be http.

13_NORMALIZABLE_SCHEMES = ("http", "https", None)

15# Almost all of these patterns were derived from the

16# 'rfc3986' module: https://github.com/python-hyper/rfc3986

17_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")

18_HOST_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}|%")

19_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")

20_HOST_INVALID_CHAR_RE = re.compile(r"[\x00-\x20\x7f]")

21_URI_RE = re.compile(

22 r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"

23 r"(?://([^\\/?#]*))?"

24 r"([^?#]*)"

25 r"(?:\?([^#]*))?"

26 r"(?:#(.*))?$",

27 re.UNICODE | re.DOTALL,

28)

30_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"

31_HEX_PAT = "[0-9A-Fa-f]{1,4}"

32_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)

33_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}

34_variations = [

35 # 6( h16 ":" ) ls32

36 "(?:%(hex)s:){6}%(ls32)s",

37 # "::" 5( h16 ":" ) ls32

38 "::(?:%(hex)s:){5}%(ls32)s",

39 # [ h16 ] "::" 4( h16 ":" ) ls32

40 "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",

41 # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32

42 "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",

43 # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32

44 "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",

45 # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32

46 "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",

47 # [ *4( h16 ":" ) h16 ] "::" ls32

48 "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",

49 # [ *5( h16 ":" ) h16 ] "::" h16

50 "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",

51 # [ *6( h16 ":" ) h16 ] "::"

52 "(?:(?:%(hex)s:){0,6}%(hex)s)?::",

53]

55_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"

56_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"

57_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"

58_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"

59_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"

60_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")

62_IPV4_RE = re.compile(

63 r"^(?:0[xX][0-9a-fA-F]+|[0-9]+)(?:\.(?:0[xX][0-9a-fA-F]+|[0-9]+)){0,3}$"

64)

65_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")

66_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")

67_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")

68_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")

70_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (

71 _REG_NAME_PAT,

72 _IPV4_PAT,

73 _IPV6_ADDRZ_PAT,

74)

75_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)

77_UNRESERVED_CHARS = set(

78 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"

79)

80_SUB_DELIM_CHARS = set("!$&'()*+,;=")

81_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}

82_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}

83_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}

86class Url(

87 typing.NamedTuple(

88 "Url",

89 [

90 ("scheme", typing.Optional[str]),

91 ("auth", typing.Optional[str]),

92 ("host", typing.Optional[str]),

93 ("port", typing.Optional[int]),

94 ("path", typing.Optional[str]),

95 ("query", typing.Optional[str]),

96 ("fragment", typing.Optional[str]),

97 ],

98 )

99):

100 """

101 Data structure for representing an HTTP URL. Used as a return value for

102 :func:`parse_url`. Both the scheme and host are normalized as they are

103 both case-insensitive according to RFC 3986.

104

105 :param auth: User information as defined in RFC 3986 3.2.1. This

106 component is kept percent-encoded. Use :attr:`auth_decoded` or

107 :attr:`auth_decoded_joined` to get the decoded form.

108 """

109

110 def __new__( # type: ignore[no-untyped-def]

111 cls,

112 scheme: str | None = None,

113 auth: str | None = None,

114 host: str | None = None,

115 port: int | None = None,

116 path: str | None = None,

117 query: str | None = None,

118 fragment: str | None = None,

119 ):

120 if path and not path.startswith("/"):

121 path = "/" + path

122 if scheme is not None:

123 scheme = scheme.lower()

124 return super().__new__(cls, scheme, auth, host, port, path, query, fragment)

125

126 @property

127 def auth_decoded(self) -> tuple[None, None] | tuple[str, str | None]:

128 """

129 User information with %-escapes decoded as UTF-8, returned as a

130 ``(username, password)`` tuple.

131

132 Both values are ``None`` if ``auth`` is ``None``.

133 ``password`` is ``None`` if not present in the auth component.

134 """

135 if self.auth is None:

136 return None, None

137 username, sep, password = self.auth.partition(":")

138 return (

139 _unquote(username, encoding="utf-8"),

140 _unquote(password, encoding="utf-8") if sep else None,

141 )

142

143 @property

144 def auth_decoded_joined(self) -> str | None:

145 """

146 User information with %-escapes decoded as UTF-8, as a string

147 prepared for encoding into an 'authorization: basic ...' header.

148

149 This property does not choose the encoding used for an

150 'authorization: basic ...' header. Use the ``basic_auth_encoding`` or

151 ``proxy_basic_auth_encoding`` parameters of

152 :func:`urllib3.util.make_headers` when converting the returned string

153 into Basic authentication bytes.

154

155 This is a convenience property that joins the username and

156 password with a colon, if both are present.

157 If only the username is present, a trailing colon is still

158 appended.

159 If ``auth`` is ``None``, this returns ``None``.

160 """

161 username, password = self.auth_decoded

162 if username is None:

163 return None

164 return f"{username}:{password or ''}"

165

166 @property

167 def hostname(self) -> str | None:

168 """For backwards-compatibility with urlparse. We're nice like that."""

169 return self.host

170

171 @property

172 def request_uri(self) -> str:

173 """Absolute path including the query string."""

174 uri = self.path or "/"

175

176 if self.query is not None:

177 uri += "?" + self.query

178

179 return uri

180

181 @property

182 def authority(self) -> str | None:

183 """

184 Authority component as defined in RFC 3986 3.2.

185 This includes userinfo (auth), host and port.

186

187 i.e.

188 userinfo@host:port

189 """

190 userinfo = self.auth

191 netloc = self.netloc

192 if netloc is None or userinfo is None:

193 return netloc

194 else:

195 return f"{userinfo}@{netloc}"

196

197 @property

198 def netloc(self) -> str | None:

199 """

200 Network location including host and port.

201

202 If you need the equivalent of urllib.parse's ``netloc``,

203 use the ``authority`` property instead.

204 """

205 if self.host is None:

206 return None

207 if self.port is not None:

208 return f"{self.host}:{self.port}"

209 return self.host

210

211 @property

212 def url(self) -> str:

213 """

214 Convert self into a url

215

216 This function should more or less round-trip with :func:`.parse_url`. The

217 returned url may not be exactly the same as the url inputted to

218 :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls

219 with a blank port will have : removed).

220

221 Example:

222

223 .. code-block:: python

224

225 import urllib3

226

227 U = urllib3.util.parse_url("https://google.com/mail/")

228

229 print(U.url)

230 # "https://google.com/mail/"

231

232 print( urllib3.util.Url("https", "username:password",

233 "host.com", 80, "/path", "query", "fragment"

234 ).url

235 )

236 # "https://username:password@host.com:80/path?query#fragment"

237 """

238 scheme, auth, host, port, path, query, fragment = self

239 url = ""

240

241 # We use "is not None" we want things to happen with empty strings (or 0 port)

242 if scheme is not None:

243 url += scheme + "://"

244 if auth is not None:

245 url += auth + "@"

246 if host is not None:

247 url += host

248 if port is not None:

249 url += ":" + str(port)

250 if path is not None:

251 url += path

252 if query is not None:

253 url += "?" + query

254 if fragment is not None:

255 url += "#" + fragment

256

257 return url

258

259 def __str__(self) -> str:

260 return self.url

261

262

263@typing.overload

264def _encode_invalid_chars(

265 component: str, allowed_chars: typing.Container[str]

266) -> str: # Abstract

267 ...

268

269

270@typing.overload

271def _encode_invalid_chars(

272 component: None, allowed_chars: typing.Container[str]

273) -> None: # Abstract

274 ...

275

276

277def _encode_invalid_chars(

278 component: str | None, allowed_chars: typing.Container[str]

279) -> str | None:

280 """Percent-encodes a URI component without reapplying

281 onto an already percent-encoded component.

282 """

283 if component is None:

284 return component

285

286 component = to_str(component)

287

288 # Fast path: an ASCII component that consists only of allowed characters

289 # and contains no '%' needs neither encoding nor percent-normalization.

290 # Components containing '%' always take the general path below because

291 # how '%' is treated depends on whether the whole component is already

292 # percent-encoded.

293 if (

294 "%" not in component

295 and component.isascii()

296 and all(map(allowed_chars.__contains__, component))

297 ):

298 return component

299

300 # Normalize existing percent-encoded bytes.

301 # Try to see if the component we're encoding is already percent-encoded

302 # so we can skip all '%' characters but still encode all others.

303 component, percent_encodings = _PERCENT_RE.subn(

304 lambda match: match.group(0).upper(), component

305 )

306

307 uri_bytes = component.encode("utf-8", "surrogatepass")

308 is_percent_encoded = percent_encodings == uri_bytes.count(b"%")

309 encoded_component = bytearray()

310

311 # Iterating over bytes yields integers; 0x25 is '%'.

312 for byte_ord in uri_bytes:

313 if (is_percent_encoded and byte_ord == 0x25) or (

314 byte_ord < 128 and chr(byte_ord) in allowed_chars

315 ):

316 encoded_component.append(byte_ord)

317 else:

318 encoded_component += b"%%%02X" % byte_ord

319

320 return encoded_component.decode()

321

322

323def _remove_path_dot_segments(path: str) -> str:

324 # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code

325 segments = path.split("/") # Turn the path into a list of segments

326 output = [] # Initialize the variable to use to store output

327

328 for segment in segments:

329 # '.' is the current directory, so ignore it, it is superfluous

330 if segment == ".":

331 continue

332 # Anything other than '..', should be appended to the output

333 if segment != "..":

334 output.append(segment)

335 # In this case segment == '..', if we can, we should pop the last

336 # element

337 elif output:

338 output.pop()

339

340 # If the path starts with '/' and the output is empty or the first string

341 # is non-empty

342 if path.startswith("/") and (not output or output[0]):

343 output.insert(0, "")

344

345 # If the path starts with '/.' or '/..' ensure we add one more empty

346 # string to add a trailing '/'

347 if path.endswith(("/.", "/..")):

348 output.append("")

349

350 return "/".join(output)

351

352

353@typing.overload

354def _normalize_host(host: None, scheme: str | None) -> None: ...

355

356

357@typing.overload

358def _normalize_host(host: str, scheme: str | None) -> str: ...

359

360

361def _normalize_host(host: str | None, scheme: str | None) -> str | None:

362 if host:

363 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host)

364 if invalid_host_char:

365 raise LocationParseError(

366 f"Host {host!r} contains invalid character "

367 f"{invalid_host_char.group()!r}"

368 )

369 if scheme in _NORMALIZABLE_SCHEMES:

370 is_ipv6 = _IPV6_ADDRZ_RE.match(host)

371 if is_ipv6:

372 # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as

373 # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID

374 # separator as necessary to return a valid RFC 4007 scoped IP.

375 match = _ZONE_ID_RE.search(host)

376 if match:

377 start, end = match.span(1)

378 zone_id = host[start:end]

379

380 if zone_id.startswith("%25") and zone_id != "%25":

381 zone_id = zone_id[3:]

382 else:

383 zone_id = zone_id[1:]

384 zone_id = _PERCENT_RE.sub(

385 partial(_normalize_zone_id_percent_encoding, error_host=host),

386 zone_id,

387 )

388 zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)

389 return f"{host[:start].lower()}%{zone_id}{host[end:]}"

390 else:

391 return host.lower()

392 elif not _IPV4_RE.match(host):

393 if "%" in host:

394 host = _HOST_PERCENT_RE.sub(_normalize_host_percent_encoding, host)

395 return to_str(

396 b".".join([_idna_encode(label) for label in host.split(".")]),

397 "ascii",

398 )

399 return host

400

401

402def _decode_percent_encoding(match: re.Match[str], *, error_host: str) -> str:

403 percent_encoded_octet = match.group(0)

404 decoded_octet = chr(int(percent_encoded_octet[1:], 16))

405 # Keep this narrower than _HOST_INVALID_CHAR_RE because "%20" is valid

406 # percent-encoding and remains encoded.

407 if decoded_octet < "\x20" or decoded_octet == "\x7f":

408 raise LocationParseError(

409 f"Host {error_host!r} contains invalid percent-encoded "

410 f"control character {percent_encoded_octet!r}"

411 )

412 return decoded_octet

413

414

415def _normalize_host_percent_encoding(match: re.Match[str]) -> str:

416 # Reject invalid percent encodings

417 if match.group(0) == "%":

418 raise LocationParseError(f"{match.string!r} is not a valid host")

419 decoded_octet = _decode_percent_encoding(match, error_host=match.string)

420 if decoded_octet in _UNRESERVED_CHARS:

421 return decoded_octet

422 return match.group(0).upper()

423

424

425def _normalize_zone_id_percent_encoding(

426 match: re.Match[str], *, error_host: str

427) -> str:

428 _decode_percent_encoding(match, error_host=error_host)

429 return match.group(0).upper()

430

431

432def _idna_encode(name: str) -> bytes:

433 if not name.isascii():

434 try:

435 import idna

436 except ImportError:

437 raise LocationParseError(

438 "Unable to parse URL without the 'idna' module"

439 ) from None

440

441 try:

442 return idna.encode(name.lower(), strict=True, std3_rules=True)

443 except idna.IDNAError:

444 raise LocationParseError(

445 f"Name '{name}' is not a valid IDNA label"

446 ) from None

447

448 return _PERCENT_RE.sub(lambda match: match.group(0).upper(), name.lower()).encode(

449 "ascii"

450 )

451

452

453def _encode_target(target: str) -> str:

454 """Percent-encodes a request target so that there are no invalid characters

455

456 Pre-condition for this function is that 'target' must start with '/'.

457 If that is the case then _TARGET_RE will always produce a match.

458 """

459 match = _TARGET_RE.match(target)

460 if not match: # Defensive:

461 raise LocationParseError(f"{target!r} is not a valid request URI")

462

463 path, query = match.groups()

464 encoded_target = _encode_invalid_chars(path, _PATH_CHARS)

465 if query is not None:

466 query = _encode_invalid_chars(query, _QUERY_CHARS)

467 encoded_target += "?" + query

468 return encoded_target

469

470

471def parse_url(url: str) -> Url:

472 """

473 Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is

474 performed to parse incomplete urls. Fields not provided will be None.

475 This parser is RFC 3986 and RFC 6874 compliant.

476

477 The parser logic and helper functions are based heavily on

478 work done in the ``rfc3986`` module.

479

480 :param str url: URL to parse into a :class:`.Url` namedtuple.

481

482 Partly backwards-compatible with :mod:`urllib.parse`.

483

484 Example:

485

486 .. code-block:: python

487

488 import urllib3

489

490 print( urllib3.util.parse_url('http://google.com/mail/'))

491 # Url(scheme='http', host='google.com', port=None, path='/mail/', ...)

492

493 print( urllib3.util.parse_url('google.com:80'))

494 # Url(scheme=None, host='google.com', port=80, path=None, ...)

495

496 print( urllib3.util.parse_url('/foo?bar'))

497 # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)

498 """

499 if not url:

500 # Empty

501 return Url()

502

503 if not _SCHEME_RE.search(url):

504 url = "//" + url

505

506 scheme: str | None

507 authority: str | None

508 auth: str | None

509 host: str | None

510 port: str | None

511 port_int: int | None

512 path: str | None

513 query: str | None

514 fragment: str | None

515

516 scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]

517 normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES

518

519 if scheme:

520 scheme = scheme.lower()

521

522 if authority:

523 auth, _, host_port = authority.rpartition("@")

524 auth = auth or None

525 host_port_match = _HOST_PORT_RE.fullmatch(host_port)

526 if not host_port_match:

527 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host_port)

528 if invalid_host_char:

529 raise LocationParseError(

530 f"Host {host_port!r} contains invalid character "

531 f"{invalid_host_char.group()!r}"

532 )

533 raise LocationParseError(f"{host_port!r} is not a valid host or port")

534 host, port = host_port_match.groups()

535 if auth and normalize_uri:

536 auth = _encode_invalid_chars(auth, _USERINFO_CHARS)

537 if port == "":

538 port = None

539 else:

540 auth, host, port = None, None, None

541

542 if port is not None:

543 port_int = int(port)

544 if not (0 <= port_int <= 65535):

545 raise LocationParseError(url)

546 else:

547 port_int = None

548

549 host = _normalize_host(host, scheme)

550

551 if normalize_uri and path:

552 path = _remove_path_dot_segments(path)

553 path = _encode_invalid_chars(path, _PATH_CHARS)

554 if normalize_uri and query:

555 query = _encode_invalid_chars(query, _QUERY_CHARS)

556 if normalize_uri and fragment:

557 fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)

558

559 # For the sake of backwards compatibility we put empty

560 # string values for path if there are any defined values

561 # beyond the path in the URL.

562 # TODO: Remove this when we break backwards compatibility.

563 if not path:

564 if query is not None or fragment is not None:

565 path = ""

566 else:

567 path = None

568

569 return Url(

570 scheme=scheme,

571 auth=auth,

572 host=host,

573 port=port_int,

574 path=path,

575 query=query,

576 fragment=fragment,

577 )