Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/url.py: 89%

1from __future__ import annotations

3import re

4import typing

5from functools import partial

6from urllib.parse import unquote as _unquote

8from ..exceptions import LocationParseError

9from .util import to_str

11# We only want to normalize urls with an HTTP(S) scheme.

12# urllib3 infers URLs without a scheme (None) to be http.

13_NORMALIZABLE_SCHEMES = ("http", "https", None)

15# Almost all of these patterns were derived from the

16# 'rfc3986' module: https://github.com/python-hyper/rfc3986

17_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")

18_HOST_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}|%")

19_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")

20_HOST_INVALID_CHAR_RE = re.compile(r"[\x00-\x20\x7f]")

21_URI_RE = re.compile(

22 r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"

23 r"(?://([^\\/?#]*))?"

24 r"([^?#]*)"

25 r"(?:\?([^#]*))?"

26 r"(?:#(.*))?$",

27 re.UNICODE | re.DOTALL,

28)

30_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"

31_HEX_PAT = "[0-9A-Fa-f]{1,4}"

32_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)

33_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}

34_variations = [

35 # 6( h16 ":" ) ls32

36 "(?:%(hex)s:){6}%(ls32)s",

37 # "::" 5( h16 ":" ) ls32

38 "::(?:%(hex)s:){5}%(ls32)s",

39 # [ h16 ] "::" 4( h16 ":" ) ls32

40 "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",

41 # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32

42 "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",

43 # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32

44 "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",

45 # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32

46 "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",

47 # [ *4( h16 ":" ) h16 ] "::" ls32

48 "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",

49 # [ *5( h16 ":" ) h16 ] "::" h16

50 "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",

51 # [ *6( h16 ":" ) h16 ] "::"

52 "(?:(?:%(hex)s:){0,6}%(hex)s)?::",

53]

55_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"

56_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"

57_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"

58_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"

59_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"

60_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")

62_IPV4_RE = re.compile(

63 r"^(?:0[xX][0-9a-fA-F]+|[0-9]+)(?:\.(?:0[xX][0-9a-fA-F]+|[0-9]+)){0,3}$"

64)

65_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")

66_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")

67_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")

68_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")

70_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (

71 _REG_NAME_PAT,

72 _IPV4_PAT,

73 _IPV6_ADDRZ_PAT,

74)

75_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)

77_UNRESERVED_CHARS = set(

78 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"

79)

80_SUB_DELIM_CHARS = set("!$&'()*+,;=")

81_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}

82_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}

83_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}

86class Url(

87 typing.NamedTuple(

88 "Url",

89 [

90 ("scheme", typing.Optional[str]),

91 ("auth", typing.Optional[str]),

92 ("host", typing.Optional[str]),

93 ("port", typing.Optional[int]),

94 ("path", typing.Optional[str]),

95 ("query", typing.Optional[str]),

96 ("fragment", typing.Optional[str]),

97 ],

98 )

99):

100 """

101 Data structure for representing an HTTP URL. Used as a return value for

102 :func:`parse_url`. Both the scheme and host are normalized as they are

103 both case-insensitive according to RFC 3986.

104

105 :param auth: User information as defined in RFC 3986 3.2.1. This

106 component is kept percent-encoded. Use :attr:`auth_decoded` or

107 :attr:`auth_decoded_joined` to get the decoded form.

108 """

109

110 def __new__( # type: ignore[no-untyped-def]

111 cls,

112 scheme: str | None = None,

113 auth: str | None = None,

114 host: str | None = None,

115 port: int | None = None,

116 path: str | None = None,

117 query: str | None = None,

118 fragment: str | None = None,

119 ):

120 if path and not path.startswith("/"):

121 path = "/" + path

122 if scheme is not None:

123 scheme = scheme.lower()

124 return super().__new__(cls, scheme, auth, host, port, path, query, fragment)

125

126 @property

127 def auth_decoded(self) -> tuple[None, None] | tuple[str, str | None]:

128 """

129 User information with %-escapes decoded as UTF-8, returned as a

130 ``(username, password)`` tuple.

131

132 Both values are ``None`` if ``auth`` is ``None``.

133 ``password`` is ``None`` if not present in the auth component.

134 """

135 if self.auth is None:

136 return None, None

137 username, sep, password = self.auth.partition(":")

138 return (

139 _unquote(username, encoding="utf-8"),

140 _unquote(password, encoding="utf-8") if sep else None,

141 )

142

143 @property

144 def auth_decoded_joined(self) -> str | None:

145 """

146 User information with %-escapes decoded as UTF-8, as a string

147 prepared for encoding into an 'authorization: basic ...' header.

148

149 This property does not choose the encoding used for an

150 'authorization: basic ...' header. Use the ``basic_auth_encoding`` or

151 ``proxy_basic_auth_encoding`` parameters of

152 :func:`urllib3.util.make_headers` when converting the returned string

153 into Basic authentication bytes.

154

155 This is a convenience property that joins the username and

156 password with a colon, if both are present.

157 If only the username is present, a trailing colon is still

158 appended.

159 If ``auth`` is ``None``, this returns ``None``.

160 """

161 username, password = self.auth_decoded

162 if username is None:

163 return None

164 return f"{username}:{password or ''}"

165

166 @property

167 def hostname(self) -> str | None:

168 """For backwards-compatibility with urlparse. We're nice like that."""

169 return self.host

170

171 @property

172 def request_uri(self) -> str:

173 """Absolute path including the query string."""

174 uri = self.path or "/"

175

176 if self.query is not None:

177 uri += "?" + self.query

178

179 return uri

180

181 @property

182 def authority(self) -> str | None:

183 """

184 Authority component as defined in RFC 3986 3.2.

185 This includes userinfo (auth), host and port.

186

187 i.e.

188 userinfo@host:port

189 """

190 userinfo = self.auth

191 netloc = self.netloc

192 if netloc is None or userinfo is None:

193 return netloc

194 else:

195 return f"{userinfo}@{netloc}"

196

197 @property

198 def netloc(self) -> str | None:

199 """

200 Network location including host and port.

201

202 If you need the equivalent of urllib.parse's ``netloc``,

203 use the ``authority`` property instead.

204 """

205 if self.host is None:

206 return None

207 if self.port is not None:

208 return f"{self.host}:{self.port}"

209 return self.host

210

211 @property

212 def url(self) -> str:

213 """

214 Convert self into a url

215

216 This function should more or less round-trip with :func:`.parse_url`. The

217 returned url may not be exactly the same as the url inputted to

218 :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls

219 with a blank port will have : removed).

220

221 Example:

222

223 .. code-block:: python

224

225 import urllib3

226

227 U = urllib3.util.parse_url("https://google.com/mail/")

228

229 print(U.url)

230 # "https://google.com/mail/"

231

232 print( urllib3.util.Url("https", "username:password",

233 "host.com", 80, "/path", "query", "fragment"

234 ).url

235 )

236 # "https://username:password@host.com:80/path?query#fragment"

237 """

238 scheme, auth, host, port, path, query, fragment = self

239 url = ""

240

241 # We use "is not None" we want things to happen with empty strings (or 0 port)

242 if scheme is not None:

243 url += scheme + "://"

244 if auth is not None:

245 url += auth + "@"

246 if host is not None:

247 url += host

248 if port is not None:

249 url += ":" + str(port)

250 if path is not None:

251 url += path

252 if query is not None:

253 url += "?" + query

254 if fragment is not None:

255 url += "#" + fragment

256

257 return url

258

259 def __str__(self) -> str:

260 return self.url

261

262

263@typing.overload

264def _encode_invalid_chars(

265 component: str, allowed_chars: typing.Container[str]

266) -> str: # Abstract

267 ...

268

269

270@typing.overload

271def _encode_invalid_chars(

272 component: None, allowed_chars: typing.Container[str]

273) -> None: # Abstract

274 ...

275

276

277def _encode_invalid_chars(

278 component: str | None, allowed_chars: typing.Container[str]

279) -> str | None:

280 """Percent-encodes a URI component without reapplying

281 onto an already percent-encoded component.

282 """

283 if component is None:

284 return component

285

286 component = to_str(component)

287

288 # Normalize existing percent-encoded bytes.

289 # Try to see if the component we're encoding is already percent-encoded

290 # so we can skip all '%' characters but still encode all others.

291 component, percent_encodings = _PERCENT_RE.subn(

292 lambda match: match.group(0).upper(), component

293 )

294

295 uri_bytes = component.encode("utf-8", "surrogatepass")

296 is_percent_encoded = percent_encodings == uri_bytes.count(b"%")

297 encoded_component = bytearray()

298

299 for i in range(0, len(uri_bytes)):

300 # Will return a single character bytestring

301 byte = uri_bytes[i : i + 1]

302 byte_ord = ord(byte)

303 if (is_percent_encoded and byte == b"%") or (

304 byte_ord < 128 and byte.decode() in allowed_chars

305 ):

306 encoded_component += byte

307 continue

308 encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper()))

309

310 return encoded_component.decode()

311

312

313def _remove_path_dot_segments(path: str) -> str:

314 # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code

315 segments = path.split("/") # Turn the path into a list of segments

316 output = [] # Initialize the variable to use to store output

317

318 for segment in segments:

319 # '.' is the current directory, so ignore it, it is superfluous

320 if segment == ".":

321 continue

322 # Anything other than '..', should be appended to the output

323 if segment != "..":

324 output.append(segment)

325 # In this case segment == '..', if we can, we should pop the last

326 # element

327 elif output:

328 output.pop()

329

330 # If the path starts with '/' and the output is empty or the first string

331 # is non-empty

332 if path.startswith("/") and (not output or output[0]):

333 output.insert(0, "")

334

335 # If the path starts with '/.' or '/..' ensure we add one more empty

336 # string to add a trailing '/'

337 if path.endswith(("/.", "/..")):

338 output.append("")

339

340 return "/".join(output)

341

342

343@typing.overload

344def _normalize_host(host: None, scheme: str | None) -> None: ...

345

346

347@typing.overload

348def _normalize_host(host: str, scheme: str | None) -> str: ...

349

350

351def _normalize_host(host: str | None, scheme: str | None) -> str | None:

352 if host:

353 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host)

354 if invalid_host_char:

355 raise LocationParseError(

356 f"Host {host!r} contains invalid character "

357 f"{invalid_host_char.group()!r}"

358 )

359 if scheme in _NORMALIZABLE_SCHEMES:

360 is_ipv6 = _IPV6_ADDRZ_RE.match(host)

361 if is_ipv6:

362 # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as

363 # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID

364 # separator as necessary to return a valid RFC 4007 scoped IP.

365 match = _ZONE_ID_RE.search(host)

366 if match:

367 start, end = match.span(1)

368 zone_id = host[start:end]

369

370 if zone_id.startswith("%25") and zone_id != "%25":

371 zone_id = zone_id[3:]

372 else:

373 zone_id = zone_id[1:]

374 zone_id = _PERCENT_RE.sub(

375 partial(_normalize_zone_id_percent_encoding, error_host=host),

376 zone_id,

377 )

378 zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)

379 return f"{host[:start].lower()}%{zone_id}{host[end:]}"

380 else:

381 return host.lower()

382 elif not _IPV4_RE.match(host):

383 if "%" in host:

384 host = _HOST_PERCENT_RE.sub(_normalize_host_percent_encoding, host)

385 return to_str(

386 b".".join([_idna_encode(label) for label in host.split(".")]),

387 "ascii",

388 )

389 return host

390

391

392def _decode_percent_encoding(match: re.Match[str], *, error_host: str) -> str:

393 percent_encoded_octet = match.group(0)

394 decoded_octet = chr(int(percent_encoded_octet[1:], 16))

395 # Keep this narrower than _HOST_INVALID_CHAR_RE because "%20" is valid

396 # percent-encoding and remains encoded.

397 if decoded_octet < "\x20" or decoded_octet == "\x7f":

398 raise LocationParseError(

399 f"Host {error_host!r} contains invalid percent-encoded "

400 f"control character {percent_encoded_octet!r}"

401 )

402 return decoded_octet

403

404

405def _normalize_host_percent_encoding(match: re.Match[str]) -> str:

406 # Reject invalid percent encodings

407 if match.group(0) == "%":

408 raise LocationParseError(f"{match.string!r} is not a valid host")

409 decoded_octet = _decode_percent_encoding(match, error_host=match.string)

410 if decoded_octet in _UNRESERVED_CHARS:

411 return decoded_octet

412 return match.group(0).upper()

413

414

415def _normalize_zone_id_percent_encoding(

416 match: re.Match[str], *, error_host: str

417) -> str:

418 _decode_percent_encoding(match, error_host=error_host)

419 return match.group(0).upper()

420

421

422def _idna_encode(name: str) -> bytes:

423 if not name.isascii():

424 try:

425 import idna

426 except ImportError:

427 raise LocationParseError(

428 "Unable to parse URL without the 'idna' module"

429 ) from None

430

431 try:

432 return idna.encode(name.lower(), strict=True, std3_rules=True)

433 except idna.IDNAError:

434 raise LocationParseError(

435 f"Name '{name}' is not a valid IDNA label"

436 ) from None

437

438 return _PERCENT_RE.sub(lambda match: match.group(0).upper(), name.lower()).encode(

439 "ascii"

440 )

441

442

443def _encode_target(target: str) -> str:

444 """Percent-encodes a request target so that there are no invalid characters

445

446 Pre-condition for this function is that 'target' must start with '/'.

447 If that is the case then _TARGET_RE will always produce a match.

448 """

449 match = _TARGET_RE.match(target)

450 if not match: # Defensive:

451 raise LocationParseError(f"{target!r} is not a valid request URI")

452

453 path, query = match.groups()

454 encoded_target = _encode_invalid_chars(path, _PATH_CHARS)

455 if query is not None:

456 query = _encode_invalid_chars(query, _QUERY_CHARS)

457 encoded_target += "?" + query

458 return encoded_target

459

460

461def parse_url(url: str) -> Url:

462 """

463 Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is

464 performed to parse incomplete urls. Fields not provided will be None.

465 This parser is RFC 3986 and RFC 6874 compliant.

466

467 The parser logic and helper functions are based heavily on

468 work done in the ``rfc3986`` module.

469

470 :param str url: URL to parse into a :class:`.Url` namedtuple.

471

472 Partly backwards-compatible with :mod:`urllib.parse`.

473

474 Example:

475

476 .. code-block:: python

477

478 import urllib3

479

480 print( urllib3.util.parse_url('http://google.com/mail/'))

481 # Url(scheme='http', host='google.com', port=None, path='/mail/', ...)

482

483 print( urllib3.util.parse_url('google.com:80'))

484 # Url(scheme=None, host='google.com', port=80, path=None, ...)

485

486 print( urllib3.util.parse_url('/foo?bar'))

487 # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)

488 """

489 if not url:

490 # Empty

491 return Url()

492

493 if not _SCHEME_RE.search(url):

494 url = "//" + url

495

496 scheme: str | None

497 authority: str | None

498 auth: str | None

499 host: str | None

500 port: str | None

501 port_int: int | None

502 path: str | None

503 query: str | None

504 fragment: str | None

505

506 scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]

507 normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES

508

509 if scheme:

510 scheme = scheme.lower()

511

512 if authority:

513 auth, _, host_port = authority.rpartition("@")

514 auth = auth or None

515 host_port_match = _HOST_PORT_RE.fullmatch(host_port)

516 if not host_port_match:

517 invalid_host_char = _HOST_INVALID_CHAR_RE.search(host_port)

518 if invalid_host_char:

519 raise LocationParseError(

520 f"Host {host_port!r} contains invalid character "

521 f"{invalid_host_char.group()!r}"

522 )

523 raise LocationParseError(f"{host_port!r} is not a valid host or port")

524 host, port = host_port_match.groups()

525 if auth and normalize_uri:

526 auth = _encode_invalid_chars(auth, _USERINFO_CHARS)

527 if port == "":

528 port = None

529 else:

530 auth, host, port = None, None, None

531

532 if port is not None:

533 port_int = int(port)

534 if not (0 <= port_int <= 65535):

535 raise LocationParseError(url)

536 else:

537 port_int = None

538

539 host = _normalize_host(host, scheme)

540

541 if normalize_uri and path:

542 path = _remove_path_dot_segments(path)

543 path = _encode_invalid_chars(path, _PATH_CHARS)

544 if normalize_uri and query:

545 query = _encode_invalid_chars(query, _QUERY_CHARS)

546 if normalize_uri and fragment:

547 fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)

548

549 # For the sake of backwards compatibility we put empty

550 # string values for path if there are any defined values

551 # beyond the path in the URL.

552 # TODO: Remove this when we break backwards compatibility.

553 if not path:

554 if query is not None or fragment is not None:

555 path = ""

556 else:

557 path = None

558

559 return Url(

560 scheme=scheme,

561 auth=auth,

562 host=host,

563 port=port_int,

564 path=path,

565 query=query,

566 fragment=fragment,

567 )