Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/connection.py: 24%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import datetime
4import http.client
5import logging
6import os
7import re
8import socket
9import sys
10import threading
11import typing
12import warnings
13from http.client import HTTPConnection as _HTTPConnection
14from http.client import HTTPException as HTTPException # noqa: F401
15from http.client import ResponseNotReady
16from socket import timeout as SocketTimeout
18if typing.TYPE_CHECKING:
19 from .response import HTTPResponse
20 from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
21 from .util.ssltransport import SSLTransport
23from ._collections import HTTPHeaderDict
24from .http2 import probe as http2_probe
25from .util.response import assert_header_parsing
26from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
27from .util.util import to_str
28from .util.wait import wait_for_read
30try: # Compiled with SSL?
31 import ssl
33 BaseSSLError = ssl.SSLError
34except (ImportError, AttributeError):
35 ssl = None # type: ignore[assignment]
37 class BaseSSLError(BaseException): # type: ignore[no-redef]
38 pass
41from ._base_connection import _TYPE_BODY
42from ._base_connection import ProxyConfig as ProxyConfig
43from ._base_connection import _ResponseOptions as _ResponseOptions
44from ._version import __version__
45from .exceptions import (
46 ConnectTimeoutError,
47 HeaderParsingError,
48 NameResolutionError,
49 NewConnectionError,
50 ProxyError,
51 SystemTimeWarning,
52)
53from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
54from .util.request import body_to_chunks
55from .util.ssl_ import assert_fingerprint as _assert_fingerprint
56from .util.ssl_ import (
57 create_urllib3_context,
58 is_ipaddress,
59 resolve_cert_reqs,
60 resolve_ssl_version,
61 ssl_wrap_socket,
62)
63from .util.ssl_match_hostname import CertificateError, match_hostname
64from .util.url import Url
66# Not a no-op, we're adding this to the namespace so it can be imported.
67ConnectionError = ConnectionError
68BrokenPipeError = BrokenPipeError
71log = logging.getLogger(__name__)
73port_by_scheme = {"http": 80, "https": 443}
75# When it comes time to update this value as a part of regular maintenance
76# (ie test_recent_date is failing) update it to ~6 months before the current date.
77RECENT_DATE = datetime.date(2025, 1, 1)
79_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
80# Starting the optional OWS match at the beginning of a whitespace run avoids
81# quadratic backtracking for long header values.
82_OBSOLETE_FOLD_RE = re.compile(r"(?:(?<![ \t])[ \t]+)?\r\n[ \t]+")
85def _normalize_header_value(value: str) -> str:
86 if "\r\n" not in value:
87 return value
88 return _OBSOLETE_FOLD_RE.sub(" ", value)
91def _normalize_header_values(
92 message: http.client.HTTPMessage,
93) -> list[tuple[str, str]]:
94 header_items = message.items()
95 headers_changed = False
96 for index, (name, value) in enumerate(header_items):
97 normalized_value = _normalize_header_value(value)
98 if normalized_value != value:
99 header_items[index] = (name, normalized_value)
100 headers_changed = True
102 if headers_changed:
103 # Keep the original message in sync for downstream consumers such as
104 # cookie jars while preserving its identity and header ordering.
105 for name, _ in header_items:
106 del message[name]
107 for name, value in header_items:
108 message[name] = value
110 return header_items
113class HTTPConnection(_HTTPConnection):
114 """
115 Based on :class:`http.client.HTTPConnection` but provides an extra constructor
116 backwards-compatibility layer between older and newer Pythons.
118 Additional keyword parameters are used to configure attributes of the connection.
119 Accepted parameters include:
121 - ``source_address``: Set the source address for the current connection.
122 - ``blocksize``: Read size for file-like request bodies, in bytes. Defaults to
123 16384 (16 KiB). Does not split or resize elements of iterable bodies.
124 Larger values may improve upload throughput but use more memory per active
125 upload. See :ref:`upload_buffering` for tuning guidance.
126 - ``socket_options``: Set specific options on the underlying socket. If not specified, then
127 defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
128 Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
130 For example, if you wish to enable TCP Keep Alive in addition to the defaults,
131 you might pass:
133 .. code-block:: python
135 HTTPConnection.default_socket_options + [
136 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
137 ]
139 Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
140 """
142 default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
144 #: Disable Nagle's algorithm by default.
145 #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
146 #:
147 #: Use the ``socket_options`` parameter of :class:`~urllib3.PoolManager`,
148 #: :class:`~urllib3.ProxyManager`, or :class:`~urllib3.HTTPConnectionPool`
149 #: to change this behavior.
150 default_socket_options: typing.ClassVar[
151 typing.Final[connection._TYPE_SOCKET_OPTIONS]
152 ] = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
154 #: Whether this connection verifies the host's certificate.
155 is_verified: bool = False
157 #: Whether this proxy connection verified the proxy host's certificate.
158 # If no proxy is currently connected to the value will be ``None``.
159 proxy_is_verified: bool | None = None
161 blocksize: int
162 source_address: tuple[str, int] | None
163 socket_options: connection._TYPE_SOCKET_OPTIONS | None
165 _has_connected_to_proxy: bool
166 _response_options: _ResponseOptions | None
167 _tunnel_host: str | None
168 _tunnel_port: int | None
169 _tunnel_scheme: str | None
171 def __init__(
172 self,
173 host: str,
174 port: int | None = None,
175 *,
176 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
177 source_address: tuple[str, int] | None = None,
178 blocksize: int = 16384,
179 socket_options: None | (
180 connection._TYPE_SOCKET_OPTIONS
181 ) = default_socket_options,
182 proxy: Url | None = None,
183 proxy_config: ProxyConfig | None = None,
184 ) -> None:
185 super().__init__(
186 host=host,
187 port=port,
188 timeout=Timeout.resolve_default_timeout(timeout),
189 source_address=source_address,
190 blocksize=blocksize,
191 )
192 self.socket_options = socket_options
193 self.proxy = proxy
194 self.proxy_config = proxy_config
196 self._has_connected_to_proxy = False
197 self._response_options = None
198 self._tunnel_host: str | None = None
199 self._tunnel_port: int | None = None
200 self._tunnel_scheme: str | None = None
202 def __str__(self) -> str:
203 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
205 def __repr__(self) -> str:
206 return f"<{self} at {id(self):#x}>"
208 @property
209 def host(self) -> str:
210 """
211 Getter method to remove any trailing dots that indicate the hostname is an FQDN.
213 In general, SSL certificates don't include the trailing dot indicating a
214 fully-qualified domain name, and thus, they don't validate properly when
215 checked against a domain name that includes the dot. In addition, some
216 servers may not expect to receive the trailing dot when provided.
218 However, the hostname with trailing dot is critical to DNS resolution; doing a
219 lookup with the trailing dot will properly only resolve the appropriate FQDN,
220 whereas a lookup without a trailing dot will search the system's search domain
221 list. Thus, it's important to keep the original host around for use only in
222 those cases where it's appropriate (i.e., when doing DNS lookup to establish the
223 actual TCP connection across which we're going to send HTTP requests).
224 """
225 return self._dns_host.rstrip(".")
227 @host.setter
228 def host(self, value: str) -> None:
229 """
230 Setter for the `host` property.
232 We assume that only urllib3 uses the _dns_host attribute; httplib itself
233 only uses `host`, and it seems reasonable that other libraries follow suit.
234 """
235 self._dns_host = value
237 def _new_conn(self) -> socket.socket:
238 """Establish a socket connection and set nodelay settings on it.
240 :return: New socket connection.
241 """
242 try:
243 sock = connection.create_connection(
244 (self._dns_host, self.port),
245 self.timeout,
246 source_address=self.source_address,
247 socket_options=self.socket_options,
248 )
249 except socket.gaierror as e:
250 raise NameResolutionError(self.host, self, e) from e
251 except SocketTimeout as e:
252 raise ConnectTimeoutError(
253 self,
254 f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
255 ) from e
257 except OSError as e:
258 raise NewConnectionError(
259 self, f"Failed to establish a new connection: {e}"
260 ) from e
262 sys.audit("http.client.connect", self, self.host, self.port)
264 return sock
266 def set_tunnel(
267 self,
268 host: str,
269 port: int | None = None,
270 headers: typing.Mapping[str, str] | None = None,
271 scheme: str = "http",
272 ) -> None:
273 if scheme not in ("http", "https"):
274 raise ValueError(
275 f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
276 )
277 super().set_tunnel(host, port=port, headers=headers)
278 self._tunnel_scheme = scheme
280 if sys.version_info < (3, 11, 16) or ((3, 12) <= sys.version_info < (3, 12, 14)):
281 # Taken from python/cpython#100986 which was backported in 3.11.9 and 3.12.3.
282 # When using connection_from_host, host will come without brackets.
283 # With a security patch from python/cpython#146211.
284 def _wrap_ipv6(self, ip: bytes) -> bytes:
285 if b":" in ip and ip[0] != b"["[0]:
286 return b"[" + ip + b"]"
287 return ip
289 # Copied from CPython 3.12.13 Lib/http/client.py
290 _is_legal_header_name = staticmethod(re.compile(rb"[^:\s][^:\r\n]*").fullmatch)
291 _is_illegal_header_value = staticmethod(
292 re.compile(rb"\n(?![ \t])|\r(?![ \t\n])").search
293 )
294 _contains_disallowed_url_pchar_re = re.compile("[\x00-\x20\x7f]")
296 if sys.version_info < (3, 11, 16):
297 # `_tunnel` copied from 3.11.15 backporting
298 # https://github.com/python/cpython/commit/0d4026432591d43185568dd31cef6a034c4b9261
299 # and https://github.com/python/cpython/commit/6fbc61070fda2ffb8889e77e3b24bca4249ab4d1
300 # plus a fix from https://github.com/python/cpython/commit/56b7100b04e44ea27989242b176beb8f016b2c53
301 def _tunnel(self) -> None:
302 if self._contains_disallowed_url_pchar_re.search(self._tunnel_host): # type: ignore[arg-type]
303 raise ValueError(
304 "Tunnel host can't contain control characters %r"
305 % (self._tunnel_host,)
306 )
307 _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
308 connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
309 self._wrap_ipv6(self._tunnel_host.encode("ascii")), # type: ignore[union-attr]
310 self._tunnel_port,
311 )
312 headers = [connect]
313 for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
314 header_bytes = header.encode("latin-1")
315 value_bytes = value.encode("latin-1")
316 if not self._is_legal_header_name(header_bytes):
317 raise ValueError(f"Invalid header name {header_bytes!r}")
318 if self._is_illegal_header_value(value_bytes):
319 raise ValueError(f"Invalid header value {value_bytes!r}")
320 headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes))
321 headers.append(b"\r\n")
322 # Making a single send() call instead of one per line encourages
323 # the host OS to use a more optimal packet size instead of
324 # potentially emitting a series of small packets.
325 self.send(b"".join(headers))
326 del headers
328 response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
329 try:
330 version, code, message = response._read_status() # type: ignore[attr-defined]
332 if code != http.HTTPStatus.OK:
333 self.close()
334 raise OSError(
335 f"Tunnel connection failed: {code} {message.strip()}"
336 )
337 while True:
338 line = response.fp.readline(_MAXLINE + 1)
339 if len(line) > _MAXLINE:
340 raise http.client.LineTooLong("header line")
341 if not line:
342 # for sites which EOF without sending a trailer
343 break
344 if line in (b"\r\n", b"\n", b""):
345 break
347 if self.debuglevel > 0:
348 print("header:", line.decode())
349 finally:
350 response.close()
352 elif (3, 12) <= sys.version_info < (3, 12, 14):
353 # `_tunnel` copied from 3.12.13 backporting
354 # https://github.com/python/cpython/commit/23aef575c7629abcd4aaf028ebd226fb41a4b3c8
355 # plus a fix from https://github.com/python/cpython/commit/c00c386faa579ad71196d33408644478488e43ec
356 def _tunnel(self) -> None: # noqa: F811
357 if self._contains_disallowed_url_pchar_re.search(self._tunnel_host): # type: ignore[arg-type]
358 raise ValueError(
359 "Tunnel host can't contain control characters %r"
360 % (self._tunnel_host,)
361 )
362 connect = b"CONNECT %s:%d HTTP/1.1\r\n" % ( # type: ignore[str-format]
363 self._wrap_ipv6(self._tunnel_host.encode("idna")), # type: ignore[union-attr]
364 self._tunnel_port,
365 )
366 headers = [connect]
367 for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
368 header_bytes = header.encode("latin-1")
369 value_bytes = value.encode("latin-1")
370 if not self._is_legal_header_name(header_bytes):
371 raise ValueError(f"Invalid header name {header_bytes!r}")
372 if self._is_illegal_header_value(value_bytes):
373 raise ValueError(f"Invalid header value {value_bytes!r}")
374 headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes))
375 headers.append(b"\r\n")
376 # Making a single send() call instead of one per line encourages
377 # the host OS to use a more optimal packet size instead of
378 # potentially emitting a series of small packets.
379 self.send(b"".join(headers))
380 del headers
382 response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
383 try:
384 version, code, message = response._read_status() # type: ignore[attr-defined]
386 self._raw_proxy_headers = http.client._read_headers(response.fp) # type: ignore[attr-defined]
388 if self.debuglevel > 0:
389 for header in self._raw_proxy_headers:
390 print("header:", header.decode())
392 if code != http.HTTPStatus.OK:
393 self.close()
394 raise OSError(
395 f"Tunnel connection failed: {code} {message.strip()}"
396 )
398 finally:
399 response.close()
401 def connect(self) -> None:
402 self.sock = self._new_conn()
403 if self._tunnel_host:
404 # If we're tunneling it means we're connected to our proxy.
405 self._has_connected_to_proxy = True
407 # TODO: Fix tunnel so it doesn't depend on self.sock state.
408 self._tunnel()
410 # If there's a proxy to be connected to we are fully connected.
411 # This is set twice (once above and here) due to forwarding proxies
412 # not using tunnelling.
413 self._has_connected_to_proxy = bool(self.proxy)
415 if self._has_connected_to_proxy:
416 self.proxy_is_verified = False
418 @property
419 def is_closed(self) -> bool:
420 return self.sock is None
422 @property
423 def is_connected(self) -> bool:
424 if self.sock is None:
425 return False
426 return not wait_for_read(self.sock, timeout=0.0)
428 @property
429 def has_connected_to_proxy(self) -> bool:
430 return self._has_connected_to_proxy
432 @property
433 def proxy_is_forwarding(self) -> bool:
434 """
435 Return True if a forwarding proxy is configured, else return False
436 """
437 return bool(self.proxy) and self._tunnel_host is None
439 @property
440 def proxy_is_tunneling(self) -> bool:
441 """
442 Return True if a tunneling proxy is configured, else return False
443 """
444 return self._tunnel_host is not None
446 def close(self) -> None:
447 try:
448 super().close()
449 finally:
450 # Reset all stateful properties so connection
451 # can be re-used without leaking prior configs.
452 self.sock = None
453 self.is_verified = False
454 self.proxy_is_verified = None
455 self._has_connected_to_proxy = False
456 self._response_options = None
457 self._tunnel_host = None
458 self._tunnel_port = None
459 self._tunnel_scheme = None
461 def putrequest(
462 self,
463 method: str,
464 url: str,
465 skip_host: bool = False,
466 skip_accept_encoding: bool = False,
467 ) -> None:
468 """"""
469 # Empty docstring because the indentation of CPython's implementation
470 # is broken but we don't want this method in our documentation.
471 match = _CONTAINS_CONTROL_CHAR_RE.search(method)
472 if match:
473 raise ValueError(
474 f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
475 )
477 return super().putrequest(
478 method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
479 )
481 def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
482 """"""
483 if len(values) == 1:
484 # Avoid generator creation, any(), and argument unpacking
485 # on the common single-value path.
486 value = values[0]
487 if not (isinstance(value, str) and value == SKIP_HEADER):
488 super().putheader(header, value)
489 return
490 skip_header = True
491 else:
492 skip_header = any(
493 isinstance(value, str) and value == SKIP_HEADER for value in values
494 )
496 if not skip_header:
497 super().putheader(header, *values)
498 elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
499 skippable_headers = "', '".join(
500 [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
501 )
502 raise ValueError(
503 f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
504 )
506 # `request` method's signature intentionally violates LSP.
507 # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
508 def request( # type: ignore[override]
509 self,
510 method: str,
511 url: str,
512 body: _TYPE_BODY | None = None,
513 headers: typing.Mapping[str, str] | None = None,
514 *,
515 chunked: bool = False,
516 preload_content: bool = True,
517 decode_content: bool = True,
518 enforce_content_length: bool = True,
519 ) -> None:
520 # Update the inner socket's timeout value to send the request.
521 # This only triggers if the connection is re-used.
522 if self.sock is not None:
523 self.sock.settimeout(self.timeout)
525 # Store these values to be fed into the HTTPResponse
526 # object later. TODO: Remove this in favor of a real
527 # HTTP lifecycle mechanism.
529 # We have to store these before we call .request()
530 # because sometimes we can still salvage a response
531 # off the wire even if we aren't able to completely
532 # send the request body.
533 self._response_options = _ResponseOptions(
534 request_method=method,
535 request_url=url,
536 preload_content=preload_content,
537 decode_content=decode_content,
538 enforce_content_length=enforce_content_length,
539 )
541 if headers is None:
542 headers = {}
543 header_keys = frozenset(to_str(k.lower()) for k in headers)
544 skip_accept_encoding = "accept-encoding" in header_keys
545 skip_host = "host" in header_keys
546 self.putrequest(
547 method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
548 )
550 # Transform the body into an iterable of sendall()-able chunks
551 # and detect if an explicit Content-Length is doable.
552 chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
553 chunks = chunks_and_cl.chunks
554 content_length = chunks_and_cl.content_length
556 # When chunked is explicit set to 'True' we respect that.
557 if chunked:
558 if "transfer-encoding" not in header_keys:
559 self.putheader("Transfer-Encoding", "chunked")
560 else:
561 # Detect whether a framing mechanism is already in use. If so
562 # we respect that value, otherwise we pick chunked vs content-length
563 # depending on the type of 'body'.
564 if "content-length" in header_keys:
565 chunked = False
566 elif "transfer-encoding" in header_keys:
567 chunked = True
569 # Otherwise we go off the recommendation of 'body_to_chunks()'.
570 else:
571 chunked = False
572 if content_length is None:
573 if chunks is not None:
574 chunked = True
575 self.putheader("Transfer-Encoding", "chunked")
576 else:
577 self.putheader("Content-Length", str(content_length))
579 # Now that framing headers are out of the way we send all the other headers.
580 if "user-agent" not in header_keys:
581 self.putheader("User-Agent", _get_default_user_agent())
582 for header, value in headers.items():
583 self.putheader(header, value)
584 self.endheaders()
586 # If we're given a body we start sending that in chunks.
587 if chunks is not None:
588 for chunk in chunks:
589 # Sending empty chunks isn't allowed for TE: chunked
590 # as it indicates the end of the body.
591 if not chunk:
592 continue
593 if isinstance(chunk, str):
594 chunk = chunk.encode("utf-8")
595 if chunked:
596 self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
597 else:
598 self.send(chunk)
600 # Regardless of whether we have a body or not, if we're in
601 # chunked mode we want to send an explicit empty chunk.
602 if chunked:
603 self.send(b"0\r\n\r\n")
605 def request_chunked(
606 self,
607 method: str,
608 url: str,
609 body: _TYPE_BODY | None = None,
610 headers: typing.Mapping[str, str] | None = None,
611 ) -> None:
612 """
613 Alternative to the common request method, which sends the
614 body with chunked encoding and not as one block
615 """
616 warnings.warn(
617 "HTTPConnection.request_chunked() is deprecated and will be removed "
618 "in urllib3 v3.0. Instead use HTTPConnection.request(..., chunked=True).",
619 category=FutureWarning,
620 stacklevel=2,
621 )
622 self.request(method, url, body=body, headers=headers, chunked=True)
624 def getresponse( # type: ignore[override]
625 self,
626 ) -> HTTPResponse:
627 """
628 Get the response from the server.
630 If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
632 If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
633 """
634 # Raise the same error as http.client.HTTPConnection
635 if self._response_options is None:
636 raise ResponseNotReady()
638 # Reset this attribute for being used again.
639 resp_options = self._response_options
640 self._response_options = None
642 # Since the connection's timeout value may have been updated
643 # we need to set the timeout on the socket.
644 self.sock.settimeout(self.timeout)
646 # This is needed here to avoid circular import errors
647 from .response import HTTPResponse
649 # Save a reference to the shutdown function before ownership is passed
650 # to httplib_response
651 # TODO should we implement it everywhere?
652 _shutdown = getattr(self.sock, "shutdown", None)
654 # Get the response from http.client.HTTPConnection
655 httplib_response = super().getresponse()
657 try:
658 assert_header_parsing(httplib_response.msg)
659 except (HeaderParsingError, TypeError) as hpe:
660 log.warning(
661 "Failed to parse headers (url=%s): %s",
662 _url_from_connection(self, resp_options.request_url),
663 hpe,
664 exc_info=True,
665 )
667 header_items = _normalize_header_values(httplib_response.msg)
668 headers = HTTPHeaderDict(header_items)
670 response = HTTPResponse(
671 body=httplib_response,
672 headers=headers,
673 status=httplib_response.status,
674 version=httplib_response.version,
675 version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
676 reason=httplib_response.reason,
677 preload_content=resp_options.preload_content,
678 decode_content=resp_options.decode_content,
679 original_response=httplib_response,
680 enforce_content_length=resp_options.enforce_content_length,
681 request_method=resp_options.request_method,
682 request_url=resp_options.request_url,
683 sock_shutdown=_shutdown,
684 )
685 return response
688class HTTPSConnection(HTTPConnection):
689 """
690 Many of the parameters to this constructor are passed to the underlying SSL
691 socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
692 """
694 default_port = port_by_scheme["https"] # type: ignore[misc]
696 cert_reqs: int | str | None = None
697 ca_certs: str | None = None
698 ca_cert_dir: str | None = None
699 ca_cert_data: None | str | bytes = None
700 ssl_version: int | str | None = None
701 ssl_minimum_version: int | None = None
702 ssl_maximum_version: int | None = None
703 assert_fingerprint: str | None = None
704 _connect_callback: typing.Callable[..., None] | None = None
706 def __init__(
707 self,
708 host: str,
709 port: int | None = None,
710 *,
711 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
712 source_address: tuple[str, int] | None = None,
713 blocksize: int = 16384,
714 socket_options: None | (
715 connection._TYPE_SOCKET_OPTIONS
716 ) = HTTPConnection.default_socket_options,
717 proxy: Url | None = None,
718 proxy_config: ProxyConfig | None = None,
719 cert_reqs: int | str | None = None,
720 assert_hostname: None | str | typing.Literal[False] = None,
721 assert_fingerprint: str | None = None,
722 server_hostname: str | None = None,
723 ssl_context: ssl.SSLContext | None = None,
724 ca_certs: str | None = None,
725 ca_cert_dir: str | None = None,
726 ca_cert_data: None | str | bytes = None,
727 ssl_minimum_version: int | None = None,
728 ssl_maximum_version: int | None = None,
729 ssl_version: int | str | None = None, # Deprecated
730 cert_file: str | None = None,
731 key_file: str | None = None,
732 key_password: str | None = None,
733 ) -> None:
734 super().__init__(
735 host,
736 port=port,
737 timeout=timeout,
738 source_address=source_address,
739 blocksize=blocksize,
740 socket_options=socket_options,
741 proxy=proxy,
742 proxy_config=proxy_config,
743 )
745 self.key_file = key_file
746 self.cert_file = cert_file
747 self.key_password = key_password
748 self.ssl_context = ssl_context
749 self.server_hostname = server_hostname
750 self.assert_hostname = assert_hostname
751 self.assert_fingerprint = assert_fingerprint
752 self.ssl_version = ssl_version
753 self.ssl_minimum_version = ssl_minimum_version
754 self.ssl_maximum_version = ssl_maximum_version
755 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
756 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
757 self.ca_cert_data = ca_cert_data
759 # cert_reqs depends on ssl_context so calculate last.
760 if cert_reqs is None:
761 if self.ssl_context is not None:
762 cert_reqs = self.ssl_context.verify_mode
763 else:
764 cert_reqs = resolve_cert_reqs(None)
765 self.cert_reqs = cert_reqs
766 self._connect_callback = None
768 def set_cert(
769 self,
770 key_file: str | None = None,
771 cert_file: str | None = None,
772 cert_reqs: int | str | None = None,
773 key_password: str | None = None,
774 ca_certs: str | None = None,
775 assert_hostname: None | str | typing.Literal[False] = None,
776 assert_fingerprint: str | None = None,
777 ca_cert_dir: str | None = None,
778 ca_cert_data: None | str | bytes = None,
779 ) -> None:
780 """
781 This method should only be called once, before the connection is used.
782 """
783 warnings.warn(
784 "HTTPSConnection.set_cert() is deprecated and will be removed "
785 "in urllib3 v3.0. Instead provide the parameters to the "
786 "HTTPSConnection constructor.",
787 category=FutureWarning,
788 stacklevel=2,
789 )
791 # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
792 # have an SSLContext object in which case we'll use its verify_mode.
793 if cert_reqs is None:
794 if self.ssl_context is not None:
795 cert_reqs = self.ssl_context.verify_mode
796 else:
797 cert_reqs = resolve_cert_reqs(None)
799 self.key_file = key_file
800 self.cert_file = cert_file
801 self.cert_reqs = cert_reqs
802 self.key_password = key_password
803 self.assert_hostname = assert_hostname
804 self.assert_fingerprint = assert_fingerprint
805 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
806 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
807 self.ca_cert_data = ca_cert_data
809 def connect(self) -> None:
810 # Today we don't need to be doing this step before the /actual/ socket
811 # connection, however in the future we'll need to decide whether to
812 # create a new socket or re-use an existing "shared" socket as a part
813 # of the HTTP/2 handshake dance.
814 if self._tunnel_host is not None and self._tunnel_port is not None:
815 probe_http2_host = self._tunnel_host
816 probe_http2_port = self._tunnel_port
817 else:
818 probe_http2_host = self.host
819 probe_http2_port = self.port
821 # Check if the target origin supports HTTP/2.
822 # If the value comes back as 'None' it means that the current thread
823 # is probing for HTTP/2 support. Otherwise, we're waiting for another
824 # probe to complete, or we get a value right away.
825 target_supports_http2: bool | None
826 if "h2" in ssl_.ALPN_PROTOCOLS:
827 target_supports_http2 = http2_probe.acquire_and_get(
828 host=probe_http2_host, port=probe_http2_port
829 )
830 else:
831 # If HTTP/2 isn't going to be offered it doesn't matter if
832 # the target supports HTTP/2. Don't want to make a probe.
833 target_supports_http2 = False
835 if self._connect_callback is not None:
836 self._connect_callback(
837 "before connect",
838 thread_id=threading.get_ident(),
839 target_supports_http2=target_supports_http2,
840 )
842 try:
843 sock: socket.socket | ssl.SSLSocket
844 self.sock = sock = self._new_conn()
845 server_hostname: str = self.host
846 tls_in_tls = False
848 # Do we need to establish a tunnel?
849 if self.proxy_is_tunneling:
850 # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
851 if self._tunnel_scheme == "https":
852 # _connect_tls_proxy will verify and assign proxy_is_verified
853 self.sock = sock = self._connect_tls_proxy(self.host, sock)
854 tls_in_tls = True
855 elif self._tunnel_scheme == "http":
856 self.proxy_is_verified = False
858 # If we're tunneling it means we're connected to our proxy.
859 self._has_connected_to_proxy = True
861 self._tunnel()
862 # Override the host with the one we're requesting data from.
863 server_hostname = typing.cast(str, self._tunnel_host)
865 if self.server_hostname is not None:
866 server_hostname = self.server_hostname
868 is_time_off = datetime.date.today() < RECENT_DATE
869 if is_time_off:
870 warnings.warn(
871 (
872 f"System time is way off (before {RECENT_DATE}). This will probably "
873 "lead to SSL verification errors"
874 ),
875 SystemTimeWarning,
876 )
878 # Remove trailing '.' from fqdn hostnames to allow certificate validation
879 server_hostname_rm_dot = server_hostname.rstrip(".")
881 # Forwarding proxies should use proxy SSL context for
882 # wrapping since that's the connection being established,
883 # whereas tunneling proxies should use the connection's SSL
884 # context.
885 # However, for backwards compatibility reasons, if the proxy
886 # is forwarding but no proxy SSL context is provided, we
887 # fall back to using the connection's SSL context until
888 # urllib3 v3.0. Appropriate warning is emitted in
889 # ``ProxyManager.__init__``.
890 wrapped_socket: ssl.SSLSocket | SSLTransport
891 if self.proxy_is_forwarding and self.proxy_config is not None:
892 wrapped_socket = self._connect_tls_proxy(self.host, sock)
893 is_verified = self.proxy_is_verified is True
894 else:
895 wrapped_socket, is_verified = _ssl_wrap_socket_and_match_hostname(
896 sock=sock,
897 cert_reqs=self.cert_reqs,
898 ssl_version=self.ssl_version,
899 ssl_minimum_version=self.ssl_minimum_version,
900 ssl_maximum_version=self.ssl_maximum_version,
901 ca_certs=self.ca_certs,
902 ca_cert_dir=self.ca_cert_dir,
903 ca_cert_data=self.ca_cert_data,
904 cert_file=self.cert_file,
905 key_file=self.key_file,
906 key_password=self.key_password,
907 server_hostname=server_hostname_rm_dot,
908 ssl_context=self.ssl_context,
909 tls_in_tls=tls_in_tls,
910 assert_hostname=self.assert_hostname,
911 assert_fingerprint=self.assert_fingerprint,
912 )
913 self.sock = wrapped_socket
915 # If an error occurs during connection/handshake we may need to release
916 # our lock so another connection can probe the origin.
917 except BaseException:
918 if self._connect_callback is not None:
919 self._connect_callback(
920 "after connect failure",
921 thread_id=threading.get_ident(),
922 target_supports_http2=target_supports_http2,
923 )
925 if target_supports_http2 is None:
926 http2_probe.set_and_release(
927 host=probe_http2_host, port=probe_http2_port, supports_http2=None
928 )
929 raise
931 # If this connection doesn't know if the origin supports HTTP/2
932 # we report back to the HTTP/2 probe our result.
933 if target_supports_http2 is None:
934 supports_http2 = wrapped_socket.selected_alpn_protocol() == "h2"
935 http2_probe.set_and_release(
936 host=probe_http2_host,
937 port=probe_http2_port,
938 supports_http2=supports_http2,
939 )
941 # Forwarding proxies can never have a verified target since
942 # the proxy is the one doing the verification. Should instead
943 # use a CONNECT tunnel in order to verify the target.
944 # See: https://github.com/urllib3/urllib3/issues/3267.
945 if self.proxy_is_forwarding:
946 self.is_verified = False
947 else:
948 self.is_verified = is_verified
950 # If there's a proxy to be connected to we are fully connected.
951 # This is set twice (once above and here) due to forwarding proxies
952 # not using tunnelling.
953 self._has_connected_to_proxy = bool(self.proxy)
955 # Set `self.proxy_is_verified` unless it's already set while
956 # establishing a tunnel.
957 if self._has_connected_to_proxy and self.proxy_is_verified is None:
958 self.proxy_is_verified = is_verified
960 def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
961 """
962 Establish a TLS connection to the proxy using proxy-specific policy.
963 """
964 proxy_config = typing.cast(ProxyConfig, self.proxy_config)
965 proxy_ssl_context = proxy_config.ssl_context
967 ssl_context: ssl.SSLContext | None
968 cert_reqs: int | str | None
969 if proxy_ssl_context is not None:
970 # Prefer the proxy's cert policy for the proxy connection
971 ssl_context = proxy_ssl_context
972 cert_reqs = proxy_ssl_context.verify_mode
973 ca_certs = None
974 ca_cert_dir = None
975 ca_cert_data = None
976 ssl_version = None
977 ssl_minimum_version = None
978 ssl_maximum_version = None
979 else:
980 # Otherwise we inherit the pool's cert policies
981 ssl_context = self.ssl_context if self.proxy_is_forwarding else None
982 cert_reqs = self.cert_reqs
983 ca_certs = self.ca_certs
984 ca_cert_dir = self.ca_cert_dir
985 ca_cert_data = self.ca_cert_data
986 ssl_version = self.ssl_version
987 ssl_minimum_version = self.ssl_minimum_version
988 ssl_maximum_version = self.ssl_maximum_version
990 sock_and_verified = _ssl_wrap_socket_and_match_hostname(
991 sock,
992 cert_reqs=cert_reqs,
993 ssl_version=ssl_version,
994 ssl_minimum_version=ssl_minimum_version,
995 ssl_maximum_version=ssl_maximum_version,
996 ca_certs=ca_certs,
997 ca_cert_dir=ca_cert_dir,
998 ca_cert_data=ca_cert_data,
999 server_hostname=hostname,
1000 ssl_context=ssl_context,
1001 assert_hostname=proxy_config.assert_hostname,
1002 assert_fingerprint=proxy_config.assert_fingerprint,
1003 # Features that aren't implemented for proxies yet:
1004 cert_file=None,
1005 key_file=None,
1006 key_password=None,
1007 tls_in_tls=False,
1008 )
1009 self.proxy_is_verified = sock_and_verified.is_verified
1010 return sock_and_verified.socket # type: ignore[return-value]
1013class _WrappedAndVerifiedSocket(typing.NamedTuple):
1014 """
1015 Wrapped socket and whether the connection is
1016 verified after the TLS handshake
1017 """
1019 socket: ssl.SSLSocket | SSLTransport
1020 is_verified: bool
1023def _ssl_wrap_socket_and_match_hostname(
1024 sock: socket.socket,
1025 *,
1026 cert_reqs: None | str | int,
1027 ssl_version: None | str | int,
1028 ssl_minimum_version: int | None,
1029 ssl_maximum_version: int | None,
1030 cert_file: str | None,
1031 key_file: str | None,
1032 key_password: str | None,
1033 ca_certs: str | None,
1034 ca_cert_dir: str | None,
1035 ca_cert_data: None | str | bytes,
1036 assert_hostname: None | str | typing.Literal[False],
1037 assert_fingerprint: str | None,
1038 server_hostname: str | None,
1039 ssl_context: ssl.SSLContext | None,
1040 tls_in_tls: bool = False,
1041) -> _WrappedAndVerifiedSocket:
1042 """Logic for constructing an SSLContext from all TLS parameters, passing
1043 that down into ssl_wrap_socket, and then doing certificate verification
1044 either via hostname or fingerprint. This function exists to guarantee
1045 that both proxies and targets have the same behavior when connecting via TLS.
1046 """
1047 default_ssl_context = False
1048 if ssl_context is None:
1049 default_ssl_context = True
1050 context = create_urllib3_context(
1051 ssl_version=resolve_ssl_version(ssl_version),
1052 ssl_minimum_version=ssl_minimum_version,
1053 ssl_maximum_version=ssl_maximum_version,
1054 cert_reqs=resolve_cert_reqs(cert_reqs),
1055 )
1056 else:
1057 context = ssl_context
1059 context.verify_mode = resolve_cert_reqs(cert_reqs)
1061 # In some cases, we want to verify hostnames ourselves
1062 if (
1063 # `ssl` can't verify fingerprints or alternate hostnames
1064 assert_fingerprint
1065 or assert_hostname
1066 # assert_hostname can be set to False to disable hostname checking
1067 or assert_hostname is False
1068 # We still support OpenSSL 1.0.2, which prevents us from verifying
1069 # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
1070 or ssl_.IS_PYOPENSSL
1071 or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
1072 ):
1073 context.check_hostname = False
1075 # Try to load OS default certs if none are given. We need to do the hasattr() check
1076 # for custom pyOpenSSL SSLContext objects because they don't support
1077 # load_default_certs().
1078 if (
1079 not ca_certs
1080 and not ca_cert_dir
1081 and not ca_cert_data
1082 and default_ssl_context
1083 and hasattr(context, "load_default_certs")
1084 ):
1085 context.load_default_certs()
1087 # Ensure that IPv6 addresses are in the proper format and don't have a
1088 # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
1089 # and interprets them as DNS hostnames.
1090 if server_hostname is not None:
1091 normalized = server_hostname.strip("[]")
1092 if "%" in normalized:
1093 normalized = normalized[: normalized.rfind("%")]
1094 if is_ipaddress(normalized):
1095 server_hostname = normalized
1097 ssl_sock = ssl_wrap_socket(
1098 sock=sock,
1099 keyfile=key_file,
1100 certfile=cert_file,
1101 key_password=key_password,
1102 ca_certs=ca_certs,
1103 ca_cert_dir=ca_cert_dir,
1104 ca_cert_data=ca_cert_data,
1105 server_hostname=server_hostname,
1106 ssl_context=context,
1107 tls_in_tls=tls_in_tls,
1108 )
1110 try:
1111 if assert_fingerprint:
1112 _assert_fingerprint(
1113 ssl_sock.getpeercert(binary_form=True), assert_fingerprint
1114 )
1115 elif (
1116 context.verify_mode != ssl.CERT_NONE
1117 and not context.check_hostname
1118 and assert_hostname is not False
1119 ):
1120 cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
1122 # Need to signal to our match_hostname whether to use 'commonName' or not.
1123 # If we're using our own constructed SSLContext we explicitly set 'False'
1124 # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
1125 if default_ssl_context:
1126 hostname_checks_common_name = False
1127 else:
1128 hostname_checks_common_name = (
1129 getattr(context, "hostname_checks_common_name", False) or False
1130 )
1132 _match_hostname(
1133 cert,
1134 assert_hostname or server_hostname, # type: ignore[arg-type]
1135 hostname_checks_common_name,
1136 )
1138 return _WrappedAndVerifiedSocket(
1139 socket=ssl_sock,
1140 is_verified=context.verify_mode == ssl.CERT_REQUIRED
1141 or bool(assert_fingerprint),
1142 )
1143 except BaseException:
1144 ssl_sock.close()
1145 raise
1148def _match_hostname(
1149 cert: _TYPE_PEER_CERT_RET_DICT | None,
1150 asserted_hostname: str,
1151 hostname_checks_common_name: bool = False,
1152) -> None:
1153 # Our upstream implementation of ssl.match_hostname()
1154 # only applies this normalization to IP addresses so it doesn't
1155 # match DNS SANs so we do the same thing!
1156 stripped_hostname = asserted_hostname.strip("[]")
1157 if is_ipaddress(stripped_hostname):
1158 asserted_hostname = stripped_hostname
1160 try:
1161 match_hostname(cert, asserted_hostname, hostname_checks_common_name)
1162 except CertificateError as e:
1163 log.warning(
1164 "Certificate did not match expected hostname: %s. Certificate: %s",
1165 asserted_hostname,
1166 cert,
1167 )
1168 # Add cert to exception and reraise so client code can inspect
1169 # the cert when catching the exception, if they want to
1170 e._peer_cert = cert # type: ignore[attr-defined]
1171 raise
1174def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
1175 # Look for the phrase 'wrong version number', if found
1176 # then we should warn the user that we're very sure that
1177 # this proxy is HTTP-only and they have a configuration issue.
1178 error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
1179 is_likely_http_proxy = (
1180 "wrong version number" in error_normalized
1181 or "unknown protocol" in error_normalized
1182 or "record layer failure" in error_normalized
1183 )
1184 http_proxy_warning = (
1185 ". Your proxy appears to only use HTTP and not HTTPS, "
1186 "try changing your proxy URL to be HTTP. See: "
1187 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1188 "#https-proxy-error-http-proxy"
1189 )
1190 new_err = ProxyError(
1191 f"Unable to connect to proxy"
1192 f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
1193 err,
1194 )
1195 new_err.__cause__ = err
1196 return new_err
1199def _get_default_user_agent() -> str:
1200 return f"python-urllib3/{__version__}"
1203class DummyConnection:
1204 """Used to detect a failed ConnectionCls import."""
1207if not ssl:
1208 HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
1211VerifiedHTTPSConnection = HTTPSConnection
1214def _url_from_connection(
1215 conn: HTTPConnection | HTTPSConnection, path: str | None = None
1216) -> str:
1217 """Returns the URL from a given connection. This is mainly used for testing and logging."""
1219 scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
1221 return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url