Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/connection.py: 25%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import datetime
4import http.client
5import logging
6import os
7import re
8import socket
9import sys
10import threading
11import typing
12import warnings
13from http.client import HTTPConnection as _HTTPConnection
14from http.client import HTTPException as HTTPException # noqa: F401
15from http.client import ResponseNotReady
16from socket import timeout as SocketTimeout
18if typing.TYPE_CHECKING:
19 from .response import HTTPResponse
20 from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
21 from .util.ssltransport import SSLTransport
23from ._collections import HTTPHeaderDict
24from .http2 import probe as http2_probe
25from .util.response import assert_header_parsing
26from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
27from .util.util import to_str
28from .util.wait import wait_for_read
30try: # Compiled with SSL?
31 import ssl
33 BaseSSLError = ssl.SSLError
34except (ImportError, AttributeError):
35 ssl = None # type: ignore[assignment]
37 class BaseSSLError(BaseException): # type: ignore[no-redef]
38 pass
41from ._base_connection import _TYPE_BODY
42from ._base_connection import ProxyConfig as ProxyConfig
43from ._base_connection import _ResponseOptions as _ResponseOptions
44from ._version import __version__
45from .exceptions import (
46 ConnectTimeoutError,
47 HeaderParsingError,
48 NameResolutionError,
49 NewConnectionError,
50 ProxyError,
51 SystemTimeWarning,
52)
53from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
54from .util.request import body_to_chunks
55from .util.ssl_ import assert_fingerprint as _assert_fingerprint
56from .util.ssl_ import (
57 create_urllib3_context,
58 is_ipaddress,
59 resolve_cert_reqs,
60 resolve_ssl_version,
61 ssl_wrap_socket,
62)
63from .util.ssl_match_hostname import CertificateError, match_hostname
64from .util.url import Url
66# Not a no-op, we're adding this to the namespace so it can be imported.
67ConnectionError = ConnectionError
68BrokenPipeError = BrokenPipeError
71log = logging.getLogger(__name__)
73port_by_scheme = {"http": 80, "https": 443}
75# When it comes time to update this value as a part of regular maintenance
76# (ie test_recent_date is failing) update it to ~6 months before the current date.
77RECENT_DATE = datetime.date(2025, 1, 1)
79_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
80# Starting the optional OWS match at the beginning of a whitespace run avoids
81# quadratic backtracking for long header values.
82_OBSOLETE_FOLD_RE = re.compile(r"(?:(?<![ \t])[ \t]+)?\r\n[ \t]+")
85def _normalize_header_value(value: str) -> str:
86 if "\r\n" not in value:
87 return value
88 return _OBSOLETE_FOLD_RE.sub(" ", value)
91def _normalize_header_values(
92 message: http.client.HTTPMessage,
93) -> list[tuple[str, str]]:
94 header_items = message.items()
95 headers_changed = False
96 for index, (name, value) in enumerate(header_items):
97 normalized_value = _normalize_header_value(value)
98 if normalized_value != value:
99 header_items[index] = (name, normalized_value)
100 headers_changed = True
102 if headers_changed:
103 # Keep the original message in sync for downstream consumers such as
104 # cookie jars while preserving its identity and header ordering.
105 for name, _ in header_items:
106 del message[name]
107 for name, value in header_items:
108 message[name] = value
110 return header_items
113class HTTPConnection(_HTTPConnection):
114 """
115 Based on :class:`http.client.HTTPConnection` but provides an extra constructor
116 backwards-compatibility layer between older and newer Pythons.
118 Additional keyword parameters are used to configure attributes of the connection.
119 Accepted parameters include:
121 - ``source_address``: Set the source address for the current connection.
122 - ``socket_options``: Set specific options on the underlying socket. If not specified, then
123 defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
124 Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
126 For example, if you wish to enable TCP Keep Alive in addition to the defaults,
127 you might pass:
129 .. code-block:: python
131 HTTPConnection.default_socket_options + [
132 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
133 ]
135 Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
136 """
138 default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
140 #: Disable Nagle's algorithm by default.
141 #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
142 #:
143 #: Use the ``socket_options`` parameter of :class:`~urllib3.PoolManager`,
144 #: :class:`~urllib3.ProxyManager`, or :class:`~urllib3.HTTPConnectionPool`
145 #: to change this behavior.
146 default_socket_options: typing.ClassVar[
147 typing.Final[connection._TYPE_SOCKET_OPTIONS]
148 ] = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
150 #: Whether this connection verifies the host's certificate.
151 is_verified: bool = False
153 #: Whether this proxy connection verified the proxy host's certificate.
154 # If no proxy is currently connected to the value will be ``None``.
155 proxy_is_verified: bool | None = None
157 blocksize: int
158 source_address: tuple[str, int] | None
159 socket_options: connection._TYPE_SOCKET_OPTIONS | None
161 _has_connected_to_proxy: bool
162 _response_options: _ResponseOptions | None
163 _tunnel_host: str | None
164 _tunnel_port: int | None
165 _tunnel_scheme: str | None
167 def __init__(
168 self,
169 host: str,
170 port: int | None = None,
171 *,
172 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
173 source_address: tuple[str, int] | None = None,
174 blocksize: int = 16384,
175 socket_options: None | (
176 connection._TYPE_SOCKET_OPTIONS
177 ) = default_socket_options,
178 proxy: Url | None = None,
179 proxy_config: ProxyConfig | None = None,
180 ) -> None:
181 super().__init__(
182 host=host,
183 port=port,
184 timeout=Timeout.resolve_default_timeout(timeout),
185 source_address=source_address,
186 blocksize=blocksize,
187 )
188 self.socket_options = socket_options
189 self.proxy = proxy
190 self.proxy_config = proxy_config
192 self._has_connected_to_proxy = False
193 self._response_options = None
194 self._tunnel_host: str | None = None
195 self._tunnel_port: int | None = None
196 self._tunnel_scheme: str | None = None
198 def __str__(self) -> str:
199 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
201 def __repr__(self) -> str:
202 return f"<{self} at {id(self):#x}>"
204 @property
205 def host(self) -> str:
206 """
207 Getter method to remove any trailing dots that indicate the hostname is an FQDN.
209 In general, SSL certificates don't include the trailing dot indicating a
210 fully-qualified domain name, and thus, they don't validate properly when
211 checked against a domain name that includes the dot. In addition, some
212 servers may not expect to receive the trailing dot when provided.
214 However, the hostname with trailing dot is critical to DNS resolution; doing a
215 lookup with the trailing dot will properly only resolve the appropriate FQDN,
216 whereas a lookup without a trailing dot will search the system's search domain
217 list. Thus, it's important to keep the original host around for use only in
218 those cases where it's appropriate (i.e., when doing DNS lookup to establish the
219 actual TCP connection across which we're going to send HTTP requests).
220 """
221 return self._dns_host.rstrip(".")
223 @host.setter
224 def host(self, value: str) -> None:
225 """
226 Setter for the `host` property.
228 We assume that only urllib3 uses the _dns_host attribute; httplib itself
229 only uses `host`, and it seems reasonable that other libraries follow suit.
230 """
231 self._dns_host = value
233 def _new_conn(self) -> socket.socket:
234 """Establish a socket connection and set nodelay settings on it.
236 :return: New socket connection.
237 """
238 try:
239 sock = connection.create_connection(
240 (self._dns_host, self.port),
241 self.timeout,
242 source_address=self.source_address,
243 socket_options=self.socket_options,
244 )
245 except socket.gaierror as e:
246 raise NameResolutionError(self.host, self, e) from e
247 except SocketTimeout as e:
248 raise ConnectTimeoutError(
249 self,
250 f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
251 ) from e
253 except OSError as e:
254 raise NewConnectionError(
255 self, f"Failed to establish a new connection: {e}"
256 ) from e
258 sys.audit("http.client.connect", self, self.host, self.port)
260 return sock
262 def set_tunnel(
263 self,
264 host: str,
265 port: int | None = None,
266 headers: typing.Mapping[str, str] | None = None,
267 scheme: str = "http",
268 ) -> None:
269 if scheme not in ("http", "https"):
270 raise ValueError(
271 f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
272 )
273 super().set_tunnel(host, port=port, headers=headers)
274 self._tunnel_scheme = scheme
276 if sys.version_info < (3, 11, 16) or ((3, 12) <= sys.version_info < (3, 12, 14)):
277 # Taken from python/cpython#100986 which was backported in 3.11.9 and 3.12.3.
278 # When using connection_from_host, host will come without brackets.
279 # With a security patch from python/cpython#146211.
280 def _wrap_ipv6(self, ip: bytes) -> bytes:
281 if b":" in ip and ip[0] != b"["[0]:
282 return b"[" + ip + b"]"
283 return ip
285 # Copied from CPython 3.12.13 Lib/http/client.py
286 _is_legal_header_name = staticmethod(re.compile(rb"[^:\s][^:\r\n]*").fullmatch)
287 _is_illegal_header_value = staticmethod(
288 re.compile(rb"\n(?![ \t])|\r(?![ \t\n])").search
289 )
290 _contains_disallowed_url_pchar_re = re.compile("[\x00-\x20\x7f]")
292 if sys.version_info < (3, 11, 16):
293 # `_tunnel` copied from 3.11.15 backporting
294 # https://github.com/python/cpython/commit/0d4026432591d43185568dd31cef6a034c4b9261
295 # and https://github.com/python/cpython/commit/6fbc61070fda2ffb8889e77e3b24bca4249ab4d1
296 # plus a fix from https://github.com/python/cpython/commit/56b7100b04e44ea27989242b176beb8f016b2c53
297 def _tunnel(self) -> None:
298 if self._contains_disallowed_url_pchar_re.search(self._tunnel_host): # type: ignore[arg-type]
299 raise ValueError(
300 "Tunnel host can't contain control characters %r"
301 % (self._tunnel_host,)
302 )
303 _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
304 connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
305 self._wrap_ipv6(self._tunnel_host.encode("ascii")), # type: ignore[union-attr]
306 self._tunnel_port,
307 )
308 headers = [connect]
309 for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
310 header_bytes = header.encode("latin-1")
311 value_bytes = value.encode("latin-1")
312 if not self._is_legal_header_name(header_bytes):
313 raise ValueError(f"Invalid header name {header_bytes!r}")
314 if self._is_illegal_header_value(value_bytes):
315 raise ValueError(f"Invalid header value {value_bytes!r}")
316 headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes))
317 headers.append(b"\r\n")
318 # Making a single send() call instead of one per line encourages
319 # the host OS to use a more optimal packet size instead of
320 # potentially emitting a series of small packets.
321 self.send(b"".join(headers))
322 del headers
324 response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
325 try:
326 version, code, message = response._read_status() # type: ignore[attr-defined]
328 if code != http.HTTPStatus.OK:
329 self.close()
330 raise OSError(
331 f"Tunnel connection failed: {code} {message.strip()}"
332 )
333 while True:
334 line = response.fp.readline(_MAXLINE + 1)
335 if len(line) > _MAXLINE:
336 raise http.client.LineTooLong("header line")
337 if not line:
338 # for sites which EOF without sending a trailer
339 break
340 if line in (b"\r\n", b"\n", b""):
341 break
343 if self.debuglevel > 0:
344 print("header:", line.decode())
345 finally:
346 response.close()
348 elif (3, 12) <= sys.version_info < (3, 12, 14):
349 # `_tunnel` copied from 3.12.13 backporting
350 # https://github.com/python/cpython/commit/23aef575c7629abcd4aaf028ebd226fb41a4b3c8
351 # plus a fix from https://github.com/python/cpython/commit/c00c386faa579ad71196d33408644478488e43ec
352 def _tunnel(self) -> None: # noqa: F811
353 if self._contains_disallowed_url_pchar_re.search(self._tunnel_host): # type: ignore[arg-type]
354 raise ValueError(
355 "Tunnel host can't contain control characters %r"
356 % (self._tunnel_host,)
357 )
358 connect = b"CONNECT %s:%d HTTP/1.1\r\n" % ( # type: ignore[str-format]
359 self._wrap_ipv6(self._tunnel_host.encode("idna")), # type: ignore[union-attr]
360 self._tunnel_port,
361 )
362 headers = [connect]
363 for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
364 header_bytes = header.encode("latin-1")
365 value_bytes = value.encode("latin-1")
366 if not self._is_legal_header_name(header_bytes):
367 raise ValueError(f"Invalid header name {header_bytes!r}")
368 if self._is_illegal_header_value(value_bytes):
369 raise ValueError(f"Invalid header value {value_bytes!r}")
370 headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes))
371 headers.append(b"\r\n")
372 # Making a single send() call instead of one per line encourages
373 # the host OS to use a more optimal packet size instead of
374 # potentially emitting a series of small packets.
375 self.send(b"".join(headers))
376 del headers
378 response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
379 try:
380 version, code, message = response._read_status() # type: ignore[attr-defined]
382 self._raw_proxy_headers = http.client._read_headers(response.fp) # type: ignore[attr-defined]
384 if self.debuglevel > 0:
385 for header in self._raw_proxy_headers:
386 print("header:", header.decode())
388 if code != http.HTTPStatus.OK:
389 self.close()
390 raise OSError(
391 f"Tunnel connection failed: {code} {message.strip()}"
392 )
394 finally:
395 response.close()
397 def connect(self) -> None:
398 self.sock = self._new_conn()
399 if self._tunnel_host:
400 # If we're tunneling it means we're connected to our proxy.
401 self._has_connected_to_proxy = True
403 # TODO: Fix tunnel so it doesn't depend on self.sock state.
404 self._tunnel()
406 # If there's a proxy to be connected to we are fully connected.
407 # This is set twice (once above and here) due to forwarding proxies
408 # not using tunnelling.
409 self._has_connected_to_proxy = bool(self.proxy)
411 if self._has_connected_to_proxy:
412 self.proxy_is_verified = False
414 @property
415 def is_closed(self) -> bool:
416 return self.sock is None
418 @property
419 def is_connected(self) -> bool:
420 if self.sock is None:
421 return False
422 return not wait_for_read(self.sock, timeout=0.0)
424 @property
425 def has_connected_to_proxy(self) -> bool:
426 return self._has_connected_to_proxy
428 @property
429 def proxy_is_forwarding(self) -> bool:
430 """
431 Return True if a forwarding proxy is configured, else return False
432 """
433 return bool(self.proxy) and self._tunnel_host is None
435 @property
436 def proxy_is_tunneling(self) -> bool:
437 """
438 Return True if a tunneling proxy is configured, else return False
439 """
440 return self._tunnel_host is not None
442 def close(self) -> None:
443 try:
444 super().close()
445 finally:
446 # Reset all stateful properties so connection
447 # can be re-used without leaking prior configs.
448 self.sock = None
449 self.is_verified = False
450 self.proxy_is_verified = None
451 self._has_connected_to_proxy = False
452 self._response_options = None
453 self._tunnel_host = None
454 self._tunnel_port = None
455 self._tunnel_scheme = None
457 def putrequest(
458 self,
459 method: str,
460 url: str,
461 skip_host: bool = False,
462 skip_accept_encoding: bool = False,
463 ) -> None:
464 """"""
465 # Empty docstring because the indentation of CPython's implementation
466 # is broken but we don't want this method in our documentation.
467 match = _CONTAINS_CONTROL_CHAR_RE.search(method)
468 if match:
469 raise ValueError(
470 f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
471 )
473 return super().putrequest(
474 method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
475 )
477 def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
478 """"""
479 if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
480 super().putheader(header, *values)
481 elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
482 skippable_headers = "', '".join(
483 [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
484 )
485 raise ValueError(
486 f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
487 )
489 # `request` method's signature intentionally violates LSP.
490 # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
491 def request( # type: ignore[override]
492 self,
493 method: str,
494 url: str,
495 body: _TYPE_BODY | None = None,
496 headers: typing.Mapping[str, str] | None = None,
497 *,
498 chunked: bool = False,
499 preload_content: bool = True,
500 decode_content: bool = True,
501 enforce_content_length: bool = True,
502 ) -> None:
503 # Update the inner socket's timeout value to send the request.
504 # This only triggers if the connection is re-used.
505 if self.sock is not None:
506 self.sock.settimeout(self.timeout)
508 # Store these values to be fed into the HTTPResponse
509 # object later. TODO: Remove this in favor of a real
510 # HTTP lifecycle mechanism.
512 # We have to store these before we call .request()
513 # because sometimes we can still salvage a response
514 # off the wire even if we aren't able to completely
515 # send the request body.
516 self._response_options = _ResponseOptions(
517 request_method=method,
518 request_url=url,
519 preload_content=preload_content,
520 decode_content=decode_content,
521 enforce_content_length=enforce_content_length,
522 )
524 if headers is None:
525 headers = {}
526 header_keys = frozenset(to_str(k.lower()) for k in headers)
527 skip_accept_encoding = "accept-encoding" in header_keys
528 skip_host = "host" in header_keys
529 self.putrequest(
530 method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
531 )
533 # Transform the body into an iterable of sendall()-able chunks
534 # and detect if an explicit Content-Length is doable.
535 chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
536 chunks = chunks_and_cl.chunks
537 content_length = chunks_and_cl.content_length
539 # When chunked is explicit set to 'True' we respect that.
540 if chunked:
541 if "transfer-encoding" not in header_keys:
542 self.putheader("Transfer-Encoding", "chunked")
543 else:
544 # Detect whether a framing mechanism is already in use. If so
545 # we respect that value, otherwise we pick chunked vs content-length
546 # depending on the type of 'body'.
547 if "content-length" in header_keys:
548 chunked = False
549 elif "transfer-encoding" in header_keys:
550 chunked = True
552 # Otherwise we go off the recommendation of 'body_to_chunks()'.
553 else:
554 chunked = False
555 if content_length is None:
556 if chunks is not None:
557 chunked = True
558 self.putheader("Transfer-Encoding", "chunked")
559 else:
560 self.putheader("Content-Length", str(content_length))
562 # Now that framing headers are out of the way we send all the other headers.
563 if "user-agent" not in header_keys:
564 self.putheader("User-Agent", _get_default_user_agent())
565 for header, value in headers.items():
566 self.putheader(header, value)
567 self.endheaders()
569 # If we're given a body we start sending that in chunks.
570 if chunks is not None:
571 for chunk in chunks:
572 # Sending empty chunks isn't allowed for TE: chunked
573 # as it indicates the end of the body.
574 if not chunk:
575 continue
576 if isinstance(chunk, str):
577 chunk = chunk.encode("utf-8")
578 if chunked:
579 self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
580 else:
581 self.send(chunk)
583 # Regardless of whether we have a body or not, if we're in
584 # chunked mode we want to send an explicit empty chunk.
585 if chunked:
586 self.send(b"0\r\n\r\n")
588 def request_chunked(
589 self,
590 method: str,
591 url: str,
592 body: _TYPE_BODY | None = None,
593 headers: typing.Mapping[str, str] | None = None,
594 ) -> None:
595 """
596 Alternative to the common request method, which sends the
597 body with chunked encoding and not as one block
598 """
599 warnings.warn(
600 "HTTPConnection.request_chunked() is deprecated and will be removed "
601 "in urllib3 v3.0. Instead use HTTPConnection.request(..., chunked=True).",
602 category=FutureWarning,
603 stacklevel=2,
604 )
605 self.request(method, url, body=body, headers=headers, chunked=True)
607 def getresponse( # type: ignore[override]
608 self,
609 ) -> HTTPResponse:
610 """
611 Get the response from the server.
613 If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
615 If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
616 """
617 # Raise the same error as http.client.HTTPConnection
618 if self._response_options is None:
619 raise ResponseNotReady()
621 # Reset this attribute for being used again.
622 resp_options = self._response_options
623 self._response_options = None
625 # Since the connection's timeout value may have been updated
626 # we need to set the timeout on the socket.
627 self.sock.settimeout(self.timeout)
629 # This is needed here to avoid circular import errors
630 from .response import HTTPResponse
632 # Save a reference to the shutdown function before ownership is passed
633 # to httplib_response
634 # TODO should we implement it everywhere?
635 _shutdown = getattr(self.sock, "shutdown", None)
637 # Get the response from http.client.HTTPConnection
638 httplib_response = super().getresponse()
640 try:
641 assert_header_parsing(httplib_response.msg)
642 except (HeaderParsingError, TypeError) as hpe:
643 log.warning(
644 "Failed to parse headers (url=%s): %s",
645 _url_from_connection(self, resp_options.request_url),
646 hpe,
647 exc_info=True,
648 )
650 header_items = _normalize_header_values(httplib_response.msg)
651 headers = HTTPHeaderDict(header_items)
653 response = HTTPResponse(
654 body=httplib_response,
655 headers=headers,
656 status=httplib_response.status,
657 version=httplib_response.version,
658 version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
659 reason=httplib_response.reason,
660 preload_content=resp_options.preload_content,
661 decode_content=resp_options.decode_content,
662 original_response=httplib_response,
663 enforce_content_length=resp_options.enforce_content_length,
664 request_method=resp_options.request_method,
665 request_url=resp_options.request_url,
666 sock_shutdown=_shutdown,
667 )
668 return response
671class HTTPSConnection(HTTPConnection):
672 """
673 Many of the parameters to this constructor are passed to the underlying SSL
674 socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
675 """
677 default_port = port_by_scheme["https"] # type: ignore[misc]
679 cert_reqs: int | str | None = None
680 ca_certs: str | None = None
681 ca_cert_dir: str | None = None
682 ca_cert_data: None | str | bytes = None
683 ssl_version: int | str | None = None
684 ssl_minimum_version: int | None = None
685 ssl_maximum_version: int | None = None
686 assert_fingerprint: str | None = None
687 _connect_callback: typing.Callable[..., None] | None = None
689 def __init__(
690 self,
691 host: str,
692 port: int | None = None,
693 *,
694 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
695 source_address: tuple[str, int] | None = None,
696 blocksize: int = 16384,
697 socket_options: None | (
698 connection._TYPE_SOCKET_OPTIONS
699 ) = HTTPConnection.default_socket_options,
700 proxy: Url | None = None,
701 proxy_config: ProxyConfig | None = None,
702 cert_reqs: int | str | None = None,
703 assert_hostname: None | str | typing.Literal[False] = None,
704 assert_fingerprint: str | None = None,
705 server_hostname: str | None = None,
706 ssl_context: ssl.SSLContext | None = None,
707 ca_certs: str | None = None,
708 ca_cert_dir: str | None = None,
709 ca_cert_data: None | str | bytes = None,
710 ssl_minimum_version: int | None = None,
711 ssl_maximum_version: int | None = None,
712 ssl_version: int | str | None = None, # Deprecated
713 cert_file: str | None = None,
714 key_file: str | None = None,
715 key_password: str | None = None,
716 ) -> None:
717 super().__init__(
718 host,
719 port=port,
720 timeout=timeout,
721 source_address=source_address,
722 blocksize=blocksize,
723 socket_options=socket_options,
724 proxy=proxy,
725 proxy_config=proxy_config,
726 )
728 self.key_file = key_file
729 self.cert_file = cert_file
730 self.key_password = key_password
731 self.ssl_context = ssl_context
732 self.server_hostname = server_hostname
733 self.assert_hostname = assert_hostname
734 self.assert_fingerprint = assert_fingerprint
735 self.ssl_version = ssl_version
736 self.ssl_minimum_version = ssl_minimum_version
737 self.ssl_maximum_version = ssl_maximum_version
738 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
739 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
740 self.ca_cert_data = ca_cert_data
742 # cert_reqs depends on ssl_context so calculate last.
743 if cert_reqs is None:
744 if self.ssl_context is not None:
745 cert_reqs = self.ssl_context.verify_mode
746 else:
747 cert_reqs = resolve_cert_reqs(None)
748 self.cert_reqs = cert_reqs
749 self._connect_callback = None
751 def set_cert(
752 self,
753 key_file: str | None = None,
754 cert_file: str | None = None,
755 cert_reqs: int | str | None = None,
756 key_password: str | None = None,
757 ca_certs: str | None = None,
758 assert_hostname: None | str | typing.Literal[False] = None,
759 assert_fingerprint: str | None = None,
760 ca_cert_dir: str | None = None,
761 ca_cert_data: None | str | bytes = None,
762 ) -> None:
763 """
764 This method should only be called once, before the connection is used.
765 """
766 warnings.warn(
767 "HTTPSConnection.set_cert() is deprecated and will be removed "
768 "in urllib3 v3.0. Instead provide the parameters to the "
769 "HTTPSConnection constructor.",
770 category=FutureWarning,
771 stacklevel=2,
772 )
774 # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
775 # have an SSLContext object in which case we'll use its verify_mode.
776 if cert_reqs is None:
777 if self.ssl_context is not None:
778 cert_reqs = self.ssl_context.verify_mode
779 else:
780 cert_reqs = resolve_cert_reqs(None)
782 self.key_file = key_file
783 self.cert_file = cert_file
784 self.cert_reqs = cert_reqs
785 self.key_password = key_password
786 self.assert_hostname = assert_hostname
787 self.assert_fingerprint = assert_fingerprint
788 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
789 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
790 self.ca_cert_data = ca_cert_data
792 def connect(self) -> None:
793 # Today we don't need to be doing this step before the /actual/ socket
794 # connection, however in the future we'll need to decide whether to
795 # create a new socket or re-use an existing "shared" socket as a part
796 # of the HTTP/2 handshake dance.
797 if self._tunnel_host is not None and self._tunnel_port is not None:
798 probe_http2_host = self._tunnel_host
799 probe_http2_port = self._tunnel_port
800 else:
801 probe_http2_host = self.host
802 probe_http2_port = self.port
804 # Check if the target origin supports HTTP/2.
805 # If the value comes back as 'None' it means that the current thread
806 # is probing for HTTP/2 support. Otherwise, we're waiting for another
807 # probe to complete, or we get a value right away.
808 target_supports_http2: bool | None
809 if "h2" in ssl_.ALPN_PROTOCOLS:
810 target_supports_http2 = http2_probe.acquire_and_get(
811 host=probe_http2_host, port=probe_http2_port
812 )
813 else:
814 # If HTTP/2 isn't going to be offered it doesn't matter if
815 # the target supports HTTP/2. Don't want to make a probe.
816 target_supports_http2 = False
818 if self._connect_callback is not None:
819 self._connect_callback(
820 "before connect",
821 thread_id=threading.get_ident(),
822 target_supports_http2=target_supports_http2,
823 )
825 try:
826 sock: socket.socket | ssl.SSLSocket
827 self.sock = sock = self._new_conn()
828 server_hostname: str = self.host
829 tls_in_tls = False
831 # Do we need to establish a tunnel?
832 if self.proxy_is_tunneling:
833 # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
834 if self._tunnel_scheme == "https":
835 # _connect_tls_proxy will verify and assign proxy_is_verified
836 self.sock = sock = self._connect_tls_proxy(self.host, sock)
837 tls_in_tls = True
838 elif self._tunnel_scheme == "http":
839 self.proxy_is_verified = False
841 # If we're tunneling it means we're connected to our proxy.
842 self._has_connected_to_proxy = True
844 self._tunnel()
845 # Override the host with the one we're requesting data from.
846 server_hostname = typing.cast(str, self._tunnel_host)
848 if self.server_hostname is not None:
849 server_hostname = self.server_hostname
851 is_time_off = datetime.date.today() < RECENT_DATE
852 if is_time_off:
853 warnings.warn(
854 (
855 f"System time is way off (before {RECENT_DATE}). This will probably "
856 "lead to SSL verification errors"
857 ),
858 SystemTimeWarning,
859 )
861 # Remove trailing '.' from fqdn hostnames to allow certificate validation
862 server_hostname_rm_dot = server_hostname.rstrip(".")
864 # Forwarding proxies should use proxy SSL context for
865 # wrapping since that's the connection being established,
866 # whereas tunneling proxies should use the connection's SSL
867 # context.
868 # However, for backwards compatibility reasons, if the proxy
869 # is forwarding but no proxy SSL context is provided, we
870 # fall back to using the connection's SSL context until
871 # urllib3 v3.0. Appropriate warning is emitted in
872 # ``ProxyManager.__init__``.
873 wrapped_socket: ssl.SSLSocket | SSLTransport
874 if self.proxy_is_forwarding and self.proxy_config is not None:
875 wrapped_socket = self._connect_tls_proxy(self.host, sock)
876 is_verified = self.proxy_is_verified is True
877 else:
878 wrapped_socket, is_verified = _ssl_wrap_socket_and_match_hostname(
879 sock=sock,
880 cert_reqs=self.cert_reqs,
881 ssl_version=self.ssl_version,
882 ssl_minimum_version=self.ssl_minimum_version,
883 ssl_maximum_version=self.ssl_maximum_version,
884 ca_certs=self.ca_certs,
885 ca_cert_dir=self.ca_cert_dir,
886 ca_cert_data=self.ca_cert_data,
887 cert_file=self.cert_file,
888 key_file=self.key_file,
889 key_password=self.key_password,
890 server_hostname=server_hostname_rm_dot,
891 ssl_context=self.ssl_context,
892 tls_in_tls=tls_in_tls,
893 assert_hostname=self.assert_hostname,
894 assert_fingerprint=self.assert_fingerprint,
895 )
896 self.sock = wrapped_socket
898 # If an error occurs during connection/handshake we may need to release
899 # our lock so another connection can probe the origin.
900 except BaseException:
901 if self._connect_callback is not None:
902 self._connect_callback(
903 "after connect failure",
904 thread_id=threading.get_ident(),
905 target_supports_http2=target_supports_http2,
906 )
908 if target_supports_http2 is None:
909 http2_probe.set_and_release(
910 host=probe_http2_host, port=probe_http2_port, supports_http2=None
911 )
912 raise
914 # If this connection doesn't know if the origin supports HTTP/2
915 # we report back to the HTTP/2 probe our result.
916 if target_supports_http2 is None:
917 supports_http2 = wrapped_socket.selected_alpn_protocol() == "h2"
918 http2_probe.set_and_release(
919 host=probe_http2_host,
920 port=probe_http2_port,
921 supports_http2=supports_http2,
922 )
924 # Forwarding proxies can never have a verified target since
925 # the proxy is the one doing the verification. Should instead
926 # use a CONNECT tunnel in order to verify the target.
927 # See: https://github.com/urllib3/urllib3/issues/3267.
928 if self.proxy_is_forwarding:
929 self.is_verified = False
930 else:
931 self.is_verified = is_verified
933 # If there's a proxy to be connected to we are fully connected.
934 # This is set twice (once above and here) due to forwarding proxies
935 # not using tunnelling.
936 self._has_connected_to_proxy = bool(self.proxy)
938 # Set `self.proxy_is_verified` unless it's already set while
939 # establishing a tunnel.
940 if self._has_connected_to_proxy and self.proxy_is_verified is None:
941 self.proxy_is_verified = is_verified
943 def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
944 """
945 Establish a TLS connection to the proxy using proxy-specific policy.
946 """
947 proxy_config = typing.cast(ProxyConfig, self.proxy_config)
948 proxy_ssl_context = proxy_config.ssl_context
950 ssl_context: ssl.SSLContext | None
951 cert_reqs: int | str | None
952 if proxy_ssl_context is not None:
953 # Prefer the proxy's cert policy for the proxy connection
954 ssl_context = proxy_ssl_context
955 cert_reqs = proxy_ssl_context.verify_mode
956 ca_certs = None
957 ca_cert_dir = None
958 ca_cert_data = None
959 ssl_version = None
960 ssl_minimum_version = None
961 ssl_maximum_version = None
962 else:
963 # Otherwise we inherit the pool's cert policies
964 ssl_context = self.ssl_context if self.proxy_is_forwarding else None
965 cert_reqs = self.cert_reqs
966 ca_certs = self.ca_certs
967 ca_cert_dir = self.ca_cert_dir
968 ca_cert_data = self.ca_cert_data
969 ssl_version = self.ssl_version
970 ssl_minimum_version = self.ssl_minimum_version
971 ssl_maximum_version = self.ssl_maximum_version
973 sock_and_verified = _ssl_wrap_socket_and_match_hostname(
974 sock,
975 cert_reqs=cert_reqs,
976 ssl_version=ssl_version,
977 ssl_minimum_version=ssl_minimum_version,
978 ssl_maximum_version=ssl_maximum_version,
979 ca_certs=ca_certs,
980 ca_cert_dir=ca_cert_dir,
981 ca_cert_data=ca_cert_data,
982 server_hostname=hostname,
983 ssl_context=ssl_context,
984 assert_hostname=proxy_config.assert_hostname,
985 assert_fingerprint=proxy_config.assert_fingerprint,
986 # Features that aren't implemented for proxies yet:
987 cert_file=None,
988 key_file=None,
989 key_password=None,
990 tls_in_tls=False,
991 )
992 self.proxy_is_verified = sock_and_verified.is_verified
993 return sock_and_verified.socket # type: ignore[return-value]
996class _WrappedAndVerifiedSocket(typing.NamedTuple):
997 """
998 Wrapped socket and whether the connection is
999 verified after the TLS handshake
1000 """
1002 socket: ssl.SSLSocket | SSLTransport
1003 is_verified: bool
1006def _ssl_wrap_socket_and_match_hostname(
1007 sock: socket.socket,
1008 *,
1009 cert_reqs: None | str | int,
1010 ssl_version: None | str | int,
1011 ssl_minimum_version: int | None,
1012 ssl_maximum_version: int | None,
1013 cert_file: str | None,
1014 key_file: str | None,
1015 key_password: str | None,
1016 ca_certs: str | None,
1017 ca_cert_dir: str | None,
1018 ca_cert_data: None | str | bytes,
1019 assert_hostname: None | str | typing.Literal[False],
1020 assert_fingerprint: str | None,
1021 server_hostname: str | None,
1022 ssl_context: ssl.SSLContext | None,
1023 tls_in_tls: bool = False,
1024) -> _WrappedAndVerifiedSocket:
1025 """Logic for constructing an SSLContext from all TLS parameters, passing
1026 that down into ssl_wrap_socket, and then doing certificate verification
1027 either via hostname or fingerprint. This function exists to guarantee
1028 that both proxies and targets have the same behavior when connecting via TLS.
1029 """
1030 default_ssl_context = False
1031 if ssl_context is None:
1032 default_ssl_context = True
1033 context = create_urllib3_context(
1034 ssl_version=resolve_ssl_version(ssl_version),
1035 ssl_minimum_version=ssl_minimum_version,
1036 ssl_maximum_version=ssl_maximum_version,
1037 cert_reqs=resolve_cert_reqs(cert_reqs),
1038 )
1039 else:
1040 context = ssl_context
1042 context.verify_mode = resolve_cert_reqs(cert_reqs)
1044 # In some cases, we want to verify hostnames ourselves
1045 if (
1046 # `ssl` can't verify fingerprints or alternate hostnames
1047 assert_fingerprint
1048 or assert_hostname
1049 # assert_hostname can be set to False to disable hostname checking
1050 or assert_hostname is False
1051 # We still support OpenSSL 1.0.2, which prevents us from verifying
1052 # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
1053 or ssl_.IS_PYOPENSSL
1054 or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
1055 ):
1056 context.check_hostname = False
1058 # Try to load OS default certs if none are given. We need to do the hasattr() check
1059 # for custom pyOpenSSL SSLContext objects because they don't support
1060 # load_default_certs().
1061 if (
1062 not ca_certs
1063 and not ca_cert_dir
1064 and not ca_cert_data
1065 and default_ssl_context
1066 and hasattr(context, "load_default_certs")
1067 ):
1068 context.load_default_certs()
1070 # Ensure that IPv6 addresses are in the proper format and don't have a
1071 # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
1072 # and interprets them as DNS hostnames.
1073 if server_hostname is not None:
1074 normalized = server_hostname.strip("[]")
1075 if "%" in normalized:
1076 normalized = normalized[: normalized.rfind("%")]
1077 if is_ipaddress(normalized):
1078 server_hostname = normalized
1080 ssl_sock = ssl_wrap_socket(
1081 sock=sock,
1082 keyfile=key_file,
1083 certfile=cert_file,
1084 key_password=key_password,
1085 ca_certs=ca_certs,
1086 ca_cert_dir=ca_cert_dir,
1087 ca_cert_data=ca_cert_data,
1088 server_hostname=server_hostname,
1089 ssl_context=context,
1090 tls_in_tls=tls_in_tls,
1091 )
1093 try:
1094 if assert_fingerprint:
1095 _assert_fingerprint(
1096 ssl_sock.getpeercert(binary_form=True), assert_fingerprint
1097 )
1098 elif (
1099 context.verify_mode != ssl.CERT_NONE
1100 and not context.check_hostname
1101 and assert_hostname is not False
1102 ):
1103 cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
1105 # Need to signal to our match_hostname whether to use 'commonName' or not.
1106 # If we're using our own constructed SSLContext we explicitly set 'False'
1107 # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
1108 if default_ssl_context:
1109 hostname_checks_common_name = False
1110 else:
1111 hostname_checks_common_name = (
1112 getattr(context, "hostname_checks_common_name", False) or False
1113 )
1115 _match_hostname(
1116 cert,
1117 assert_hostname or server_hostname, # type: ignore[arg-type]
1118 hostname_checks_common_name,
1119 )
1121 return _WrappedAndVerifiedSocket(
1122 socket=ssl_sock,
1123 is_verified=context.verify_mode == ssl.CERT_REQUIRED
1124 or bool(assert_fingerprint),
1125 )
1126 except BaseException:
1127 ssl_sock.close()
1128 raise
1131def _match_hostname(
1132 cert: _TYPE_PEER_CERT_RET_DICT | None,
1133 asserted_hostname: str,
1134 hostname_checks_common_name: bool = False,
1135) -> None:
1136 # Our upstream implementation of ssl.match_hostname()
1137 # only applies this normalization to IP addresses so it doesn't
1138 # match DNS SANs so we do the same thing!
1139 stripped_hostname = asserted_hostname.strip("[]")
1140 if is_ipaddress(stripped_hostname):
1141 asserted_hostname = stripped_hostname
1143 try:
1144 match_hostname(cert, asserted_hostname, hostname_checks_common_name)
1145 except CertificateError as e:
1146 log.warning(
1147 "Certificate did not match expected hostname: %s. Certificate: %s",
1148 asserted_hostname,
1149 cert,
1150 )
1151 # Add cert to exception and reraise so client code can inspect
1152 # the cert when catching the exception, if they want to
1153 e._peer_cert = cert # type: ignore[attr-defined]
1154 raise
1157def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
1158 # Look for the phrase 'wrong version number', if found
1159 # then we should warn the user that we're very sure that
1160 # this proxy is HTTP-only and they have a configuration issue.
1161 error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
1162 is_likely_http_proxy = (
1163 "wrong version number" in error_normalized
1164 or "unknown protocol" in error_normalized
1165 or "record layer failure" in error_normalized
1166 )
1167 http_proxy_warning = (
1168 ". Your proxy appears to only use HTTP and not HTTPS, "
1169 "try changing your proxy URL to be HTTP. See: "
1170 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1171 "#https-proxy-error-http-proxy"
1172 )
1173 new_err = ProxyError(
1174 f"Unable to connect to proxy"
1175 f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
1176 err,
1177 )
1178 new_err.__cause__ = err
1179 return new_err
1182def _get_default_user_agent() -> str:
1183 return f"python-urllib3/{__version__}"
1186class DummyConnection:
1187 """Used to detect a failed ConnectionCls import."""
1190if not ssl:
1191 HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
1194VerifiedHTTPSConnection = HTTPSConnection
1197def _url_from_connection(
1198 conn: HTTPConnection | HTTPSConnection, path: str | None = None
1199) -> str:
1200 """Returns the URL from a given connection. This is mainly used for testing and logging."""
1202 scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
1204 return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url