Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import errno
4import logging
5import queue
6import sys
7import typing
8import warnings
9import weakref
10from socket import timeout as SocketTimeout
11from types import TracebackType
13from ._base_connection import _TYPE_BODY
14from ._collections import HTTPHeaderDict
15from ._request_methods import RequestMethods
16from .connection import (
17 BaseSSLError,
18 BrokenPipeError,
19 DummyConnection,
20 HTTPConnection,
21 HTTPException,
22 HTTPSConnection,
23 ProxyConfig,
24 _wrap_proxy_error,
25)
26from .connection import port_by_scheme as port_by_scheme
27from .exceptions import (
28 ClosedPoolError,
29 EmptyPoolError,
30 FullPoolError,
31 HostChangedError,
32 InsecureRequestWarning,
33 LocationValueError,
34 MaxRetryError,
35 NewConnectionError,
36 ProtocolError,
37 ProxyError,
38 ReadTimeoutError,
39 SSLError,
40 TimeoutError,
41)
42from .response import BaseHTTPResponse
43from .util.connection import is_connection_dropped
44from .util.proxy import connection_requires_http_tunnel
45from .util.request import _TYPE_BODY_POSITION, set_file_position
46from .util.retry import Retry
47from .util.ssl_match_hostname import CertificateError
48from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
49from .util.url import Url, _encode_target
50from .util.url import _normalize_host as normalize_host
51from .util.url import parse_url
52from .util.util import to_str
54if typing.TYPE_CHECKING:
55 import ssl
57 from typing_extensions import Self
59 from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
61log = logging.getLogger(__name__)
63_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
64_DEFAULT_QUEUE_CLASS = queue.LifoQueue
67# Pool objects
68class ConnectionPool:
69 """
70 Base class for all connection pools, such as
71 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
73 .. note::
74 ConnectionPool.urlopen() does not normalize or percent-encode target URIs
75 which is useful if your target server doesn't support percent-encoded
76 target URIs.
77 """
79 scheme: str | None = None
80 QueueCls = _DEFAULT_QUEUE_CLASS
82 def __init__(self, host: str, port: int | None = None) -> None:
83 if not host:
84 raise LocationValueError("No host specified.")
86 self.host = _normalize_host(host, scheme=self.scheme)
87 self.port = port
89 # This property uses 'normalize_host()' (not '_normalize_host()')
90 # to avoid removing square braces around IPv6 addresses.
91 # This value is sent to `HTTPConnection.set_tunnel()` if called
92 # because square braces are required for HTTP CONNECT tunneling.
93 self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()
95 def __str__(self) -> str:
96 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
98 def __enter__(self) -> Self:
99 return self
101 def __exit__(
102 self,
103 exc_type: type[BaseException] | None,
104 exc_val: BaseException | None,
105 exc_tb: TracebackType | None,
106 ) -> typing.Literal[False]:
107 self.close()
108 # Return False to re-raise any potential exceptions
109 return False
111 def close(self) -> None:
112 """
113 Close all pooled connections and disable the pool.
114 """
116 def _new_pool_queue(self, maxsize: int) -> queue.LifoQueue[typing.Any]:
117 if self.QueueCls is _DEFAULT_QUEUE_CLASS:
118 return queue.LifoQueue(maxsize)
119 return self.QueueCls(maxsize)
122# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
123_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
126class HTTPConnectionPool(ConnectionPool, RequestMethods):
127 """
128 Thread-safe connection pool for one host.
130 :param host:
131 Host used for this HTTP Connection (e.g. "localhost"), passed into
132 :class:`http.client.HTTPConnection`.
134 :param port:
135 Port used for this HTTP Connection (None is equivalent to 80), passed
136 into :class:`http.client.HTTPConnection`.
138 :param timeout:
139 Socket timeout in seconds for each individual connection. This can
140 be a float or integer, which sets the timeout for the HTTP request,
141 or an instance of :class:`urllib3.util.Timeout` which gives you more
142 fine-grained control over request timeouts. After the constructor has
143 been parsed, this is always a `urllib3.util.Timeout` object.
145 :param maxsize:
146 Number of connections to save that can be reused. More than 1 is useful
147 in multithreaded situations. If ``block`` is set to False, more
148 connections will be created but they will not be saved once they've
149 been used.
151 :param block:
152 If set to True, no more than ``maxsize`` connections will be used at
153 a time. When no free connections are available, the call will block
154 until a connection has been released. This is a useful side effect for
155 particular multithreaded situations where one does not want to use more
156 than maxsize connections per host to prevent flooding.
158 :param headers:
159 Headers to include with all requests, unless other headers are given
160 explicitly.
162 :param retries:
163 Retry configuration to use by default with requests in this pool.
165 :param _proxy:
166 Parsed proxy URL, should not be used directly, instead, see
167 :class:`urllib3.ProxyManager`
169 :param _proxy_headers:
170 A dictionary with proxy headers, should not be used directly,
171 instead, see :class:`urllib3.ProxyManager`
173 :param \\**conn_kw:
174 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
175 :class:`urllib3.connection.HTTPSConnection` instances.
176 """
178 scheme = "http"
179 ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection
181 def __init__(
182 self,
183 host: str,
184 port: int | None = None,
185 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
186 maxsize: int = 1,
187 block: bool = False,
188 headers: typing.Mapping[str, str] | None = None,
189 retries: Retry | bool | int | None = None,
190 _proxy: Url | None = None,
191 _proxy_headers: typing.Mapping[str, str] | None = None,
192 _proxy_config: ProxyConfig | None = None,
193 **conn_kw: typing.Any,
194 ):
195 ConnectionPool.__init__(self, host, port)
196 RequestMethods.__init__(self, headers)
198 if not isinstance(timeout, Timeout):
199 timeout = Timeout.from_float(timeout)
201 if retries is None:
202 retries = Retry.DEFAULT
204 self.timeout = timeout
205 self.retries = retries
207 self.pool: queue.LifoQueue[typing.Any] | None = self._new_pool_queue(maxsize)
208 self.block = block
210 self.proxy = _proxy
211 self.proxy_headers = _proxy_headers or {}
212 self.proxy_config = _proxy_config
214 # Fill the queue up so that doing get() on it will block properly
215 for _ in range(maxsize):
216 self.pool.put(None)
218 # These are mostly for testing and debugging purposes.
219 self.num_connections = 0
220 self.num_requests = 0
221 self.conn_kw = conn_kw
223 if self.proxy:
224 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
225 # Defaulting `socket_options` to an empty list avoids it defaulting to
226 # ``HTTPConnection.default_socket_options``.
227 self.conn_kw.setdefault("socket_options", [])
229 self.conn_kw["proxy"] = self.proxy
230 self.conn_kw["proxy_config"] = self.proxy_config
232 # Do not pass 'self' as callback to 'finalize'.
233 # Then the 'finalize' would keep an endless living (leak) to self.
234 # By just passing a reference to the pool allows the garbage collector
235 # to free self if nobody else has a reference to it.
236 pool = self.pool
238 # Close all the HTTPConnections in the pool before the
239 # HTTPConnectionPool object is garbage collected.
240 weakref.finalize(self, _close_pool_connections, pool)
242 def _new_conn(self) -> BaseHTTPConnection:
243 """
244 Return a fresh :class:`HTTPConnection`.
245 """
246 self.num_connections += 1
247 log.debug(
248 "Starting new HTTP connection (%d): %s:%s",
249 self.num_connections,
250 self.host,
251 self.port or "80",
252 )
254 conn = self.ConnectionCls(
255 host=self.host,
256 port=self.port,
257 timeout=self.timeout.connect_timeout,
258 **self.conn_kw,
259 )
260 return conn
262 def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
263 """
264 Get a connection. Will return a pooled connection if one is available.
266 If no connections are available and :prop:`.block` is ``False``, then a
267 fresh connection is returned.
269 :param timeout:
270 Seconds to wait before giving up and raising
271 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
272 :prop:`.block` is ``True``.
273 """
274 conn = None
276 if self.pool is None:
277 raise ClosedPoolError(self, "Pool is closed.")
279 try:
280 conn = self.pool.get(block=self.block, timeout=timeout)
282 except AttributeError: # self.pool is None
283 raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
285 except queue.Empty:
286 if self.block:
287 raise EmptyPoolError(
288 self,
289 "Pool is empty and a new connection can't be opened due to blocking mode.",
290 ) from None
291 pass # Oh well, we'll create a new connection then
293 # If this is a persistent connection, check if it got disconnected
294 if conn and is_connection_dropped(conn):
295 log.debug("Resetting dropped connection: %s", self.host)
296 conn.close()
298 return conn or self._new_conn()
300 def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
301 """
302 Put a connection back into the pool.
304 :param conn:
305 Connection object for the current host and port as returned by
306 :meth:`._new_conn` or :meth:`._get_conn`.
308 If the pool is already full, the connection is closed and discarded
309 because we exceeded maxsize. If connections are discarded frequently,
310 then maxsize should be increased.
312 If the pool is closed, then the connection will be closed and discarded.
313 """
314 if self.pool is not None:
315 try:
316 self.pool.put(conn, block=False)
317 return # Everything is dandy, done.
318 except AttributeError:
319 # self.pool is None.
320 pass
321 except queue.Full:
322 # Connection never got put back into the pool, close it.
323 if conn:
324 conn.close()
326 if self.block:
327 # This should never happen if you got the conn from self._get_conn
328 raise FullPoolError(
329 self,
330 "Pool reached maximum size and no more connections are allowed.",
331 ) from None
333 log.warning(
334 "Connection pool is full, discarding connection: %s. Connection pool size: %s",
335 self.host,
336 self.pool.qsize(),
337 )
339 # Connection never got put back into the pool, close it.
340 if conn:
341 conn.close()
343 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
344 """
345 Called right before a request is made, after the socket is created.
346 """
348 def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
349 # Nothing to do for HTTP connections.
350 pass
352 def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
353 """Helper that always returns a :class:`urllib3.util.Timeout`"""
354 if timeout is _DEFAULT_TIMEOUT:
355 return self.timeout.clone()
357 if isinstance(timeout, Timeout):
358 return timeout.clone()
359 else:
360 # User passed us an int/float. This is for backwards compatibility,
361 # can be removed later
362 return Timeout.from_float(timeout)
364 def _raise_timeout(
365 self,
366 err: BaseSSLError | OSError | SocketTimeout,
367 url: str,
368 timeout_value: _TYPE_TIMEOUT | None,
369 ) -> None:
370 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
372 if isinstance(err, SocketTimeout):
373 raise ReadTimeoutError(
374 self, url, f"Read timed out. (read timeout={timeout_value})"
375 ) from err
377 # See the above comment about EAGAIN in Python 3.
378 if hasattr(err, "errno") and err.errno in _blocking_errnos:
379 raise ReadTimeoutError(
380 self, url, f"Read timed out. (read timeout={timeout_value})"
381 ) from err
383 def _make_request(
384 self,
385 conn: BaseHTTPConnection,
386 method: str,
387 url: str,
388 body: _TYPE_BODY | None = None,
389 headers: typing.Mapping[str, str] | None = None,
390 retries: Retry | None = None,
391 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
392 chunked: bool = False,
393 response_conn: BaseHTTPConnection | None = None,
394 preload_content: bool = True,
395 decode_content: bool = True,
396 enforce_content_length: bool = True,
397 ) -> BaseHTTPResponse:
398 """
399 Perform a request on a given urllib connection object taken from our
400 pool.
402 :param conn:
403 a connection from one of our connection pools
405 :param method:
406 HTTP request method (such as GET, POST, PUT, etc.)
408 :param url:
409 The URL to perform the request on.
411 :param body:
412 Data to send in the request body, either :class:`str`, :class:`bytes`,
413 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
415 :param headers:
416 Dictionary of custom headers to send, such as User-Agent,
417 If-None-Match, etc. If None, pool headers are used. If provided,
418 these headers completely replace any pool-specific headers.
420 :param retries:
421 Configure the number of retries to allow before raising a
422 :class:`~urllib3.exceptions.MaxRetryError` exception.
424 Pass ``None`` to retry until you receive a response. Pass a
425 :class:`~urllib3.util.retry.Retry` object for fine-grained control
426 over different types of retries.
427 Pass an integer number to retry connection errors that many times,
428 but no other types of errors. Pass zero to never retry.
430 If ``False``, then retries are disabled and any exception is raised
431 immediately. Also, instead of raising a MaxRetryError on redirects,
432 the redirect response will be returned.
434 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
436 :param timeout:
437 If specified, overrides the default timeout for this one
438 request. It may be a float (in seconds) or an instance of
439 :class:`urllib3.util.Timeout`.
441 :param chunked:
442 If True, urllib3 will send the body using chunked transfer
443 encoding. Otherwise, urllib3 will send the body using the standard
444 content-length form. Defaults to False.
446 :param response_conn:
447 Set this to ``None`` if you will handle releasing the connection or
448 set the connection to have the response release it.
450 :param preload_content:
451 If True, the response's body will be preloaded during construction.
453 :param decode_content:
454 If True, will attempt to decode the body based on the
455 'content-encoding' header.
457 :param enforce_content_length:
458 Enforce content length checking. Body returned by server must match
459 value of Content-Length header, if present. Otherwise, raise error.
460 """
461 self.num_requests += 1
463 timeout_obj = self._get_timeout(timeout)
464 timeout_obj.start_connect()
465 conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
467 try:
468 # Trigger any extra validation we need to do.
469 try:
470 self._validate_conn(conn)
471 except (SocketTimeout, BaseSSLError) as e:
472 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
473 raise
475 # _validate_conn() starts the connection to an HTTPS proxy
476 # so we need to wrap errors with 'ProxyError' here too.
477 except (
478 OSError,
479 NewConnectionError,
480 TimeoutError,
481 BaseSSLError,
482 CertificateError,
483 SSLError,
484 ) as e:
485 new_e: Exception = e
486 if isinstance(e, (BaseSSLError, CertificateError)):
487 new_e = SSLError(e)
488 # If the connection didn't successfully connect to it's proxy
489 # then there
490 if isinstance(
491 new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
492 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
493 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
494 raise new_e
496 # conn.request() calls http.client.*.request, not the method in
497 # urllib3.request. It also calls makefile (recv) on the socket.
498 try:
499 conn.request(
500 method,
501 url,
502 body=body,
503 headers=headers,
504 chunked=chunked,
505 preload_content=preload_content,
506 decode_content=decode_content,
507 enforce_content_length=enforce_content_length,
508 )
510 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
511 # legitimately able to close the connection after sending a valid response.
512 # With this behaviour, the received response is still readable.
513 except BrokenPipeError:
514 pass
515 except OSError as e:
516 # MacOS/Linux
517 # EPROTOTYPE and ECONNRESET are needed on macOS
518 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
519 # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
520 if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
521 raise
523 # Reset the timeout for the recv() on the socket
524 read_timeout = timeout_obj.read_timeout
526 if not conn.is_closed:
527 # In Python 3 socket.py will catch EAGAIN and return None when you
528 # try and read into the file pointer created by http.client, which
529 # instead raises a BadStatusLine exception. Instead of catching
530 # the exception and assuming all BadStatusLine exceptions are read
531 # timeouts, check for a zero timeout before making the request.
532 if read_timeout == 0:
533 raise ReadTimeoutError(
534 self, url, f"Read timed out. (read timeout={read_timeout})"
535 )
536 conn.timeout = read_timeout
538 # Receive the response from the server
539 try:
540 response = conn.getresponse()
541 except (BaseSSLError, OSError) as e:
542 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
543 raise
545 # Set properties that are used by the pooling layer.
546 response.retries = retries
547 response._connection = response_conn # type: ignore[attr-defined]
548 response._pool = self # type: ignore[attr-defined]
550 log.debug(
551 '%s://%s:%s "%s %s %s" %s %s',
552 self.scheme,
553 self.host,
554 self.port,
555 method,
556 url,
557 response.version_string,
558 response.status,
559 response.length_remaining,
560 )
562 return response
564 def close(self) -> None:
565 """
566 Close all pooled connections and disable the pool.
567 """
568 if self.pool is None:
569 return
570 # Disable access to the pool
571 old_pool, self.pool = self.pool, None
573 # Close all the HTTPConnections in the pool.
574 _close_pool_connections(old_pool)
576 def is_same_host(self, url: str) -> bool:
577 """
578 Check if the given ``url`` is a member of the same host as this
579 connection pool.
580 """
581 if url.startswith("/"):
582 return True
584 # TODO: Add optional support for socket.gethostbyname checking.
585 scheme, _, host, port, *_ = parse_url(url)
586 scheme = scheme or "http"
587 if host is not None:
588 host = _normalize_host(host, scheme=scheme)
590 # Use explicit default port for comparison when none is given
591 if self.port is not None and port is None:
592 port = port_by_scheme.get(scheme)
593 elif self.port is None and port == port_by_scheme.get(scheme):
594 port = None
596 return (scheme, host, port) == (self.scheme, self.host, self.port)
598 def urlopen( # type: ignore[override]
599 self,
600 method: str,
601 url: str,
602 body: _TYPE_BODY | None = None,
603 headers: typing.Mapping[str, str] | None = None,
604 retries: Retry | bool | int | None = None,
605 redirect: bool = True,
606 assert_same_host: bool = True,
607 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
608 pool_timeout: int | None = None,
609 release_conn: bool | None = None,
610 chunked: bool = False,
611 body_pos: _TYPE_BODY_POSITION | None = None,
612 preload_content: bool = True,
613 decode_content: bool = True,
614 **response_kw: typing.Any,
615 ) -> BaseHTTPResponse:
616 """
617 Get a connection from the pool and perform an HTTP request. This is the
618 lowest level call for making a request, so you'll need to specify all
619 the raw details.
621 .. note::
623 More commonly, it's appropriate to use a convenience method
624 such as :meth:`request`.
626 .. note::
628 `release_conn` will only behave as expected if
629 `preload_content=False` because we want to make
630 `preload_content=False` the default behaviour someday soon without
631 breaking backwards compatibility.
633 :param method:
634 HTTP request method (such as GET, POST, PUT, etc.)
636 :param url:
637 The URL to perform the request on.
639 :param body:
640 Data to send in the request body, either :class:`str`, :class:`bytes`,
641 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
643 :param headers:
644 Dictionary of custom headers to send, such as User-Agent,
645 If-None-Match, etc. If None, pool headers are used. If provided,
646 these headers completely replace any pool-specific headers.
648 :param retries:
649 Configure the number of retries to allow before raising a
650 :class:`~urllib3.exceptions.MaxRetryError` exception.
652 If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
653 :class:`~urllib3.util.retry.Retry` object for fine-grained control
654 over different types of retries.
655 Pass an integer number to retry connection errors that many times,
656 but no other types of errors. Pass zero to never retry.
658 If ``False``, then retries are disabled and any exception is raised
659 immediately. Also, instead of raising a MaxRetryError on redirects,
660 the redirect response will be returned.
662 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
664 :param redirect:
665 If True, automatically handle redirects (status codes 301, 302,
666 303, 307, 308). Each redirect counts as a retry. Disabling retries
667 will disable redirect, too.
669 :param assert_same_host:
670 If ``True``, will make sure that the host of the pool requests is
671 consistent else will raise HostChangedError. When ``False``, you can
672 use the pool on an HTTP proxy and request foreign hosts.
674 :param timeout:
675 If specified, overrides the default timeout for this one
676 request. It may be a float (in seconds) or an instance of
677 :class:`urllib3.util.Timeout`.
679 :param pool_timeout:
680 If set and the pool is set to block=True, then this method will
681 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
682 connection is available within the time period.
684 :param bool preload_content:
685 If True, the response's body will be preloaded into memory.
687 :param bool decode_content:
688 If True, will attempt to decode the body based on the
689 'content-encoding' header.
691 :param release_conn:
692 If False, then the urlopen call will not release the connection
693 back into the pool once a response is received (but will release if
694 you read the entire contents of the response such as when
695 `preload_content=True`). This is useful if you're not preloading
696 the response's content immediately. You will need to call
697 ``r.release_conn()`` on the response ``r`` to return the connection
698 back into the pool. If None, it takes the value of ``preload_content``
699 which defaults to ``True``.
701 :param bool chunked:
702 If True, urllib3 will send the body using chunked transfer
703 encoding. Otherwise, urllib3 will send the body using the standard
704 content-length form. Defaults to False.
706 :param int body_pos:
707 Position to seek to in file-like body in the event of a retry or
708 redirect. Typically this won't need to be set because urllib3 will
709 auto-populate the value when needed.
710 """
711 # Ensure that the URL we're connecting to is properly encoded
712 if url.startswith("/"):
713 # URLs starting with / are inherently schemeless.
714 url = to_str(_encode_target(url))
715 destination_scheme = None
716 else:
717 parsed_url = parse_url(url)
718 destination_scheme = parsed_url.scheme
719 url = to_str(parsed_url._replace(fragment=None).url)
721 if headers is None:
722 headers = self.headers
724 if not isinstance(retries, Retry):
725 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
727 if release_conn is None:
728 release_conn = preload_content
730 # Check host
731 if assert_same_host and not self.is_same_host(url):
732 raise HostChangedError(self, url, retries)
734 conn = None
736 # Track whether `conn` needs to be released before
737 # returning/raising/recursing. Update this variable if necessary, and
738 # leave `release_conn` constant throughout the function. That way, if
739 # the function recurses, the original value of `release_conn` will be
740 # passed down into the recursive call, and its value will be respected.
741 #
742 # See issue #651 [1] for details.
743 #
744 # [1] <https://github.com/urllib3/urllib3/issues/651>
745 release_this_conn = release_conn
747 http_tunnel_required = connection_requires_http_tunnel(
748 self.proxy, self.proxy_config, destination_scheme
749 )
751 # Merge the proxy headers. Only done when not using HTTP CONNECT. We
752 # have to copy the headers dict so we can safely change it without those
753 # changes being reflected in anyone else's copy.
754 if not http_tunnel_required:
755 headers = headers.copy() # type: ignore[attr-defined]
756 headers.update(self.proxy_headers) # type: ignore[union-attr]
758 # Must keep the exception bound to a separate variable or else Python 3
759 # complains about UnboundLocalError.
760 err = None
762 # Keep track of whether we cleanly exited the except block. This
763 # ensures we do proper cleanup in finally.
764 clean_exit = False
766 # Rewind body position, if needed. Record current position
767 # for future rewinds in the event of a redirect/retry.
768 body_pos = set_file_position(body, body_pos)
770 timeout_obj = self._get_timeout(timeout)
771 try:
772 # Request a connection from the queue.
773 conn = self._get_conn(timeout=pool_timeout)
774 conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
776 # Is this a closed/new connection that requires CONNECT tunnelling?
777 if self.proxy is not None and http_tunnel_required and conn.is_closed:
778 try:
779 self._prepare_proxy(conn)
780 except (BaseSSLError, OSError, SocketTimeout) as e:
781 self._raise_timeout(
782 err=e, url=self.proxy.url, timeout_value=conn.timeout
783 )
784 raise
786 # If we're going to release the connection in ``finally:``, then
787 # the response doesn't need to know about the connection. Otherwise
788 # it will also try to release it and we'll have a double-release
789 # mess.
790 response_conn = conn if not release_conn else None
792 # Make the request on the HTTPConnection object
793 response = self._make_request(
794 conn,
795 method,
796 url,
797 timeout=timeout_obj,
798 body=body,
799 headers=headers,
800 chunked=chunked,
801 retries=retries,
802 response_conn=response_conn,
803 preload_content=preload_content,
804 decode_content=decode_content,
805 **response_kw,
806 )
808 # Everything went great!
809 clean_exit = True
811 except EmptyPoolError:
812 # Didn't get a connection from the pool, no need to clean up
813 clean_exit = True
814 release_this_conn = False
815 raise
817 except (
818 TimeoutError,
819 HTTPException,
820 OSError,
821 ProtocolError,
822 BaseSSLError,
823 SSLError,
824 CertificateError,
825 ProxyError,
826 ) as e:
827 # Discard the connection for these exceptions. It will be
828 # replaced during the next _get_conn() call.
829 clean_exit = False
830 new_e: Exception = e
831 if isinstance(e, (BaseSSLError, CertificateError)):
832 new_e = SSLError(e)
833 if isinstance(
834 new_e,
835 (
836 OSError,
837 NewConnectionError,
838 TimeoutError,
839 SSLError,
840 HTTPException,
841 ),
842 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
843 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
844 elif isinstance(new_e, (OSError, HTTPException)):
845 new_e = ProtocolError("Connection aborted.", new_e)
847 retries = retries.increment(
848 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
849 )
850 retries.sleep()
852 # Keep track of the error for the retry warning.
853 err = e
855 finally:
856 if not clean_exit:
857 # We hit some kind of exception, handled or otherwise. We need
858 # to throw the connection away unless explicitly told not to.
859 # Close the connection, set the variable to None, and make sure
860 # we put the None back in the pool to avoid leaking it.
861 if conn:
862 conn.close()
863 conn = None
864 release_this_conn = True
866 if release_this_conn:
867 # Put the connection back to be reused. If the connection is
868 # expired then it will be None, which will get replaced with a
869 # fresh connection during _get_conn.
870 self._put_conn(conn)
872 if not conn:
873 # Try again
874 log.warning(
875 "Retrying (%r) after connection broken by '%r': %s",
876 retries,
877 err,
878 url,
879 # Provide an unique attribute with the host needed by pip to
880 # rewrite this warning. Ideally, we'd go with a better solution,
881 # but backwards compatibility and other constraints make those
882 # unfeasible, so this is the least bad option.
883 # See also: https://github.com/urllib3/urllib3/issues/2580.
884 extra={"__urllib3-retry-warning": {"host": self.host}},
885 )
886 return self.urlopen(
887 method,
888 url,
889 body,
890 headers,
891 retries,
892 redirect,
893 assert_same_host,
894 timeout=timeout,
895 pool_timeout=pool_timeout,
896 release_conn=release_conn,
897 chunked=chunked,
898 body_pos=body_pos,
899 preload_content=preload_content,
900 decode_content=decode_content,
901 **response_kw,
902 )
904 # Handle redirect?
905 redirect_location = redirect and response.get_redirect_location()
906 if redirect_location:
907 if response.status == 303:
908 # Change the method according to RFC 9110, Section 15.4.4.
909 method = "GET"
910 # And lose the body not to transfer anything sensitive.
911 body = None
912 # The body is gone, so the state that describes it has to go
913 # too: there is nothing left to frame with chunked transfer
914 # encoding, and nothing left to rewind.
915 chunked = False
916 body_pos = None
917 headers = HTTPHeaderDict(headers)._prepare_for_method_change()
919 # Strip headers marked as unsafe to forward to the redirected location.
920 # Check remove_headers_on_redirect to avoid a potential network call within
921 # self.is_same_host() which may use socket.gethostbyname() in the future.
922 if retries.remove_headers_on_redirect and not self.is_same_host(
923 redirect_location
924 ):
925 new_headers = headers.copy() # type: ignore[union-attr]
926 for header in headers:
927 if header.lower() in retries.remove_headers_on_redirect:
928 new_headers.pop(header, None)
929 headers = new_headers
931 try:
932 retries = retries.increment(method, url, response=response, _pool=self)
933 except MaxRetryError:
934 if retries.raise_on_redirect:
935 response.drain_conn()
936 raise
937 return response
939 response.drain_conn()
940 retries.sleep_for_retry(response)
941 log.debug("Redirecting %s -> %s", url, redirect_location)
942 return self.urlopen(
943 method,
944 redirect_location,
945 body,
946 headers,
947 retries=retries,
948 redirect=redirect,
949 assert_same_host=assert_same_host,
950 timeout=timeout,
951 pool_timeout=pool_timeout,
952 release_conn=release_conn,
953 chunked=chunked,
954 body_pos=body_pos,
955 preload_content=preload_content,
956 decode_content=decode_content,
957 **response_kw,
958 )
960 # Check if we should retry the HTTP response.
961 has_retry_after = bool(response.headers.get("Retry-After"))
962 if retries.is_retry(method, response.status, has_retry_after):
963 try:
964 retries = retries.increment(method, url, response=response, _pool=self)
965 except MaxRetryError:
966 if retries.raise_on_status:
967 response.drain_conn()
968 raise
969 return response
971 response.drain_conn()
972 retries.sleep(response)
973 log.debug("Retry: %s", url)
974 return self.urlopen(
975 method,
976 url,
977 body,
978 headers,
979 retries=retries,
980 redirect=redirect,
981 assert_same_host=assert_same_host,
982 timeout=timeout,
983 pool_timeout=pool_timeout,
984 release_conn=release_conn,
985 chunked=chunked,
986 body_pos=body_pos,
987 preload_content=preload_content,
988 decode_content=decode_content,
989 **response_kw,
990 )
992 return response
995class HTTPSConnectionPool(HTTPConnectionPool):
996 """
997 Same as :class:`.HTTPConnectionPool`, but HTTPS.
999 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
1000 ``assert_hostname`` and ``host`` in this order to verify connections.
1001 If ``assert_hostname`` is False, no verification is done.
1003 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
1004 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
1005 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
1006 the connection socket into an SSL socket.
1007 """
1009 scheme = "https"
1010 ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
1012 def __init__(
1013 self,
1014 host: str,
1015 port: int | None = None,
1016 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
1017 maxsize: int = 1,
1018 block: bool = False,
1019 headers: typing.Mapping[str, str] | None = None,
1020 retries: Retry | bool | int | None = None,
1021 _proxy: Url | None = None,
1022 _proxy_headers: typing.Mapping[str, str] | None = None,
1023 key_file: str | None = None,
1024 cert_file: str | None = None,
1025 cert_reqs: int | str | None = None,
1026 key_password: str | None = None,
1027 ca_certs: str | None = None,
1028 ssl_version: int | str | None = None,
1029 ssl_minimum_version: ssl.TLSVersion | None = None,
1030 ssl_maximum_version: ssl.TLSVersion | None = None,
1031 assert_hostname: str | typing.Literal[False] | None = None,
1032 assert_fingerprint: str | None = None,
1033 ca_cert_dir: str | None = None,
1034 **conn_kw: typing.Any,
1035 ) -> None:
1036 super().__init__(
1037 host,
1038 port,
1039 timeout,
1040 maxsize,
1041 block,
1042 headers,
1043 retries,
1044 _proxy,
1045 _proxy_headers,
1046 **conn_kw,
1047 )
1049 self.key_file = key_file
1050 self.cert_file = cert_file
1051 self.cert_reqs = cert_reqs
1052 self.key_password = key_password
1053 self.ca_certs = ca_certs
1054 self.ca_cert_dir = ca_cert_dir
1055 self.ssl_version = ssl_version
1056 self.ssl_minimum_version = ssl_minimum_version
1057 self.ssl_maximum_version = ssl_maximum_version
1058 self.assert_hostname = assert_hostname
1059 self.assert_fingerprint = assert_fingerprint
1061 def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
1062 """Establishes a tunnel connection through HTTP CONNECT."""
1063 if self.proxy and self.proxy.scheme == "https":
1064 tunnel_scheme = "https"
1065 else:
1066 tunnel_scheme = "http"
1068 conn.set_tunnel(
1069 scheme=tunnel_scheme,
1070 host=self._tunnel_host,
1071 port=self.port,
1072 headers=self.proxy_headers,
1073 )
1074 conn.connect()
1076 def _new_conn(self) -> BaseHTTPSConnection:
1077 """
1078 Return a fresh :class:`urllib3.connection.HTTPConnection`.
1079 """
1080 self.num_connections += 1
1081 log.debug(
1082 "Starting new HTTPS connection (%d): %s:%s",
1083 self.num_connections,
1084 self.host,
1085 self.port or "443",
1086 )
1088 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
1089 raise ImportError(
1090 "Can't connect to HTTPS URL because the SSL module is not available."
1091 )
1093 actual_host: str = self.host
1094 actual_port = self.port
1095 if self.proxy is not None and self.proxy.host is not None:
1096 actual_host = self.proxy.host
1097 actual_port = self.proxy.port
1099 return self.ConnectionCls(
1100 host=actual_host,
1101 port=actual_port,
1102 timeout=self.timeout.connect_timeout,
1103 cert_file=self.cert_file,
1104 key_file=self.key_file,
1105 key_password=self.key_password,
1106 cert_reqs=self.cert_reqs,
1107 ca_certs=self.ca_certs,
1108 ca_cert_dir=self.ca_cert_dir,
1109 assert_hostname=self.assert_hostname,
1110 assert_fingerprint=self.assert_fingerprint,
1111 ssl_version=self.ssl_version,
1112 ssl_minimum_version=self.ssl_minimum_version,
1113 ssl_maximum_version=self.ssl_maximum_version,
1114 **self.conn_kw,
1115 )
1117 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
1118 """
1119 Called right before a request is made, after the socket is created.
1120 """
1121 super()._validate_conn(conn)
1123 # Force connect early to allow us to validate the connection.
1124 if conn.is_closed:
1125 conn.connect()
1127 # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
1128 if not conn.is_verified and not conn.proxy_is_verified:
1129 warnings.warn(
1130 (
1131 f"Unverified HTTPS request is being made to host '{conn.host}'. "
1132 "Adding certificate verification is strongly advised. See: "
1133 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1134 "#tls-warnings"
1135 ),
1136 InsecureRequestWarning,
1137 )
1140def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
1141 """
1142 Given a url, return an :class:`.ConnectionPool` instance of its host.
1144 This is a shortcut for not having to parse out the scheme, host, and port
1145 of the url before creating an :class:`.ConnectionPool` instance.
1147 :param url:
1148 Absolute URL string that must include the scheme. Port is optional.
1150 :param \\**kw:
1151 Passes additional parameters to the constructor of the appropriate
1152 :class:`.ConnectionPool`. Useful for specifying things like
1153 timeout, maxsize, headers, etc.
1155 Example::
1157 >>> conn = connection_from_url('http://google.com/')
1158 >>> r = conn.request('GET', '/')
1159 """
1160 scheme, _, host, port, *_ = parse_url(url)
1161 scheme = scheme or "http"
1162 if port is None:
1163 port = port_by_scheme.get(scheme, 80)
1164 if scheme == "https":
1165 return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
1166 else:
1167 return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
1170@typing.overload
1171def _normalize_host(host: None, scheme: str | None) -> None: ...
1174@typing.overload
1175def _normalize_host(host: str, scheme: str | None) -> str: ...
1178def _normalize_host(host: str | None, scheme: str | None) -> str | None:
1179 """
1180 Normalize hosts for comparisons and use with sockets.
1181 """
1183 host = normalize_host(host, scheme)
1185 # httplib doesn't like it when we include brackets in IPv6 addresses
1186 # Specifically, if we include brackets but also pass the port then
1187 # httplib crazily doubles up the square brackets on the Host header.
1188 # Instead, we need to make sure we never pass ``None`` as the port.
1189 # However, for backward compatibility reasons we can't actually
1190 # *assert* that. See http://bugs.python.org/issue28539
1191 if host and host.startswith("[") and host.endswith("]"):
1192 host = host[1:-1]
1193 return host
1196def _url_from_pool(
1197 pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
1198) -> str:
1199 """Returns the URL from a given connection pool. This is mainly used for testing and logging."""
1200 return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
1203def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
1204 """Drains a queue of connections and closes each one."""
1205 try:
1206 while True:
1207 conn = pool.get(block=False)
1208 if conn:
1209 conn.close()
1210 except queue.Empty:
1211 pass # Done.