Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import errno
4import logging
5import queue
6import sys
7import typing
8import warnings
9import weakref
10from socket import timeout as SocketTimeout
11from types import TracebackType
13from ._base_connection import _TYPE_BODY
14from ._collections import HTTPHeaderDict
15from ._request_methods import RequestMethods
16from .connection import (
17 BaseSSLError,
18 BrokenPipeError,
19 DummyConnection,
20 HTTPConnection,
21 HTTPException,
22 HTTPSConnection,
23 ProxyConfig,
24 _wrap_proxy_error,
25)
26from .connection import port_by_scheme as port_by_scheme
27from .exceptions import (
28 ClosedPoolError,
29 EmptyPoolError,
30 FullPoolError,
31 HostChangedError,
32 InsecureRequestWarning,
33 LocationValueError,
34 MaxRetryError,
35 NewConnectionError,
36 ProtocolError,
37 ProxyError,
38 ReadTimeoutError,
39 SSLError,
40 TimeoutError,
41)
42from .response import BaseHTTPResponse
43from .util.connection import is_connection_dropped
44from .util.proxy import connection_requires_http_tunnel
45from .util.request import _TYPE_BODY_POSITION, set_file_position
46from .util.retry import Retry
47from .util.ssl_match_hostname import CertificateError
48from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
49from .util.url import Url, _encode_target
50from .util.url import _normalize_host as normalize_host
51from .util.url import parse_url
52from .util.util import to_str
54if typing.TYPE_CHECKING:
55 import ssl
57 from typing_extensions import Self
59 from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
61log = logging.getLogger(__name__)
63_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
64_DEFAULT_QUEUE_CLASS = queue.LifoQueue
67# Pool objects
68class ConnectionPool:
69 """
70 Base class for all connection pools, such as
71 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
73 .. note::
74 ConnectionPool.urlopen() does not normalize or percent-encode target URIs
75 which is useful if your target server doesn't support percent-encoded
76 target URIs.
77 """
79 scheme: str | None = None
80 QueueCls = _DEFAULT_QUEUE_CLASS
82 def __init__(self, host: str, port: int | None = None) -> None:
83 if not host:
84 raise LocationValueError("No host specified.")
86 self.host = _normalize_host(host, scheme=self.scheme)
87 self.port = port
89 # This property uses 'normalize_host()' (not '_normalize_host()')
90 # to avoid removing square braces around IPv6 addresses.
91 # This value is sent to `HTTPConnection.set_tunnel()` if called
92 # because square braces are required for HTTP CONNECT tunneling.
93 self._tunnel_host = normalize_host(host, scheme=self.scheme)
95 def __str__(self) -> str:
96 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
98 def __enter__(self) -> Self:
99 return self
101 def __exit__(
102 self,
103 exc_type: type[BaseException] | None,
104 exc_val: BaseException | None,
105 exc_tb: TracebackType | None,
106 ) -> typing.Literal[False]:
107 self.close()
108 # Return False to re-raise any potential exceptions
109 return False
111 def close(self) -> None:
112 """
113 Close all pooled connections and disable the pool.
114 """
116 def _new_pool_queue(self, maxsize: int) -> queue.LifoQueue[typing.Any]:
117 if self.QueueCls is _DEFAULT_QUEUE_CLASS:
118 return queue.LifoQueue(maxsize)
119 return self.QueueCls(maxsize)
122# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
123_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
126class HTTPConnectionPool(ConnectionPool, RequestMethods):
127 """
128 Thread-safe connection pool for one host.
130 :param host:
131 Host used for this HTTP Connection (e.g. "localhost"), passed into
132 :class:`http.client.HTTPConnection`.
134 :param port:
135 Port used for this HTTP Connection (None is equivalent to 80), passed
136 into :class:`http.client.HTTPConnection`.
138 :param timeout:
139 Socket timeout in seconds for each individual connection. This can
140 be a float or integer, which sets the timeout for the HTTP request,
141 or an instance of :class:`urllib3.util.Timeout` which gives you more
142 fine-grained control over request timeouts. After the constructor has
143 been parsed, this is always a `urllib3.util.Timeout` object.
145 :param maxsize:
146 Number of connections to save that can be reused. More than 1 is useful
147 in multithreaded situations. If ``block`` is set to False, more
148 connections will be created but they will not be saved once they've
149 been used.
151 :param block:
152 If set to True, no more than ``maxsize`` connections will be used at
153 a time. When no free connections are available, the call will block
154 until a connection has been released. This is a useful side effect for
155 particular multithreaded situations where one does not want to use more
156 than maxsize connections per host to prevent flooding.
158 :param headers:
159 Headers to include with all requests, unless other headers are given
160 explicitly.
162 :param retries:
163 Retry configuration to use by default with requests in this pool.
165 :param _proxy:
166 Parsed proxy URL, should not be used directly, instead, see
167 :class:`urllib3.ProxyManager`
169 :param _proxy_headers:
170 A dictionary with proxy headers, should not be used directly,
171 instead, see :class:`urllib3.ProxyManager`
173 :param \\**conn_kw:
174 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
175 :class:`urllib3.connection.HTTPSConnection` instances.
176 """
178 scheme = "http"
179 ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection
181 def __init__(
182 self,
183 host: str,
184 port: int | None = None,
185 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
186 maxsize: int = 1,
187 block: bool = False,
188 headers: typing.Mapping[str, str] | None = None,
189 retries: Retry | bool | int | None = None,
190 _proxy: Url | None = None,
191 _proxy_headers: typing.Mapping[str, str] | None = None,
192 _proxy_config: ProxyConfig | None = None,
193 **conn_kw: typing.Any,
194 ):
195 ConnectionPool.__init__(self, host, port)
196 RequestMethods.__init__(self, headers)
198 if not isinstance(timeout, Timeout):
199 timeout = Timeout.from_float(timeout)
201 if retries is None:
202 retries = Retry.DEFAULT
204 self.timeout = timeout
205 self.retries = retries
207 self.pool: queue.LifoQueue[typing.Any] | None = self._new_pool_queue(maxsize)
208 self.block = block
210 self.proxy = _proxy
211 self.proxy_headers = _proxy_headers or {}
212 self.proxy_config = _proxy_config
214 # Fill the queue up so that doing get() on it will block properly
215 for _ in range(maxsize):
216 self.pool.put(None)
218 # These are mostly for testing and debugging purposes.
219 self.num_connections = 0
220 self.num_requests = 0
221 self.conn_kw = conn_kw
223 if self.proxy:
224 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
225 # Defaulting `socket_options` to an empty list avoids it defaulting to
226 # ``HTTPConnection.default_socket_options``.
227 self.conn_kw.setdefault("socket_options", [])
229 self.conn_kw["proxy"] = self.proxy
230 self.conn_kw["proxy_config"] = self.proxy_config
232 # Do not pass 'self' as callback to 'finalize'.
233 # Then the 'finalize' would keep an endless living (leak) to self.
234 # By just passing a reference to the pool allows the garbage collector
235 # to free self if nobody else has a reference to it.
236 pool = self.pool
238 # Close all the HTTPConnections in the pool before the
239 # HTTPConnectionPool object is garbage collected.
240 weakref.finalize(self, _close_pool_connections, pool)
242 def _new_conn(self) -> BaseHTTPConnection:
243 """
244 Return a fresh :class:`HTTPConnection`.
245 """
246 self.num_connections += 1
247 log.debug(
248 "Starting new HTTP connection (%d): %s:%s",
249 self.num_connections,
250 self.host,
251 self.port or "80",
252 )
254 conn = self.ConnectionCls(
255 host=self.host,
256 port=self.port,
257 timeout=self.timeout.connect_timeout,
258 **self.conn_kw,
259 )
260 return conn
262 def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
263 """
264 Get a connection. Will return a pooled connection if one is available.
266 If no connections are available and :prop:`.block` is ``False``, then a
267 fresh connection is returned.
269 :param timeout:
270 Seconds to wait before giving up and raising
271 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
272 :prop:`.block` is ``True``.
273 """
274 conn = None
276 if self.pool is None:
277 raise ClosedPoolError(self, "Pool is closed.")
279 try:
280 conn = self.pool.get(block=self.block, timeout=timeout)
282 except AttributeError: # self.pool is None
283 raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
285 except queue.Empty:
286 if self.block:
287 raise EmptyPoolError(
288 self,
289 "Pool is empty and a new connection can't be opened due to blocking mode.",
290 ) from None
291 pass # Oh well, we'll create a new connection then
293 # If this is a persistent connection, check if it got disconnected
294 if conn and is_connection_dropped(conn):
295 log.debug("Resetting dropped connection: %s", self.host)
296 conn.close()
298 return conn or self._new_conn()
300 def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
301 """
302 Put a connection back into the pool.
304 :param conn:
305 Connection object for the current host and port as returned by
306 :meth:`._new_conn` or :meth:`._get_conn`.
308 If the pool is already full, the connection is closed and discarded
309 because we exceeded maxsize. If connections are discarded frequently,
310 then maxsize should be increased.
312 If the pool is closed, then the connection will be closed and discarded.
313 """
314 if self.pool is not None:
315 try:
316 self.pool.put(conn, block=False)
317 return # Everything is dandy, done.
318 except AttributeError:
319 # self.pool is None.
320 pass
321 except queue.Full:
322 # Connection never got put back into the pool, close it.
323 if conn:
324 conn.close()
326 if self.block:
327 # This should never happen if you got the conn from self._get_conn
328 raise FullPoolError(
329 self,
330 "Pool reached maximum size and no more connections are allowed.",
331 ) from None
333 log.warning(
334 "Connection pool is full, discarding connection: %s. Connection pool size: %s",
335 self.host,
336 self.pool.qsize(),
337 )
339 # Connection never got put back into the pool, close it.
340 if conn:
341 conn.close()
343 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
344 """
345 Called right before a request is made, after the socket is created.
346 """
348 def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
349 # Nothing to do for HTTP connections.
350 pass
352 def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
353 """Helper that always returns a :class:`urllib3.util.Timeout`"""
354 if timeout is _DEFAULT_TIMEOUT:
355 return self.timeout.clone()
357 if isinstance(timeout, Timeout):
358 return timeout.clone()
359 else:
360 # User passed us an int/float. This is for backwards compatibility,
361 # can be removed later
362 return Timeout.from_float(timeout)
364 def _raise_timeout(
365 self,
366 err: BaseSSLError | OSError | SocketTimeout,
367 url: str,
368 timeout_value: _TYPE_TIMEOUT | None,
369 ) -> None:
370 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
372 if isinstance(err, SocketTimeout):
373 raise ReadTimeoutError(
374 self, url, f"Read timed out. (read timeout={timeout_value})"
375 ) from err
377 # See the above comment about EAGAIN in Python 3.
378 if hasattr(err, "errno") and err.errno in _blocking_errnos:
379 raise ReadTimeoutError(
380 self, url, f"Read timed out. (read timeout={timeout_value})"
381 ) from err
383 def _make_request(
384 self,
385 conn: BaseHTTPConnection,
386 method: str,
387 url: str,
388 body: _TYPE_BODY | None = None,
389 headers: typing.Mapping[str, str] | None = None,
390 retries: Retry | None = None,
391 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
392 chunked: bool = False,
393 response_conn: BaseHTTPConnection | None = None,
394 preload_content: bool = True,
395 decode_content: bool = True,
396 enforce_content_length: bool = True,
397 ) -> BaseHTTPResponse:
398 """
399 Perform a request on a given urllib connection object taken from our
400 pool.
402 :param conn:
403 a connection from one of our connection pools
405 :param method:
406 HTTP request method (such as GET, POST, PUT, etc.)
408 :param url:
409 The URL to perform the request on.
411 :param body:
412 Data to send in the request body, either :class:`str`, :class:`bytes`,
413 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
415 :param headers:
416 Dictionary of custom headers to send, such as User-Agent,
417 If-None-Match, etc. If None, pool headers are used. If provided,
418 these headers completely replace any pool-specific headers.
420 :param retries:
421 Configure the number of retries to allow before raising a
422 :class:`~urllib3.exceptions.MaxRetryError` exception.
424 Pass ``None`` to retry until you receive a response. Pass a
425 :class:`~urllib3.util.retry.Retry` object for fine-grained control
426 over different types of retries.
427 Pass an integer number to retry connection errors that many times,
428 but no other types of errors. Pass zero to never retry.
430 If ``False``, then retries are disabled and any exception is raised
431 immediately. Also, instead of raising a MaxRetryError on redirects,
432 the redirect response will be returned.
434 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
436 :param timeout:
437 If specified, overrides the default timeout for this one
438 request. It may be a float (in seconds) or an instance of
439 :class:`urllib3.util.Timeout`.
441 :param chunked:
442 If True, urllib3 will send the body using chunked transfer
443 encoding. Otherwise, urllib3 will send the body using the standard
444 content-length form. Defaults to False.
446 :param response_conn:
447 Set this to ``None`` if you will handle releasing the connection or
448 set the connection to have the response release it.
450 :param preload_content:
451 If True, the response's body will be preloaded during construction.
453 :param decode_content:
454 If True, will attempt to decode the body based on the
455 'content-encoding' header.
457 :param enforce_content_length:
458 Enforce content length checking. Body returned by server must match
459 value of Content-Length header, if present. Otherwise, raise error.
460 """
461 self.num_requests += 1
463 timeout_obj = self._get_timeout(timeout)
464 timeout_obj.start_connect()
465 conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
467 try:
468 # Trigger any extra validation we need to do.
469 try:
470 self._validate_conn(conn)
471 except (SocketTimeout, BaseSSLError) as e:
472 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
473 raise
475 # _validate_conn() starts the connection to an HTTPS proxy
476 # so we need to wrap errors with 'ProxyError' here too.
477 except (
478 OSError,
479 NewConnectionError,
480 TimeoutError,
481 BaseSSLError,
482 CertificateError,
483 SSLError,
484 ) as e:
485 new_e: Exception = e
486 if isinstance(e, (BaseSSLError, CertificateError)):
487 new_e = SSLError(e)
488 # If the connection didn't successfully connect to it's proxy
489 # then there
490 if isinstance(
491 new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
492 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
493 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
494 raise new_e
496 # conn.request() calls http.client.*.request, not the method in
497 # urllib3.request. It also calls makefile (recv) on the socket.
498 try:
499 conn.request(
500 method,
501 url,
502 body=body,
503 headers=headers,
504 chunked=chunked,
505 preload_content=preload_content,
506 decode_content=decode_content,
507 enforce_content_length=enforce_content_length,
508 )
510 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
511 # legitimately able to close the connection after sending a valid response.
512 # With this behaviour, the received response is still readable.
513 except BrokenPipeError:
514 pass
515 except OSError as e:
516 # MacOS/Linux
517 # EPROTOTYPE and ECONNRESET are needed on macOS
518 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
519 # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
520 if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
521 raise
523 # Reset the timeout for the recv() on the socket
524 read_timeout = timeout_obj.read_timeout
526 if not conn.is_closed:
527 # In Python 3 socket.py will catch EAGAIN and return None when you
528 # try and read into the file pointer created by http.client, which
529 # instead raises a BadStatusLine exception. Instead of catching
530 # the exception and assuming all BadStatusLine exceptions are read
531 # timeouts, check for a zero timeout before making the request.
532 if read_timeout == 0:
533 raise ReadTimeoutError(
534 self, url, f"Read timed out. (read timeout={read_timeout})"
535 )
536 conn.timeout = read_timeout
538 # Receive the response from the server
539 try:
540 response = conn.getresponse()
541 except (BaseSSLError, OSError) as e:
542 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
543 raise
545 # Set properties that are used by the pooling layer.
546 response.retries = retries
547 response._connection = response_conn # type: ignore[attr-defined]
548 response._pool = self # type: ignore[attr-defined]
550 log.debug(
551 '%s://%s:%s "%s %s %s" %s %s',
552 self.scheme,
553 self.host,
554 self.port,
555 method,
556 url,
557 response.version_string,
558 response.status,
559 response.length_remaining,
560 )
562 return response
564 def close(self) -> None:
565 """
566 Close all pooled connections and disable the pool.
567 """
568 if self.pool is None:
569 return
570 # Disable access to the pool
571 old_pool, self.pool = self.pool, None
573 # Close all the HTTPConnections in the pool.
574 _close_pool_connections(old_pool)
576 def is_same_host(self, url: str) -> bool:
577 """
578 Check if the given ``url`` is a member of the same host as this
579 connection pool.
580 """
581 return _is_same_host(url, self.scheme, self.host, self.port)
583 def urlopen( # type: ignore[override]
584 self,
585 method: str,
586 url: str,
587 body: _TYPE_BODY | None = None,
588 headers: typing.Mapping[str, str] | None = None,
589 retries: Retry | bool | int | None = None,
590 redirect: bool = True,
591 assert_same_host: bool = True,
592 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
593 pool_timeout: int | None = None,
594 release_conn: bool | None = None,
595 chunked: bool = False,
596 body_pos: _TYPE_BODY_POSITION | None = None,
597 preload_content: bool = True,
598 decode_content: bool = True,
599 **response_kw: typing.Any,
600 ) -> BaseHTTPResponse:
601 """
602 Get a connection from the pool and perform an HTTP request. This is the
603 lowest level call for making a request, so you'll need to specify all
604 the raw details.
606 .. note::
608 More commonly, it's appropriate to use a convenience method
609 such as :meth:`request`.
611 .. note::
613 `release_conn` will only behave as expected if
614 `preload_content=False` because we want to make
615 `preload_content=False` the default behaviour someday soon without
616 breaking backwards compatibility.
618 :param method:
619 HTTP request method (such as GET, POST, PUT, etc.)
621 :param url:
622 The URL to perform the request on.
624 :param body:
625 Data to send in the request body, either :class:`str`, :class:`bytes`,
626 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
628 :param headers:
629 Dictionary of custom headers to send, such as User-Agent,
630 If-None-Match, etc. If None, pool headers are used. If provided,
631 these headers completely replace any pool-specific headers.
633 :param retries:
634 Configure the number of retries to allow before raising a
635 :class:`~urllib3.exceptions.MaxRetryError` exception.
637 If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
638 :class:`~urllib3.util.retry.Retry` object for fine-grained control
639 over different types of retries.
640 Pass an integer number to retry connection errors that many times,
641 but no other types of errors. Pass zero to never retry.
643 If ``False``, then retries are disabled and any exception is raised
644 immediately. Also, instead of raising a MaxRetryError on redirects,
645 the redirect response will be returned.
647 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
649 :param redirect:
650 If True, automatically handle redirects (status codes 301, 302,
651 303, 307, 308). Each redirect counts as a retry. Disabling retries
652 will disable redirect, too.
654 :param assert_same_host:
655 If ``True``, will make sure that the host of the pool requests is
656 consistent else will raise HostChangedError. When ``False``, you can
657 use the pool on an HTTP proxy and request foreign hosts.
659 :param timeout:
660 If specified, overrides the default timeout for this one
661 request. It may be a float (in seconds) or an instance of
662 :class:`urllib3.util.Timeout`.
664 :param pool_timeout:
665 If set and the pool is set to block=True, then this method will
666 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
667 connection is available within the time period.
669 :param bool preload_content:
670 If True, the response's body will be preloaded into memory.
672 :param bool decode_content:
673 If True, will attempt to decode the body based on the
674 'content-encoding' header.
676 :param release_conn:
677 If False, then the urlopen call will not release the connection
678 back into the pool once a response is received (but will release if
679 you read the entire contents of the response such as when
680 `preload_content=True`). This is useful if you're not preloading
681 the response's content immediately. You will need to call
682 ``r.release_conn()`` on the response ``r`` to return the connection
683 back into the pool. If None, it takes the value of ``preload_content``
684 which defaults to ``True``.
686 :param bool chunked:
687 If True, urllib3 will send the body using chunked transfer
688 encoding. Otherwise, urllib3 will send the body using the standard
689 content-length form. Defaults to False.
691 :param int body_pos:
692 Position to seek to in file-like body in the event of a retry or
693 redirect. Typically this won't need to be set because urllib3 will
694 auto-populate the value when needed.
695 """
696 # Preserve the input for retries so that IPv6 zone identifiers aren't
697 # decoded again when urlopen() normalizes the URL on the next attempt.
698 original_url = url
700 # Ensure that the URL we're connecting to is properly encoded
701 if url.startswith("/"):
702 # URLs starting with / are inherently schemeless.
703 url = to_str(_encode_target(url))
704 destination_scheme = None
705 else:
706 parsed_url = parse_url(url)
707 destination_scheme = parsed_url.scheme
708 url = to_str(parsed_url._replace(fragment=None).url)
710 if headers is None:
711 headers = self.headers
713 if not isinstance(retries, Retry):
714 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
716 if release_conn is None:
717 release_conn = preload_content
719 # Check host
720 if assert_same_host and not self.is_same_host(url):
721 raise HostChangedError(self, url, retries)
723 conn = None
725 # Track whether `conn` needs to be released before
726 # returning/raising/recursing. Update this variable if necessary, and
727 # leave `release_conn` constant throughout the function. That way, if
728 # the function recurses, the original value of `release_conn` will be
729 # passed down into the recursive call, and its value will be respected.
730 #
731 # See issue #651 [1] for details.
732 #
733 # [1] <https://github.com/urllib3/urllib3/issues/651>
734 release_this_conn = release_conn
736 http_tunnel_required = connection_requires_http_tunnel(
737 self.proxy, self.proxy_config, destination_scheme
738 )
740 # Merge the proxy headers. Only done when not using HTTP CONNECT. We
741 # have to copy the headers dict so we can safely change it without those
742 # changes being reflected in anyone else's copy.
743 if not http_tunnel_required:
744 headers = headers.copy() # type: ignore[attr-defined]
745 headers.update(self.proxy_headers) # type: ignore[union-attr]
747 # Must keep the exception bound to a separate variable or else Python 3
748 # complains about UnboundLocalError.
749 err = None
751 # Keep track of whether we cleanly exited the except block. This
752 # ensures we do proper cleanup in finally.
753 clean_exit = False
755 # Rewind body position, if needed. Record current position
756 # for future rewinds in the event of a redirect/retry.
757 body_pos = set_file_position(body, body_pos)
759 timeout_obj = self._get_timeout(timeout)
760 try:
761 # Request a connection from the queue.
762 conn = self._get_conn(timeout=pool_timeout)
763 conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
765 # Is this a closed/new connection that requires CONNECT tunnelling?
766 if self.proxy is not None and http_tunnel_required and conn.is_closed:
767 try:
768 self._prepare_proxy(conn)
769 except (BaseSSLError, OSError, SocketTimeout) as e:
770 self._raise_timeout(
771 err=e, url=self.proxy.url, timeout_value=conn.timeout
772 )
773 raise
775 # If we're going to release the connection in ``finally:``, then
776 # the response doesn't need to know about the connection. Otherwise
777 # it will also try to release it and we'll have a double-release
778 # mess.
779 response_conn = conn if not release_conn else None
781 # Make the request on the HTTPConnection object
782 response = self._make_request(
783 conn,
784 method,
785 url,
786 timeout=timeout_obj,
787 body=body,
788 headers=headers,
789 chunked=chunked,
790 retries=retries,
791 response_conn=response_conn,
792 preload_content=preload_content,
793 decode_content=decode_content,
794 **response_kw,
795 )
797 # Everything went great!
798 clean_exit = True
800 except EmptyPoolError:
801 # Didn't get a connection from the pool, no need to clean up
802 clean_exit = True
803 release_this_conn = False
804 raise
806 except (
807 TimeoutError,
808 HTTPException,
809 OSError,
810 ProtocolError,
811 BaseSSLError,
812 SSLError,
813 CertificateError,
814 ProxyError,
815 ) as e:
816 # Discard the connection for these exceptions. It will be
817 # replaced during the next _get_conn() call.
818 clean_exit = False
819 new_e: Exception = e
820 if isinstance(e, (BaseSSLError, CertificateError)):
821 new_e = SSLError(e)
822 if isinstance(
823 new_e,
824 (
825 OSError,
826 NewConnectionError,
827 TimeoutError,
828 SSLError,
829 HTTPException,
830 ),
831 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
832 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
833 elif isinstance(new_e, (OSError, HTTPException)):
834 new_e = ProtocolError("Connection aborted.", new_e)
836 retries = retries.increment(
837 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
838 )
839 retries.sleep()
841 # Keep track of the error for the retry warning.
842 err = e
844 finally:
845 if not clean_exit:
846 # We hit some kind of exception, handled or otherwise. We need
847 # to throw the connection away unless explicitly told not to.
848 # Close the connection, set the variable to None, and make sure
849 # we put the None back in the pool to avoid leaking it.
850 if conn:
851 conn.close()
852 conn = None
853 release_this_conn = True
855 if release_this_conn:
856 # Put the connection back to be reused. If the connection is
857 # expired then it will be None, which will get replaced with a
858 # fresh connection during _get_conn.
859 self._put_conn(conn)
861 if not conn:
862 # Try again
863 log.warning(
864 "Retrying (%r) after connection broken by '%r': %s",
865 retries,
866 err,
867 url,
868 # Provide an unique attribute with the host needed by pip to
869 # rewrite this warning. Ideally, we'd go with a better solution,
870 # but backwards compatibility and other constraints make those
871 # unfeasible, so this is the least bad option.
872 # See also: https://github.com/urllib3/urllib3/issues/2580.
873 extra={"__urllib3-retry-warning": {"host": self.host}},
874 )
875 return self.urlopen(
876 method,
877 original_url,
878 body,
879 headers,
880 retries,
881 redirect,
882 assert_same_host,
883 timeout=timeout,
884 pool_timeout=pool_timeout,
885 release_conn=release_conn,
886 chunked=chunked,
887 body_pos=body_pos,
888 preload_content=preload_content,
889 decode_content=decode_content,
890 **response_kw,
891 )
893 # Handle redirect?
894 redirect_location = redirect and response.get_redirect_location()
895 if redirect_location:
896 if response.status == 303:
897 # Change the method according to RFC 9110, Section 15.4.4.
898 method = "GET"
899 # And lose the body not to transfer anything sensitive.
900 body = None
901 # The body is gone, so the state that describes it has to go
902 # too: there is nothing left to frame with chunked transfer
903 # encoding, and nothing left to rewind.
904 chunked = False
905 body_pos = None
906 headers = HTTPHeaderDict(headers)._prepare_for_method_change()
908 # Strip headers marked as unsafe to forward to the redirected location.
909 # Check remove_headers_on_redirect to avoid a potential network call within
910 # self.is_same_host() which may use socket.gethostbyname() in the future.
911 if retries.remove_headers_on_redirect and not self.is_same_host(
912 redirect_location
913 ):
914 new_headers = headers.copy() # type: ignore[union-attr]
915 for header in headers:
916 if header.lower() in retries.remove_headers_on_redirect:
917 new_headers.pop(header, None)
918 headers = new_headers
920 try:
921 retries = retries.increment(method, url, response=response, _pool=self)
922 except MaxRetryError:
923 if retries.raise_on_redirect:
924 response.drain_conn()
925 raise
926 return response
928 response.drain_conn()
929 retries.sleep_for_retry(response)
930 log.debug("Redirecting %s -> %s", url, redirect_location)
931 return self.urlopen(
932 method,
933 redirect_location,
934 body,
935 headers,
936 retries=retries,
937 redirect=redirect,
938 assert_same_host=assert_same_host,
939 timeout=timeout,
940 pool_timeout=pool_timeout,
941 release_conn=release_conn,
942 chunked=chunked,
943 body_pos=body_pos,
944 preload_content=preload_content,
945 decode_content=decode_content,
946 **response_kw,
947 )
949 # Check if we should retry the HTTP response.
950 has_retry_after = bool(response.headers.get("Retry-After"))
951 if retries.is_retry(method, response.status, has_retry_after):
952 try:
953 retries = retries.increment(method, url, response=response, _pool=self)
954 except MaxRetryError:
955 if retries.raise_on_status:
956 response.drain_conn()
957 raise
958 return response
960 response.drain_conn()
961 retries.sleep(response)
962 log.debug("Retry: %s", url)
963 return self.urlopen(
964 method,
965 original_url,
966 body,
967 headers,
968 retries=retries,
969 redirect=redirect,
970 assert_same_host=assert_same_host,
971 timeout=timeout,
972 pool_timeout=pool_timeout,
973 release_conn=release_conn,
974 chunked=chunked,
975 body_pos=body_pos,
976 preload_content=preload_content,
977 decode_content=decode_content,
978 **response_kw,
979 )
981 return response
984class HTTPSConnectionPool(HTTPConnectionPool):
985 """
986 Same as :class:`.HTTPConnectionPool`, but HTTPS.
988 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
989 ``assert_hostname`` and ``host`` in this order to verify connections.
990 If ``assert_hostname`` is False, no verification is done.
992 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
993 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
994 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
995 the connection socket into an SSL socket.
996 """
998 scheme = "https"
999 ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
1001 def __init__(
1002 self,
1003 host: str,
1004 port: int | None = None,
1005 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
1006 maxsize: int = 1,
1007 block: bool = False,
1008 headers: typing.Mapping[str, str] | None = None,
1009 retries: Retry | bool | int | None = None,
1010 _proxy: Url | None = None,
1011 _proxy_headers: typing.Mapping[str, str] | None = None,
1012 key_file: str | None = None,
1013 cert_file: str | None = None,
1014 cert_reqs: int | str | None = None,
1015 key_password: str | None = None,
1016 ca_certs: str | None = None,
1017 ssl_version: int | str | None = None,
1018 ssl_minimum_version: ssl.TLSVersion | None = None,
1019 ssl_maximum_version: ssl.TLSVersion | None = None,
1020 assert_hostname: str | typing.Literal[False] | None = None,
1021 assert_fingerprint: str | None = None,
1022 ca_cert_dir: str | None = None,
1023 **conn_kw: typing.Any,
1024 ) -> None:
1025 super().__init__(
1026 host,
1027 port,
1028 timeout,
1029 maxsize,
1030 block,
1031 headers,
1032 retries,
1033 _proxy,
1034 _proxy_headers,
1035 **conn_kw,
1036 )
1038 self.key_file = key_file
1039 self.cert_file = cert_file
1040 self.cert_reqs = cert_reqs
1041 self.key_password = key_password
1042 self.ca_certs = ca_certs
1043 self.ca_cert_dir = ca_cert_dir
1044 self.ssl_version = ssl_version
1045 self.ssl_minimum_version = ssl_minimum_version
1046 self.ssl_maximum_version = ssl_maximum_version
1047 self.assert_hostname = assert_hostname
1048 self.assert_fingerprint = assert_fingerprint
1050 def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
1051 """Establishes a tunnel connection through HTTP CONNECT."""
1052 if self.proxy and self.proxy.scheme == "https":
1053 tunnel_scheme = "https"
1054 else:
1055 tunnel_scheme = "http"
1057 conn.set_tunnel(
1058 scheme=tunnel_scheme,
1059 host=self._tunnel_host,
1060 port=self.port,
1061 headers=self.proxy_headers,
1062 )
1063 conn.connect()
1065 def _new_conn(self) -> BaseHTTPSConnection:
1066 """
1067 Return a fresh :class:`urllib3.connection.HTTPConnection`.
1068 """
1069 self.num_connections += 1
1070 log.debug(
1071 "Starting new HTTPS connection (%d): %s:%s",
1072 self.num_connections,
1073 self.host,
1074 self.port or "443",
1075 )
1077 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
1078 raise ImportError(
1079 "Can't connect to HTTPS URL because the SSL module is not available."
1080 )
1082 actual_host: str = self.host
1083 actual_port = self.port
1084 if self.proxy is not None and self.proxy.host is not None:
1085 actual_host = self.proxy.host
1086 actual_port = self.proxy.port
1088 return self.ConnectionCls(
1089 host=actual_host,
1090 port=actual_port,
1091 timeout=self.timeout.connect_timeout,
1092 cert_file=self.cert_file,
1093 key_file=self.key_file,
1094 key_password=self.key_password,
1095 cert_reqs=self.cert_reqs,
1096 ca_certs=self.ca_certs,
1097 ca_cert_dir=self.ca_cert_dir,
1098 assert_hostname=self.assert_hostname,
1099 assert_fingerprint=self.assert_fingerprint,
1100 ssl_version=self.ssl_version,
1101 ssl_minimum_version=self.ssl_minimum_version,
1102 ssl_maximum_version=self.ssl_maximum_version,
1103 **self.conn_kw,
1104 )
1106 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
1107 """
1108 Called right before a request is made, after the socket is created.
1109 """
1110 super()._validate_conn(conn)
1112 # Force connect early to allow us to validate the connection.
1113 if conn.is_closed:
1114 conn.connect()
1116 # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
1117 if not conn.is_verified and not conn.proxy_is_verified:
1118 warnings.warn(
1119 (
1120 f"Unverified HTTPS request is being made to host '{conn.host}'. "
1121 "Adding certificate verification is strongly advised. See: "
1122 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1123 "#tls-warnings"
1124 ),
1125 InsecureRequestWarning,
1126 )
1129def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
1130 """
1131 Given a url, return an :class:`.ConnectionPool` instance of its host.
1133 This is a shortcut for not having to parse out the scheme, host, and port
1134 of the url before creating an :class:`.ConnectionPool` instance.
1136 :param url:
1137 Absolute URL string that must include the scheme. Port is optional.
1139 :param \\**kw:
1140 Passes additional parameters to the constructor of the appropriate
1141 :class:`.ConnectionPool`. Useful for specifying things like
1142 timeout, maxsize, headers, etc.
1144 Example::
1146 >>> conn = connection_from_url('http://google.com/')
1147 >>> r = conn.request('GET', '/')
1148 """
1149 scheme, _, host, port, *_ = parse_url(url)
1150 scheme = scheme or "http"
1151 if port is None:
1152 port = port_by_scheme.get(scheme, 80)
1153 if scheme == "https":
1154 return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
1155 else:
1156 return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
1159@typing.overload
1160def _normalize_host(host: None, scheme: str | None) -> None: ...
1163@typing.overload
1164def _normalize_host(host: str, scheme: str | None) -> str: ...
1167def _normalize_host(host: str | None, scheme: str | None) -> str | None:
1168 """
1169 Normalize hosts for comparisons and use with sockets.
1170 """
1172 host = normalize_host(host, scheme)
1174 # httplib doesn't like it when we include brackets in IPv6 addresses
1175 # Specifically, if we include brackets but also pass the port then
1176 # httplib crazily doubles up the square brackets on the Host header.
1177 # Instead, we need to make sure we never pass ``None`` as the port.
1178 # However, for backward compatibility reasons we can't actually
1179 # *assert* that. See http://bugs.python.org/issue28539
1180 if host and host.startswith("[") and host.endswith("]"):
1181 host = host[1:-1]
1182 return host
1185def _is_same_host(
1186 url: str, scheme: str | None, host: str | None, port: int | None
1187) -> bool:
1188 """
1189 Check if the given ``url`` is a member of the same host as the given
1190 ``scheme``, ``host`` and ``port``. ``host`` is expected to be
1191 normalized with :func:`_normalize_host` already.
1192 """
1193 if url.startswith("/"):
1194 return True
1196 # TODO: Add optional support for socket.gethostbyname checking.
1197 url_scheme, _, url_host, url_port, *_ = parse_url(url)
1198 url_scheme = url_scheme or "http"
1199 if url_host is not None:
1200 url_host = _normalize_host(url_host, scheme=url_scheme)
1202 # Use explicit default port for comparison when none is given
1203 if port is not None and url_port is None:
1204 url_port = port_by_scheme.get(url_scheme)
1205 elif port is None and url_port == port_by_scheme.get(url_scheme):
1206 url_port = None
1208 return (url_scheme, url_host, url_port) == (scheme, host, port)
1211def _url_from_pool(
1212 pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
1213) -> str:
1214 """Returns the URL from a given connection pool. This is mainly used for testing and logging."""
1215 return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
1218def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
1219 """Drains a queue of connections and closes each one."""
1220 try:
1221 while True:
1222 conn = pool.get(block=False)
1223 if conn:
1224 conn.close()
1225 except queue.Empty:
1226 pass # Done.