Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/poolmanager.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import functools
4import logging
5import typing
6import warnings
7from types import TracebackType
8from urllib.parse import urljoin
10from ._collections import HTTPHeaderDict, RecentlyUsedContainer
11from ._request_methods import RequestMethods
12from .connection import ProxyConfig
13from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
14from .exceptions import (
15 LocationValueError,
16 MaxRetryError,
17 ProxySchemeUnknown,
18 URLSchemeUnknown,
19)
20from .response import BaseHTTPResponse
21from .util.connection import _TYPE_SOCKET_OPTIONS
22from .util.proxy import connection_requires_http_tunnel
23from .util.retry import Retry
24from .util.timeout import Timeout
25from .util.url import Url, parse_url
27if typing.TYPE_CHECKING:
28 import ssl
30 from typing_extensions import Self
32__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
35log = logging.getLogger(__name__)
37SSL_KEYWORDS = (
38 "key_file",
39 "cert_file",
40 "cert_reqs",
41 "ca_certs",
42 "ca_cert_data",
43 "ssl_version",
44 "ssl_minimum_version",
45 "ssl_maximum_version",
46 "ca_cert_dir",
47 "ssl_context",
48 "key_password",
49 "server_hostname",
50 "assert_hostname",
51 "assert_fingerprint",
52)
53# Default value for `blocksize` - a new parameter introduced to
54# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
55_DEFAULT_BLOCKSIZE = 16384
58class PoolKey(typing.NamedTuple):
59 """
60 All known keyword arguments that could be provided to the pool manager, its
61 pools, or the underlying connections.
63 All custom key schemes should include the fields in this key at a minimum.
64 """
66 key_scheme: str
67 key_host: str
68 key_port: int | None
69 key_timeout: Timeout | float | int | None
70 key_retries: Retry | bool | int | None
71 key_block: bool | None
72 key_source_address: tuple[str, int] | None
73 key_key_file: str | None
74 key_key_password: str | None
75 key_cert_file: str | None
76 key_cert_reqs: str | None
77 key_ca_certs: str | None
78 key_ca_cert_data: str | bytes | None
79 key_ssl_version: int | str | None
80 key_ssl_minimum_version: ssl.TLSVersion | None
81 key_ssl_maximum_version: ssl.TLSVersion | None
82 key_ca_cert_dir: str | None
83 key_ssl_context: ssl.SSLContext | None
84 key_maxsize: int | None
85 key_headers: frozenset[tuple[str, str]] | None
86 key__proxy: Url | None
87 key__proxy_headers: frozenset[tuple[str, str]] | None
88 key__proxy_config: ProxyConfig | None
89 key_socket_options: _TYPE_SOCKET_OPTIONS | None
90 key__socks_options: frozenset[tuple[str, str]] | None
91 key_assert_hostname: bool | str | None
92 key_assert_fingerprint: str | None
93 key_server_hostname: str | None
94 key_blocksize: int | None
97def _default_key_normalizer(
98 key_class: type[PoolKey], request_context: dict[str, typing.Any]
99) -> PoolKey:
100 """
101 Create a pool key out of a request context dictionary.
103 According to RFC 3986, both the scheme and host are case-insensitive.
104 Therefore, this function normalizes both before constructing the pool
105 key for an HTTPS request. If you wish to change this behaviour, provide
106 alternate callables to ``key_fn_by_scheme``.
108 :param key_class:
109 The class to use when constructing the key. This should be a namedtuple
110 with the ``scheme`` and ``host`` keys at a minimum.
111 :type key_class: namedtuple
112 :param request_context:
113 A dictionary-like object that contain the context for a request.
114 :type request_context: dict
116 :return: A namedtuple that can be used as a connection pool key.
117 :rtype: PoolKey
118 """
119 # Since we mutate the dictionary, make a copy first
120 context = request_context.copy()
121 context["scheme"] = context["scheme"].lower()
122 # Only the address portion of a scoped IPv6 host is case-insensitive.
123 address, separator, zone = context["host"].partition("%")
124 if separator and ":" in address:
125 context["host"] = f"{address.lower()}{separator}{zone}"
126 else:
127 context["host"] = context["host"].lower()
129 # These are both dictionaries and need to be transformed into frozensets
130 for key in ("headers", "_proxy_headers", "_socks_options"):
131 if key in context and context[key] is not None:
132 context[key] = frozenset(context[key].items())
134 # The socket_options key may be a list and needs to be transformed into a
135 # tuple.
136 socket_opts = context.get("socket_options")
137 if socket_opts is not None:
138 context["socket_options"] = tuple(socket_opts)
140 # Map the kwargs to the names in the namedtuple - this is necessary since
141 # namedtuples can't have fields starting with '_'.
142 for key in list(context.keys()):
143 context["key_" + key] = context.pop(key)
145 # Default to ``None`` for keys missing from the context
146 for field in key_class._fields:
147 if field not in context:
148 context[field] = None
150 # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
151 if context.get("key_blocksize") is None:
152 context["key_blocksize"] = _DEFAULT_BLOCKSIZE
154 return key_class(**context)
157#: A dictionary that maps a scheme to a callable that creates a pool key.
158#: This can be used to alter the way pool keys are constructed, if desired.
159#: Each PoolManager makes a copy of this dictionary so they can be configured
160#: globally here, or individually on the instance.
161key_fn_by_scheme = {
162 "http": functools.partial(_default_key_normalizer, PoolKey),
163 "https": functools.partial(_default_key_normalizer, PoolKey),
164}
166pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
169class PoolManager(RequestMethods):
170 """
171 Allows for arbitrary requests while transparently keeping track of
172 necessary connection pools for you.
174 :param num_pools:
175 Number of connection pools to cache before discarding the least
176 recently used pool.
178 :param headers:
179 Headers to include with all requests, unless other headers are given
180 explicitly.
182 :param \\**connection_pool_kw:
183 Additional parameters are used to create fresh
184 :class:`urllib3.connectionpool.ConnectionPool` instances.
186 Example:
188 .. code-block:: python
190 import urllib3
192 http = urllib3.PoolManager(num_pools=2)
194 resp1 = http.request("GET", "https://google.com/")
195 resp2 = http.request("GET", "https://google.com/mail")
196 resp3 = http.request("GET", "https://yahoo.com/")
198 print(len(http.pools))
199 # 2
201 """
203 proxy: Url | None = None
204 proxy_config: ProxyConfig | None = None
206 def __init__(
207 self,
208 num_pools: int = 10,
209 headers: typing.Mapping[str, str] | None = None,
210 **connection_pool_kw: typing.Any,
211 ) -> None:
212 super().__init__(headers)
213 # PoolManager handles redirects itself in PoolManager.urlopen().
214 # It always passes redirect=False to the underlying connection pool to
215 # suppress per-pool redirect handling. If the user supplied a non-Retry
216 # value (int/bool/etc) for retries and we let the pool normalize it
217 # while redirect=False, the resulting Retry object would have redirect
218 # handling disabled, which can interfere with PoolManager's own
219 # redirect logic. Normalize here so redirects remain governed solely by
220 # PoolManager logic.
221 if "retries" in connection_pool_kw:
222 retries = connection_pool_kw["retries"]
223 if not isinstance(retries, Retry):
224 retries = Retry.from_int(retries)
225 connection_pool_kw = connection_pool_kw.copy()
226 connection_pool_kw["retries"] = retries
227 self.connection_pool_kw = connection_pool_kw
229 self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
230 self.pools = RecentlyUsedContainer(num_pools)
232 # Locally set the pool classes and keys so other PoolManagers can
233 # override them.
234 self.pool_classes_by_scheme = pool_classes_by_scheme
235 self.key_fn_by_scheme = key_fn_by_scheme.copy()
237 def __enter__(self) -> Self:
238 return self
240 def __exit__(
241 self,
242 exc_type: type[BaseException] | None,
243 exc_val: BaseException | None,
244 exc_tb: TracebackType | None,
245 ) -> typing.Literal[False]:
246 self.clear()
247 # Return False to re-raise any potential exceptions
248 return False
250 def _new_pool(
251 self,
252 scheme: str,
253 host: str,
254 port: int,
255 request_context: dict[str, typing.Any] | None = None,
256 ) -> HTTPConnectionPool:
257 """
258 Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
259 any additional pool keyword arguments.
261 If ``request_context`` is provided, it is provided as keyword arguments
262 to the pool class used. This method is used to actually create the
263 connection pools handed out by :meth:`connection_from_url` and
264 companion methods. It is intended to be overridden for customization.
265 """
266 pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
267 if request_context is None:
268 request_context = self.connection_pool_kw.copy()
270 # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
271 # set to 'None' in the request_context.
272 if request_context.get("blocksize") is None:
273 request_context["blocksize"] = _DEFAULT_BLOCKSIZE
275 # Although the context has everything necessary to create the pool,
276 # this function has historically only used the scheme, host, and port
277 # in the positional args. When an API change is acceptable these can
278 # be removed.
279 for key in ("scheme", "host", "port"):
280 request_context.pop(key, None)
282 if scheme == "http":
283 for kw in SSL_KEYWORDS:
284 request_context.pop(kw, None)
286 return pool_cls(host, port, **request_context)
288 def clear(self) -> None:
289 """
290 Empty our store of pools and direct them all to close.
292 This will not affect in-flight connections, but they will not be
293 re-used after completion.
294 """
295 self.pools.clear()
297 def connection_from_host(
298 self,
299 host: str | None,
300 port: int | None = None,
301 scheme: str | None = "http",
302 pool_kwargs: dict[str, typing.Any] | None = None,
303 ) -> HTTPConnectionPool:
304 """
305 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
307 If ``port`` isn't given, it will be derived from the ``scheme`` using
308 ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
309 provided, it is merged with the instance's ``connection_pool_kw``
310 variable and used to create the new connection pool, if one is
311 needed.
312 """
314 if not host:
315 raise LocationValueError("No host specified.")
317 request_context = self._merge_pool_kwargs(pool_kwargs)
318 request_context["scheme"] = scheme or "http"
319 if port is None:
320 port = port_by_scheme.get(request_context["scheme"].lower(), 80)
321 request_context["port"] = port
322 request_context["host"] = host
324 return self.connection_from_context(request_context)
326 def connection_from_context(
327 self, request_context: dict[str, typing.Any]
328 ) -> HTTPConnectionPool:
329 """
330 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
332 ``request_context`` must at least contain the ``scheme`` key and its
333 value must be a key in ``key_fn_by_scheme`` instance variable.
334 """
335 if "strict" in request_context:
336 warnings.warn(
337 "The 'strict' parameter is no longer needed on Python 3+. "
338 "This will raise an error in urllib3 v3.0.",
339 FutureWarning,
340 )
341 request_context.pop("strict")
343 scheme = request_context["scheme"].lower()
344 pool_key_constructor = self.key_fn_by_scheme.get(scheme)
345 if not pool_key_constructor:
346 raise URLSchemeUnknown(scheme)
347 pool_key = pool_key_constructor(request_context)
349 return self.connection_from_pool_key(pool_key, request_context=request_context)
351 def connection_from_pool_key(
352 self, pool_key: PoolKey, request_context: dict[str, typing.Any]
353 ) -> HTTPConnectionPool:
354 """
355 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
357 ``pool_key`` should be a namedtuple that only contains immutable
358 objects. At a minimum it must have the ``scheme``, ``host``, and
359 ``port`` fields.
360 """
361 with self.pools.lock:
362 # If the scheme, host, or port doesn't match existing open
363 # connections, open a new ConnectionPool.
364 pool = self.pools.get(pool_key)
365 if pool:
366 return pool
368 # Make a fresh ConnectionPool of the desired type
369 scheme = request_context["scheme"]
370 host = request_context["host"]
371 port = request_context["port"]
372 pool = self._new_pool(scheme, host, port, request_context=request_context)
373 self.pools[pool_key] = pool
375 return pool
377 def connection_from_url(
378 self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
379 ) -> HTTPConnectionPool:
380 """
381 Similar to :func:`urllib3.connectionpool.connection_from_url`.
383 If ``pool_kwargs`` is not provided and a new pool needs to be
384 constructed, ``self.connection_pool_kw`` is used to initialize
385 the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
386 is provided, it is used instead. Note that if a new pool does not
387 need to be created for the request, the provided ``pool_kwargs`` are
388 not used.
389 """
390 u = parse_url(url)
391 return self.connection_from_host(
392 u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
393 )
395 def _merge_pool_kwargs(
396 self, override: dict[str, typing.Any] | None
397 ) -> dict[str, typing.Any]:
398 """
399 Merge a dictionary of override values for self.connection_pool_kw.
401 This does not modify self.connection_pool_kw and returns a new dict.
402 Any keys in the override dictionary with a value of ``None`` are
403 removed from the merged dictionary.
404 """
405 base_pool_kwargs = self.connection_pool_kw.copy()
406 if override:
407 for key, value in override.items():
408 if value is None:
409 try:
410 del base_pool_kwargs[key]
411 except KeyError:
412 pass
413 else:
414 base_pool_kwargs[key] = value
415 return base_pool_kwargs
417 def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
418 """
419 Indicates if the proxy requires the complete destination URL in the
420 request. Normally this is only needed when not using an HTTP CONNECT
421 tunnel.
422 """
423 if self.proxy is None:
424 return False
426 return not connection_requires_http_tunnel(
427 self.proxy, self.proxy_config, parsed_url.scheme
428 )
430 def urlopen( # type: ignore[override]
431 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
432 ) -> BaseHTTPResponse:
433 """
434 Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
435 with custom cross-host redirect logic and only sends the request-uri
436 portion of the ``url``.
438 The given ``url`` parameter must be absolute, such that an appropriate
439 :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
440 """
441 u = parse_url(url)
443 if u.scheme is None:
444 warnings.warn(
445 "URLs without a scheme (ie 'https://') are deprecated and will raise an error "
446 "in urllib3 v3.0. To avoid this FutureWarning ensure all URLs "
447 "start with 'https://' or 'http://'. Read more in this issue: "
448 "https://github.com/urllib3/urllib3/issues/2920",
449 category=FutureWarning,
450 stacklevel=2,
451 )
453 conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
455 kw["assert_same_host"] = False
456 kw["redirect"] = False
458 if "headers" not in kw:
459 kw["headers"] = self.headers
461 if self._proxy_requires_url_absolute_form(u):
462 # Strip the fragment here but let the connection pool normalize the URL
463 # to avoid decoding IPv6 zone identifiers twice.
464 response = conn.urlopen(method, url.split("#", 1)[0], **kw)
465 else:
466 response = conn.urlopen(method, u.request_uri, **kw)
468 redirect_location = redirect and response.get_redirect_location()
469 if not redirect_location:
470 return response
472 # Support relative URLs for redirecting.
473 redirect_location = urljoin(url, redirect_location)
475 if response.status == 303:
476 # Change the method according to RFC 9110, Section 15.4.4.
477 method = "GET"
478 # And lose the body not to transfer anything sensitive.
479 kw["body"] = None
480 # The body is gone, so the state that describes it has to go too:
481 # there is nothing left to frame with chunked transfer encoding,
482 # and nothing left to rewind.
483 kw["chunked"] = False
484 kw["body_pos"] = None
485 kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
487 retries = kw.get("retries", response.retries)
488 if not isinstance(retries, Retry):
489 retries = Retry.from_int(retries, redirect=redirect)
491 # Strip headers marked as unsafe to forward to the redirected location.
492 # Check remove_headers_on_redirect to avoid a potential network call within
493 # conn.is_same_host() which may use socket.gethostbyname() in the future.
494 if retries.remove_headers_on_redirect and not conn.is_same_host(
495 redirect_location
496 ):
497 new_headers = kw["headers"].copy()
498 for header in kw["headers"]:
499 if header.lower() in retries.remove_headers_on_redirect:
500 new_headers.pop(header, None)
501 kw["headers"] = new_headers
503 try:
504 retries = retries.increment(method, url, response=response, _pool=conn)
505 except MaxRetryError:
506 if retries.raise_on_redirect:
507 response.drain_conn()
508 raise
509 return response
511 kw["retries"] = retries
512 kw["redirect"] = redirect
514 log.info("Redirecting %s -> %s", url, redirect_location)
516 response.drain_conn()
517 return self.urlopen(method, redirect_location, **kw)
520class ProxyManager(PoolManager):
521 """
522 Behaves just like :class:`PoolManager`, but sends all requests through
523 the defined proxy, using the CONNECT method for HTTPS URLs.
525 :param proxy_url:
526 The URL of the proxy to be used.
528 :param proxy_headers:
529 A dictionary containing headers that will be sent to the proxy. In case
530 of HTTP they are being sent with each request, while in the
531 HTTPS/CONNECT case they are sent only once. Could be used for proxy
532 authentication.
534 :param proxy_ssl_context:
535 The proxy SSL context is used to establish the TLS connection to the
536 proxy when using HTTPS proxies.
538 :param use_forwarding_for_https:
539 (Defaults to False) If set to True will forward requests to the HTTPS
540 proxy to be made on behalf of the client instead of creating a TLS
541 tunnel via the CONNECT method. **Enabling this flag means that request
542 and response headers and content will be visible from the HTTPS proxy**
543 whereas tunneling keeps request and response headers and content
544 private. IP address, target hostname, SNI, and port are always visible
545 to an HTTPS proxy even when this flag is disabled.
547 :param proxy_assert_hostname:
548 The hostname of the certificate to verify against.
550 :param proxy_assert_fingerprint:
551 The fingerprint of the certificate to verify against.
553 Example:
555 .. code-block:: python
557 import urllib3
559 proxy = urllib3.ProxyManager("https://localhost:3128/")
561 resp1 = proxy.request("GET", "http://google.com/")
562 resp2 = proxy.request("GET", "http://httpbin.org/")
564 # One pool was shared by both plain HTTP requests.
565 print(len(proxy.pools))
566 # 1
568 resp3 = proxy.request("GET", "https://httpbin.org/")
569 resp4 = proxy.request("GET", "https://twitter.com/")
571 # A separate pool was added for each HTTPS target.
572 print(len(proxy.pools))
573 # 3
575 """
577 def __init__(
578 self,
579 proxy_url: str,
580 num_pools: int = 10,
581 headers: typing.Mapping[str, str] | None = None,
582 proxy_headers: typing.Mapping[str, str] | None = None,
583 proxy_ssl_context: ssl.SSLContext | None = None,
584 use_forwarding_for_https: bool = False,
585 proxy_assert_hostname: None | str | typing.Literal[False] = None,
586 proxy_assert_fingerprint: str | None = None,
587 **connection_pool_kw: typing.Any,
588 ) -> None:
589 if isinstance(proxy_url, HTTPConnectionPool):
590 str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
591 else:
592 str_proxy_url = proxy_url
593 proxy = parse_url(str_proxy_url)
595 if proxy.scheme not in ("http", "https"):
596 raise ProxySchemeUnknown(proxy.scheme)
598 # Keep the deprecated ssl_context fallback on the manager for
599 # compatibility, while passing only explicit proxy policy to connections.
600 self.proxy_ssl_context = proxy_ssl_context
601 if (
602 use_forwarding_for_https
603 and proxy.scheme == "https"
604 and connection_pool_kw.get("ssl_context") is not None
605 ):
606 warnings.warn(
607 "Passing ssl_context when use_forwarding_for_https=True is deprecated "
608 "and will raise an error in urllib3 v3.0. "
609 "Use proxy_ssl_context to configure the TLS connection to the proxy.",
610 FutureWarning,
611 stacklevel=2,
612 )
613 if self.proxy_ssl_context is None:
614 self.proxy_ssl_context = connection_pool_kw.get("ssl_context")
616 if proxy.port is None:
617 port = port_by_scheme.get(proxy.scheme, 80)
618 proxy = proxy._replace(port=port)
620 self.proxy = proxy
621 self.proxy_headers = proxy_headers or {}
622 self.proxy_config = ProxyConfig(
623 proxy_ssl_context,
624 use_forwarding_for_https,
625 proxy_assert_hostname,
626 proxy_assert_fingerprint,
627 )
629 connection_pool_kw["_proxy"] = self.proxy
630 connection_pool_kw["_proxy_headers"] = self.proxy_headers
631 connection_pool_kw["_proxy_config"] = self.proxy_config
633 super().__init__(num_pools, headers, **connection_pool_kw)
635 def connection_from_host(
636 self,
637 host: str | None,
638 port: int | None = None,
639 scheme: str | None = "http",
640 pool_kwargs: dict[str, typing.Any] | None = None,
641 ) -> HTTPConnectionPool:
642 if scheme == "https":
643 return super().connection_from_host(
644 host, port, scheme, pool_kwargs=pool_kwargs
645 )
647 return super().connection_from_host(
648 self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
649 )
651 def _set_proxy_headers(
652 self, url: str, headers: typing.Mapping[str, str] | None = None
653 ) -> typing.Mapping[str, str]:
654 """
655 Sets headers needed by proxies: specifically, the Accept and Host
656 headers. Only sets headers not provided by the user.
657 """
658 headers_ = {"Accept": "*/*"}
660 netloc = parse_url(url).netloc
661 if netloc:
662 headers_["Host"] = netloc
664 if headers:
665 headers_.update(headers)
666 return headers_
668 def urlopen( # type: ignore[override]
669 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
670 ) -> BaseHTTPResponse:
671 "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
672 u = parse_url(url)
673 if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
674 # For connections using HTTP CONNECT, httplib sets the necessary
675 # headers on the CONNECT to the proxy. If we're not using CONNECT,
676 # we'll definitely need to set 'Host' at the very least.
677 headers = kw.get("headers", self.headers)
678 kw["headers"] = self._set_proxy_headers(url, headers)
680 return super().urlopen(method, url, redirect=redirect, **kw)
683def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
684 return ProxyManager(proxy_url=url, **kw)