Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/poolmanager.py: 33%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import functools
4import logging
5import typing
6import warnings
7from types import TracebackType
8from urllib.parse import urljoin
10from ._collections import HTTPHeaderDict, RecentlyUsedContainer
11from ._request_methods import RequestMethods
12from .connection import ProxyConfig
13from .connectionpool import (
14 HTTPConnectionPool,
15 HTTPSConnectionPool,
16 _is_same_host,
17 _normalize_host,
18 port_by_scheme,
19)
20from .exceptions import (
21 LocationValueError,
22 MaxRetryError,
23 ProxySchemeUnknown,
24 URLSchemeUnknown,
25)
26from .response import BaseHTTPResponse
27from .util.connection import _TYPE_SOCKET_OPTIONS
28from .util.proxy import connection_requires_http_tunnel
29from .util.retry import Retry
30from .util.timeout import Timeout
31from .util.url import Url, parse_url
33if typing.TYPE_CHECKING:
34 import ssl
36 from typing_extensions import Self
38__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
41log = logging.getLogger(__name__)
43SSL_KEYWORDS = (
44 "key_file",
45 "cert_file",
46 "cert_reqs",
47 "ca_certs",
48 "ca_cert_data",
49 "ssl_version",
50 "ssl_minimum_version",
51 "ssl_maximum_version",
52 "ca_cert_dir",
53 "ssl_context",
54 "key_password",
55 "server_hostname",
56 "assert_hostname",
57 "assert_fingerprint",
58)
59# Default value for `blocksize` - a new parameter introduced to
60# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
61_DEFAULT_BLOCKSIZE = 16384
64class PoolKey(typing.NamedTuple):
65 """
66 All known keyword arguments that could be provided to the pool manager, its
67 pools, or the underlying connections.
69 All custom key schemes should include the fields in this key at a minimum.
70 """
72 key_scheme: str
73 key_host: str
74 key_port: int | None
75 key_timeout: Timeout | float | int | None
76 key_retries: Retry | bool | int | None
77 key_block: bool | None
78 key_source_address: tuple[str, int] | None
79 key_key_file: str | None
80 key_key_password: str | None
81 key_cert_file: str | None
82 key_cert_reqs: str | None
83 key_ca_certs: str | None
84 key_ca_cert_data: str | bytes | None
85 key_ssl_version: int | str | None
86 key_ssl_minimum_version: ssl.TLSVersion | None
87 key_ssl_maximum_version: ssl.TLSVersion | None
88 key_ca_cert_dir: str | None
89 key_ssl_context: ssl.SSLContext | None
90 key_maxsize: int | None
91 key_headers: frozenset[tuple[str, str]] | None
92 key__proxy: Url | None
93 key__proxy_headers: frozenset[tuple[str, str]] | None
94 key__proxy_config: ProxyConfig | None
95 key_socket_options: _TYPE_SOCKET_OPTIONS | None
96 key__socks_options: frozenset[tuple[str, str]] | None
97 key_assert_hostname: bool | str | None
98 key_assert_fingerprint: str | None
99 key_server_hostname: str | None
100 key_blocksize: int | None
103def _default_key_normalizer(
104 key_class: type[PoolKey], request_context: dict[str, typing.Any]
105) -> PoolKey:
106 """
107 Create a pool key out of a request context dictionary.
109 According to RFC 3986, both the scheme and host are case-insensitive.
110 Therefore, this function normalizes both before constructing the pool
111 key for an HTTPS request. If you wish to change this behaviour, provide
112 alternate callables to ``key_fn_by_scheme``.
114 :param key_class:
115 The class to use when constructing the key. This should be a namedtuple
116 with the ``scheme`` and ``host`` keys at a minimum.
117 :type key_class: namedtuple
118 :param request_context:
119 A dictionary-like object that contain the context for a request.
120 :type request_context: dict
122 :return: A namedtuple that can be used as a connection pool key.
123 :rtype: PoolKey
124 """
125 # Since we mutate the dictionary, make a copy first
126 context = request_context.copy()
127 context["scheme"] = context["scheme"].lower()
128 # Only the address portion of a scoped IPv6 host is case-insensitive.
129 address, separator, zone = context["host"].partition("%")
130 if separator and ":" in address:
131 context["host"] = f"{address.lower()}{separator}{zone}"
132 else:
133 context["host"] = context["host"].lower()
135 # These are both dictionaries and need to be transformed into frozensets
136 for key in ("headers", "_proxy_headers", "_socks_options"):
137 if key in context and context[key] is not None:
138 context[key] = frozenset(context[key].items())
140 # The socket_options key may be a list and needs to be transformed into a
141 # tuple.
142 socket_opts = context.get("socket_options")
143 if socket_opts is not None:
144 context["socket_options"] = tuple(socket_opts)
146 # Map the kwargs to the names in the namedtuple - this is necessary since
147 # namedtuples can't have fields starting with '_'.
148 for key in list(context.keys()):
149 context["key_" + key] = context.pop(key)
151 # Default to ``None`` for keys missing from the context
152 for field in key_class._fields:
153 if field not in context:
154 context[field] = None
156 # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
157 if context.get("key_blocksize") is None:
158 context["key_blocksize"] = _DEFAULT_BLOCKSIZE
160 return key_class(**context)
163#: A dictionary that maps a scheme to a callable that creates a pool key.
164#: This can be used to alter the way pool keys are constructed, if desired.
165#: Each PoolManager makes a copy of this dictionary so they can be configured
166#: globally here, or individually on the instance.
167key_fn_by_scheme = {
168 "http": functools.partial(_default_key_normalizer, PoolKey),
169 "https": functools.partial(_default_key_normalizer, PoolKey),
170}
172pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
175class PoolManager(RequestMethods):
176 """
177 Allows for arbitrary requests while transparently keeping track of
178 necessary connection pools for you.
180 :param num_pools:
181 Number of connection pools to cache before discarding the least
182 recently used pool.
184 :param headers:
185 Headers to include with all requests, unless other headers are given
186 explicitly.
188 :param \\**connection_pool_kw:
189 Additional parameters are used to create fresh
190 :class:`urllib3.connectionpool.ConnectionPool` instances.
192 Example:
194 .. code-block:: python
196 import urllib3
198 http = urllib3.PoolManager(num_pools=2)
200 resp1 = http.request("GET", "https://google.com/")
201 resp2 = http.request("GET", "https://google.com/mail")
202 resp3 = http.request("GET", "https://yahoo.com/")
204 print(len(http.pools))
205 # 2
207 """
209 proxy: Url | None = None
210 proxy_config: ProxyConfig | None = None
212 def __init__(
213 self,
214 num_pools: int = 10,
215 headers: typing.Mapping[str, str] | None = None,
216 **connection_pool_kw: typing.Any,
217 ) -> None:
218 super().__init__(headers)
219 # PoolManager handles redirects itself in PoolManager.urlopen().
220 # It always passes redirect=False to the underlying connection pool to
221 # suppress per-pool redirect handling. If the user supplied a non-Retry
222 # value (int/bool/etc) for retries and we let the pool normalize it
223 # while redirect=False, the resulting Retry object would have redirect
224 # handling disabled, which can interfere with PoolManager's own
225 # redirect logic. Normalize here so redirects remain governed solely by
226 # PoolManager logic.
227 if "retries" in connection_pool_kw:
228 retries = connection_pool_kw["retries"]
229 if not isinstance(retries, Retry):
230 retries = Retry.from_int(retries)
231 connection_pool_kw = connection_pool_kw.copy()
232 connection_pool_kw["retries"] = retries
233 self.connection_pool_kw = connection_pool_kw
235 self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
236 self.pools = RecentlyUsedContainer(num_pools)
238 # Locally set the pool classes and keys so other PoolManagers can
239 # override them.
240 self.pool_classes_by_scheme = pool_classes_by_scheme
241 self.key_fn_by_scheme = key_fn_by_scheme.copy()
243 def __enter__(self) -> Self:
244 return self
246 def __exit__(
247 self,
248 exc_type: type[BaseException] | None,
249 exc_val: BaseException | None,
250 exc_tb: TracebackType | None,
251 ) -> typing.Literal[False]:
252 self.clear()
253 # Return False to re-raise any potential exceptions
254 return False
256 def _new_pool(
257 self,
258 scheme: str,
259 host: str,
260 port: int,
261 request_context: dict[str, typing.Any] | None = None,
262 ) -> HTTPConnectionPool:
263 """
264 Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
265 any additional pool keyword arguments.
267 If ``request_context`` is provided, it is provided as keyword arguments
268 to the pool class used. This method is used to actually create the
269 connection pools handed out by :meth:`connection_from_url` and
270 companion methods. It is intended to be overridden for customization.
271 """
272 pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
273 if request_context is None:
274 request_context = self.connection_pool_kw.copy()
276 # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
277 # set to 'None' in the request_context.
278 if request_context.get("blocksize") is None:
279 request_context["blocksize"] = _DEFAULT_BLOCKSIZE
281 # Although the context has everything necessary to create the pool,
282 # this function has historically only used the scheme, host, and port
283 # in the positional args. When an API change is acceptable these can
284 # be removed.
285 for key in ("scheme", "host", "port"):
286 request_context.pop(key, None)
288 if scheme == "http":
289 for kw in SSL_KEYWORDS:
290 request_context.pop(kw, None)
292 return pool_cls(host, port, **request_context)
294 def clear(self) -> None:
295 """
296 Empty our store of pools and direct them all to close.
298 This will not affect in-flight connections, but they will not be
299 re-used after completion.
300 """
301 self.pools.clear()
303 def connection_from_host(
304 self,
305 host: str | None,
306 port: int | None = None,
307 scheme: str | None = "http",
308 pool_kwargs: dict[str, typing.Any] | None = None,
309 ) -> HTTPConnectionPool:
310 """
311 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
313 If ``port`` isn't given, it will be derived from the ``scheme`` using
314 ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
315 provided, it is merged with the instance's ``connection_pool_kw``
316 variable and used to create the new connection pool, if one is
317 needed.
318 """
320 if not host:
321 raise LocationValueError("No host specified.")
323 request_context = self._merge_pool_kwargs(pool_kwargs)
324 request_context["scheme"] = scheme or "http"
325 if port is None:
326 port = port_by_scheme.get(request_context["scheme"].lower(), 80)
327 request_context["port"] = port
328 request_context["host"] = host
330 return self.connection_from_context(request_context)
332 def connection_from_context(
333 self, request_context: dict[str, typing.Any]
334 ) -> HTTPConnectionPool:
335 """
336 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
338 ``request_context`` must at least contain the ``scheme`` key and its
339 value must be a key in ``key_fn_by_scheme`` instance variable.
340 """
341 if "strict" in request_context:
342 warnings.warn(
343 "The 'strict' parameter is no longer needed on Python 3+. "
344 "This will raise an error in urllib3 v3.0.",
345 FutureWarning,
346 )
347 request_context.pop("strict")
349 scheme = request_context["scheme"].lower()
350 pool_key_constructor = self.key_fn_by_scheme.get(scheme)
351 if not pool_key_constructor:
352 raise URLSchemeUnknown(scheme)
353 pool_key = pool_key_constructor(request_context)
355 return self.connection_from_pool_key(pool_key, request_context=request_context)
357 def connection_from_pool_key(
358 self, pool_key: PoolKey, request_context: dict[str, typing.Any]
359 ) -> HTTPConnectionPool:
360 """
361 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
363 ``pool_key`` should be a namedtuple that only contains immutable
364 objects. At a minimum it must have the ``scheme``, ``host``, and
365 ``port`` fields.
366 """
367 with self.pools.lock:
368 # If the scheme, host, or port doesn't match existing open
369 # connections, open a new ConnectionPool.
370 pool = self.pools.get(pool_key)
371 if pool:
372 return pool
374 # Make a fresh ConnectionPool of the desired type
375 scheme = request_context["scheme"]
376 host = request_context["host"]
377 port = request_context["port"]
378 pool = self._new_pool(scheme, host, port, request_context=request_context)
379 self.pools[pool_key] = pool
381 return pool
383 def connection_from_url(
384 self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
385 ) -> HTTPConnectionPool:
386 """
387 Similar to :func:`urllib3.connectionpool.connection_from_url`.
389 If ``pool_kwargs`` is not provided and a new pool needs to be
390 constructed, ``self.connection_pool_kw`` is used to initialize
391 the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
392 is provided, it is used instead. Note that if a new pool does not
393 need to be created for the request, the provided ``pool_kwargs`` are
394 not used.
395 """
396 u = parse_url(url)
397 return self.connection_from_host(
398 u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
399 )
401 def _merge_pool_kwargs(
402 self, override: dict[str, typing.Any] | None
403 ) -> dict[str, typing.Any]:
404 """
405 Merge a dictionary of override values for self.connection_pool_kw.
407 This does not modify self.connection_pool_kw and returns a new dict.
408 Any keys in the override dictionary with a value of ``None`` are
409 removed from the merged dictionary.
410 """
411 base_pool_kwargs = self.connection_pool_kw.copy()
412 if override:
413 for key, value in override.items():
414 if value is None:
415 try:
416 del base_pool_kwargs[key]
417 except KeyError:
418 pass
419 else:
420 base_pool_kwargs[key] = value
421 return base_pool_kwargs
423 def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
424 """
425 Indicates if the proxy requires the complete destination URL in the
426 request. Normally this is only needed when not using an HTTP CONNECT
427 tunnel.
428 """
429 if self.proxy is None:
430 return False
432 return not connection_requires_http_tunnel(
433 self.proxy, self.proxy_config, parsed_url.scheme
434 )
436 def urlopen( # type: ignore[override]
437 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
438 ) -> BaseHTTPResponse:
439 """
440 Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
441 with custom cross-host redirect logic and only sends the request-uri
442 portion of the ``url``.
444 The given ``url`` parameter must be absolute, such that an appropriate
445 :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
446 """
447 u = parse_url(url)
449 if u.scheme is None:
450 warnings.warn(
451 "URLs without a scheme (ie 'https://') are deprecated and will raise an error "
452 "in urllib3 v3.0. To avoid this FutureWarning ensure all URLs "
453 "start with 'https://' or 'http://'. Read more in this issue: "
454 "https://github.com/urllib3/urllib3/issues/2920",
455 category=FutureWarning,
456 stacklevel=2,
457 )
459 conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
461 kw["assert_same_host"] = False
462 kw["redirect"] = False
464 if "headers" not in kw:
465 kw["headers"] = self.headers
467 if self._proxy_requires_url_absolute_form(u):
468 # Strip the fragment here but let the connection pool normalize the URL
469 # to avoid decoding IPv6 zone identifiers twice.
470 response = conn.urlopen(method, url.split("#", 1)[0], **kw)
471 else:
472 response = conn.urlopen(method, u.request_uri, **kw)
474 redirect_location = redirect and response.get_redirect_location()
475 if not redirect_location:
476 return response
478 # Support relative URLs for redirecting.
479 redirect_location = urljoin(url, redirect_location)
481 if response.status == 303:
482 # Change the method according to RFC 9110, Section 15.4.4.
483 method = "GET"
484 # And lose the body not to transfer anything sensitive.
485 kw["body"] = None
486 # The body is gone, so the state that describes it has to go too:
487 # there is nothing left to frame with chunked transfer encoding,
488 # and nothing left to rewind.
489 kw["chunked"] = False
490 kw["body_pos"] = None
491 kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
493 retries = kw.get("retries", response.retries)
494 if not isinstance(retries, Retry):
495 retries = Retry.from_int(
496 retries, redirect=redirect, default=response.retries
497 )
499 # Strip headers marked as unsafe to forward to the redirected location,
500 # along with Host, which ProxyManager._set_proxy_headers() derives from
501 # the request URL and so has to be recomputed for the new location.
502 # 'conn' is the proxy's pool when forwarding through an HTTP proxy, so
503 # compare the request URL itself rather than asking the pool.
504 scheme = u.scheme or "http"
505 if kw["headers"] is not None and not _is_same_host(
506 redirect_location, scheme, _normalize_host(u.host, scheme=scheme), u.port
507 ):
508 headers_to_remove = retries.remove_headers_on_redirect | {"host"}
509 new_headers = kw["headers"].copy()
510 for header in kw["headers"]:
511 if header.lower() in headers_to_remove:
512 new_headers.pop(header, None)
513 kw["headers"] = new_headers
515 try:
516 retries = retries.increment(method, url, response=response, _pool=conn)
517 except MaxRetryError:
518 if retries.raise_on_redirect:
519 response.drain_conn()
520 raise
521 return response
523 kw["retries"] = retries
524 kw["redirect"] = redirect
526 log.info("Redirecting %s -> %s", url, redirect_location)
528 response.drain_conn()
529 retries.sleep_for_retry(response)
530 return self.urlopen(method, redirect_location, **kw)
533class ProxyManager(PoolManager):
534 """
535 Behaves just like :class:`PoolManager`, but sends all requests through
536 the defined proxy, using the CONNECT method for HTTPS URLs.
538 :param proxy_url:
539 The URL of the proxy to be used.
541 :param proxy_headers:
542 A dictionary containing headers that will be sent to the proxy. In case
543 of HTTP they are being sent with each request, while in the
544 HTTPS/CONNECT case they are sent only once. Could be used for proxy
545 authentication.
547 :param proxy_ssl_context:
548 The proxy SSL context is used to establish the TLS connection to the
549 proxy when using HTTPS proxies.
551 :param use_forwarding_for_https:
552 (Defaults to False) If set to True will forward requests to the HTTPS
553 proxy to be made on behalf of the client instead of creating a TLS
554 tunnel via the CONNECT method. **Enabling this flag means that request
555 and response headers and content will be visible from the HTTPS proxy**
556 whereas tunneling keeps request and response headers and content
557 private. IP address, target hostname, SNI, and port are always visible
558 to an HTTPS proxy even when this flag is disabled.
560 :param proxy_assert_hostname:
561 The hostname of the certificate to verify against.
563 :param proxy_assert_fingerprint:
564 The fingerprint of the certificate to verify against.
566 Example:
568 .. code-block:: python
570 import urllib3
572 proxy = urllib3.ProxyManager("https://localhost:3128/")
574 resp1 = proxy.request("GET", "http://google.com/")
575 resp2 = proxy.request("GET", "http://httpbin.org/")
577 # One pool was shared by both plain HTTP requests.
578 print(len(proxy.pools))
579 # 1
581 resp3 = proxy.request("GET", "https://httpbin.org/")
582 resp4 = proxy.request("GET", "https://twitter.com/")
584 # A separate pool was added for each HTTPS target.
585 print(len(proxy.pools))
586 # 3
588 """
590 def __init__(
591 self,
592 proxy_url: str,
593 num_pools: int = 10,
594 headers: typing.Mapping[str, str] | None = None,
595 proxy_headers: typing.Mapping[str, str] | None = None,
596 proxy_ssl_context: ssl.SSLContext | None = None,
597 use_forwarding_for_https: bool = False,
598 proxy_assert_hostname: None | str | typing.Literal[False] = None,
599 proxy_assert_fingerprint: str | None = None,
600 **connection_pool_kw: typing.Any,
601 ) -> None:
602 if isinstance(proxy_url, HTTPConnectionPool):
603 str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
604 else:
605 str_proxy_url = proxy_url
606 proxy = parse_url(str_proxy_url)
608 if proxy.scheme not in ("http", "https"):
609 raise ProxySchemeUnknown(proxy.scheme)
611 # Keep the deprecated ssl_context fallback on the manager for
612 # compatibility, while passing only explicit proxy policy to connections.
613 self.proxy_ssl_context = proxy_ssl_context
614 if (
615 use_forwarding_for_https
616 and proxy.scheme == "https"
617 and connection_pool_kw.get("ssl_context") is not None
618 ):
619 warnings.warn(
620 "Passing ssl_context when use_forwarding_for_https=True is deprecated "
621 "and will raise an error in urllib3 v3.0. "
622 "Use proxy_ssl_context to configure the TLS connection to the proxy.",
623 FutureWarning,
624 stacklevel=2,
625 )
626 if self.proxy_ssl_context is None:
627 self.proxy_ssl_context = connection_pool_kw.get("ssl_context")
629 if proxy.port is None:
630 port = port_by_scheme.get(proxy.scheme, 80)
631 proxy = proxy._replace(port=port)
633 self.proxy = proxy
634 self.proxy_headers = proxy_headers or {}
635 self.proxy_config = ProxyConfig(
636 proxy_ssl_context,
637 use_forwarding_for_https,
638 proxy_assert_hostname,
639 proxy_assert_fingerprint,
640 )
642 connection_pool_kw["_proxy"] = self.proxy
643 connection_pool_kw["_proxy_headers"] = self.proxy_headers
644 connection_pool_kw["_proxy_config"] = self.proxy_config
646 super().__init__(num_pools, headers, **connection_pool_kw)
648 def connection_from_host(
649 self,
650 host: str | None,
651 port: int | None = None,
652 scheme: str | None = "http",
653 pool_kwargs: dict[str, typing.Any] | None = None,
654 ) -> HTTPConnectionPool:
655 if scheme == "https":
656 return super().connection_from_host(
657 host, port, scheme, pool_kwargs=pool_kwargs
658 )
660 return super().connection_from_host(
661 self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
662 )
664 def _set_proxy_headers(
665 self, url: str, headers: typing.Mapping[str, str] | None = None
666 ) -> typing.Mapping[str, str]:
667 """
668 Sets headers needed by proxies: specifically, the Accept and Host
669 headers. Only sets headers not provided by the user.
670 """
671 headers_ = {"Accept": "*/*"}
673 netloc = parse_url(url).netloc
674 if netloc:
675 headers_["Host"] = netloc
677 if headers:
678 headers_.update(headers)
679 return headers_
681 def urlopen( # type: ignore[override]
682 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
683 ) -> BaseHTTPResponse:
684 "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
685 u = parse_url(url)
686 if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
687 # For connections using HTTP CONNECT, httplib sets the necessary
688 # headers on the CONNECT to the proxy. If we're not using CONNECT,
689 # we'll definitely need to set 'Host' at the very least.
690 headers = kw.get("headers", self.headers)
691 kw["headers"] = self._set_proxy_headers(url, headers)
693 return super().urlopen(method, url, redirect=redirect, **kw)
696def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
697 return ProxyManager(proxy_url=url, **kw)