Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/poolmanager.py: 36%
229 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 06:35 +0000
1from __future__ import annotations
3import functools
4import logging
5import typing
6import warnings
7from types import TracebackType
8from urllib.parse import urljoin
10from ._collections import RecentlyUsedContainer
11from ._request_methods import RequestMethods
12from .connection import ProxyConfig
13from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
14from .exceptions import (
15 LocationValueError,
16 MaxRetryError,
17 ProxySchemeUnknown,
18 URLSchemeUnknown,
19)
20from .response import BaseHTTPResponse
21from .util.connection import _TYPE_SOCKET_OPTIONS
22from .util.proxy import connection_requires_http_tunnel
23from .util.retry import Retry
24from .util.timeout import Timeout
25from .util.url import Url, parse_url
27if typing.TYPE_CHECKING:
28 import ssl
30 from typing_extensions import Literal
32__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
35log = logging.getLogger(__name__)
37SSL_KEYWORDS = (
38 "key_file",
39 "cert_file",
40 "cert_reqs",
41 "ca_certs",
42 "ssl_version",
43 "ssl_minimum_version",
44 "ssl_maximum_version",
45 "ca_cert_dir",
46 "ssl_context",
47 "key_password",
48 "server_hostname",
49)
50# Default value for `blocksize` - a new parameter introduced to
51# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
52_DEFAULT_BLOCKSIZE = 16384
54_SelfT = typing.TypeVar("_SelfT")
57class PoolKey(typing.NamedTuple):
58 """
59 All known keyword arguments that could be provided to the pool manager, its
60 pools, or the underlying connections.
62 All custom key schemes should include the fields in this key at a minimum.
63 """
65 key_scheme: str
66 key_host: str
67 key_port: int | None
68 key_timeout: Timeout | float | int | None
69 key_retries: Retry | bool | int | None
70 key_block: bool | None
71 key_source_address: tuple[str, int] | None
72 key_key_file: str | None
73 key_key_password: str | None
74 key_cert_file: str | None
75 key_cert_reqs: str | None
76 key_ca_certs: str | None
77 key_ssl_version: int | str | None
78 key_ssl_minimum_version: ssl.TLSVersion | None
79 key_ssl_maximum_version: ssl.TLSVersion | None
80 key_ca_cert_dir: str | None
81 key_ssl_context: ssl.SSLContext | None
82 key_maxsize: int | None
83 key_headers: frozenset[tuple[str, str]] | None
84 key__proxy: Url | None
85 key__proxy_headers: frozenset[tuple[str, str]] | None
86 key__proxy_config: ProxyConfig | None
87 key_socket_options: _TYPE_SOCKET_OPTIONS | None
88 key__socks_options: frozenset[tuple[str, str]] | None
89 key_assert_hostname: bool | str | None
90 key_assert_fingerprint: str | None
91 key_server_hostname: str | None
92 key_blocksize: int | None
95def _default_key_normalizer(
96 key_class: type[PoolKey], request_context: dict[str, typing.Any]
97) -> PoolKey:
98 """
99 Create a pool key out of a request context dictionary.
101 According to RFC 3986, both the scheme and host are case-insensitive.
102 Therefore, this function normalizes both before constructing the pool
103 key for an HTTPS request. If you wish to change this behaviour, provide
104 alternate callables to ``key_fn_by_scheme``.
106 :param key_class:
107 The class to use when constructing the key. This should be a namedtuple
108 with the ``scheme`` and ``host`` keys at a minimum.
109 :type key_class: namedtuple
110 :param request_context:
111 A dictionary-like object that contain the context for a request.
112 :type request_context: dict
114 :return: A namedtuple that can be used as a connection pool key.
115 :rtype: PoolKey
116 """
117 # Since we mutate the dictionary, make a copy first
118 context = request_context.copy()
119 context["scheme"] = context["scheme"].lower()
120 context["host"] = context["host"].lower()
122 # These are both dictionaries and need to be transformed into frozensets
123 for key in ("headers", "_proxy_headers", "_socks_options"):
124 if key in context and context[key] is not None:
125 context[key] = frozenset(context[key].items())
127 # The socket_options key may be a list and needs to be transformed into a
128 # tuple.
129 socket_opts = context.get("socket_options")
130 if socket_opts is not None:
131 context["socket_options"] = tuple(socket_opts)
133 # Map the kwargs to the names in the namedtuple - this is necessary since
134 # namedtuples can't have fields starting with '_'.
135 for key in list(context.keys()):
136 context["key_" + key] = context.pop(key)
138 # Default to ``None`` for keys missing from the context
139 for field in key_class._fields:
140 if field not in context:
141 context[field] = None
143 # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
144 if context.get("key_blocksize") is None:
145 context["key_blocksize"] = _DEFAULT_BLOCKSIZE
147 return key_class(**context)
150#: A dictionary that maps a scheme to a callable that creates a pool key.
151#: This can be used to alter the way pool keys are constructed, if desired.
152#: Each PoolManager makes a copy of this dictionary so they can be configured
153#: globally here, or individually on the instance.
154key_fn_by_scheme = {
155 "http": functools.partial(_default_key_normalizer, PoolKey),
156 "https": functools.partial(_default_key_normalizer, PoolKey),
157}
159pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
162class PoolManager(RequestMethods):
163 """
164 Allows for arbitrary requests while transparently keeping track of
165 necessary connection pools for you.
167 :param num_pools:
168 Number of connection pools to cache before discarding the least
169 recently used pool.
171 :param headers:
172 Headers to include with all requests, unless other headers are given
173 explicitly.
175 :param \\**connection_pool_kw:
176 Additional parameters are used to create fresh
177 :class:`urllib3.connectionpool.ConnectionPool` instances.
179 Example:
181 .. code-block:: python
183 import urllib3
185 http = urllib3.PoolManager(num_pools=2)
187 resp1 = http.request("GET", "https://google.com/")
188 resp2 = http.request("GET", "https://google.com/mail")
189 resp3 = http.request("GET", "https://yahoo.com/")
191 print(len(http.pools))
192 # 2
194 """
196 proxy: Url | None = None
197 proxy_config: ProxyConfig | None = None
199 def __init__(
200 self,
201 num_pools: int = 10,
202 headers: typing.Mapping[str, str] | None = None,
203 **connection_pool_kw: typing.Any,
204 ) -> None:
205 super().__init__(headers)
206 self.connection_pool_kw = connection_pool_kw
208 self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
209 self.pools = RecentlyUsedContainer(num_pools)
211 # Locally set the pool classes and keys so other PoolManagers can
212 # override them.
213 self.pool_classes_by_scheme = pool_classes_by_scheme
214 self.key_fn_by_scheme = key_fn_by_scheme.copy()
216 def __enter__(self: _SelfT) -> _SelfT:
217 return self
219 def __exit__(
220 self,
221 exc_type: type[BaseException] | None,
222 exc_val: BaseException | None,
223 exc_tb: TracebackType | None,
224 ) -> Literal[False]:
225 self.clear()
226 # Return False to re-raise any potential exceptions
227 return False
229 def _new_pool(
230 self,
231 scheme: str,
232 host: str,
233 port: int,
234 request_context: dict[str, typing.Any] | None = None,
235 ) -> HTTPConnectionPool:
236 """
237 Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
238 any additional pool keyword arguments.
240 If ``request_context`` is provided, it is provided as keyword arguments
241 to the pool class used. This method is used to actually create the
242 connection pools handed out by :meth:`connection_from_url` and
243 companion methods. It is intended to be overridden for customization.
244 """
245 pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
246 if request_context is None:
247 request_context = self.connection_pool_kw.copy()
249 # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
250 # set to 'None' in the request_context.
251 if request_context.get("blocksize") is None:
252 request_context["blocksize"] = _DEFAULT_BLOCKSIZE
254 # Although the context has everything necessary to create the pool,
255 # this function has historically only used the scheme, host, and port
256 # in the positional args. When an API change is acceptable these can
257 # be removed.
258 for key in ("scheme", "host", "port"):
259 request_context.pop(key, None)
261 if scheme == "http":
262 for kw in SSL_KEYWORDS:
263 request_context.pop(kw, None)
265 return pool_cls(host, port, **request_context)
267 def clear(self) -> None:
268 """
269 Empty our store of pools and direct them all to close.
271 This will not affect in-flight connections, but they will not be
272 re-used after completion.
273 """
274 self.pools.clear()
276 def connection_from_host(
277 self,
278 host: str | None,
279 port: int | None = None,
280 scheme: str | None = "http",
281 pool_kwargs: dict[str, typing.Any] | None = None,
282 ) -> HTTPConnectionPool:
283 """
284 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
286 If ``port`` isn't given, it will be derived from the ``scheme`` using
287 ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
288 provided, it is merged with the instance's ``connection_pool_kw``
289 variable and used to create the new connection pool, if one is
290 needed.
291 """
293 if not host:
294 raise LocationValueError("No host specified.")
296 request_context = self._merge_pool_kwargs(pool_kwargs)
297 request_context["scheme"] = scheme or "http"
298 if not port:
299 port = port_by_scheme.get(request_context["scheme"].lower(), 80)
300 request_context["port"] = port
301 request_context["host"] = host
303 return self.connection_from_context(request_context)
305 def connection_from_context(
306 self, request_context: dict[str, typing.Any]
307 ) -> HTTPConnectionPool:
308 """
309 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
311 ``request_context`` must at least contain the ``scheme`` key and its
312 value must be a key in ``key_fn_by_scheme`` instance variable.
313 """
314 if "strict" in request_context:
315 warnings.warn(
316 "The 'strict' parameter is no longer needed on Python 3+. "
317 "This will raise an error in urllib3 v2.1.0.",
318 DeprecationWarning,
319 )
320 request_context.pop("strict")
322 scheme = request_context["scheme"].lower()
323 pool_key_constructor = self.key_fn_by_scheme.get(scheme)
324 if not pool_key_constructor:
325 raise URLSchemeUnknown(scheme)
326 pool_key = pool_key_constructor(request_context)
328 return self.connection_from_pool_key(pool_key, request_context=request_context)
330 def connection_from_pool_key(
331 self, pool_key: PoolKey, request_context: dict[str, typing.Any]
332 ) -> HTTPConnectionPool:
333 """
334 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
336 ``pool_key`` should be a namedtuple that only contains immutable
337 objects. At a minimum it must have the ``scheme``, ``host``, and
338 ``port`` fields.
339 """
340 with self.pools.lock:
341 # If the scheme, host, or port doesn't match existing open
342 # connections, open a new ConnectionPool.
343 pool = self.pools.get(pool_key)
344 if pool:
345 return pool
347 # Make a fresh ConnectionPool of the desired type
348 scheme = request_context["scheme"]
349 host = request_context["host"]
350 port = request_context["port"]
351 pool = self._new_pool(scheme, host, port, request_context=request_context)
352 self.pools[pool_key] = pool
354 return pool
356 def connection_from_url(
357 self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
358 ) -> HTTPConnectionPool:
359 """
360 Similar to :func:`urllib3.connectionpool.connection_from_url`.
362 If ``pool_kwargs`` is not provided and a new pool needs to be
363 constructed, ``self.connection_pool_kw`` is used to initialize
364 the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
365 is provided, it is used instead. Note that if a new pool does not
366 need to be created for the request, the provided ``pool_kwargs`` are
367 not used.
368 """
369 u = parse_url(url)
370 return self.connection_from_host(
371 u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
372 )
374 def _merge_pool_kwargs(
375 self, override: dict[str, typing.Any] | None
376 ) -> dict[str, typing.Any]:
377 """
378 Merge a dictionary of override values for self.connection_pool_kw.
380 This does not modify self.connection_pool_kw and returns a new dict.
381 Any keys in the override dictionary with a value of ``None`` are
382 removed from the merged dictionary.
383 """
384 base_pool_kwargs = self.connection_pool_kw.copy()
385 if override:
386 for key, value in override.items():
387 if value is None:
388 try:
389 del base_pool_kwargs[key]
390 except KeyError:
391 pass
392 else:
393 base_pool_kwargs[key] = value
394 return base_pool_kwargs
396 def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
397 """
398 Indicates if the proxy requires the complete destination URL in the
399 request. Normally this is only needed when not using an HTTP CONNECT
400 tunnel.
401 """
402 if self.proxy is None:
403 return False
405 return not connection_requires_http_tunnel(
406 self.proxy, self.proxy_config, parsed_url.scheme
407 )
409 def urlopen( # type: ignore[override]
410 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
411 ) -> BaseHTTPResponse:
412 """
413 Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
414 with custom cross-host redirect logic and only sends the request-uri
415 portion of the ``url``.
417 The given ``url`` parameter must be absolute, such that an appropriate
418 :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
419 """
420 u = parse_url(url)
422 conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
424 kw["assert_same_host"] = False
425 kw["redirect"] = False
427 if "headers" not in kw:
428 kw["headers"] = self.headers
430 if self._proxy_requires_url_absolute_form(u):
431 response = conn.urlopen(method, url, **kw)
432 else:
433 response = conn.urlopen(method, u.request_uri, **kw)
435 redirect_location = redirect and response.get_redirect_location()
436 if not redirect_location:
437 return response
439 # Support relative URLs for redirecting.
440 redirect_location = urljoin(url, redirect_location)
442 # RFC 7231, Section 6.4.4
443 if response.status == 303:
444 method = "GET"
446 retries = kw.get("retries")
447 if not isinstance(retries, Retry):
448 retries = Retry.from_int(retries, redirect=redirect)
450 # Strip headers marked as unsafe to forward to the redirected location.
451 # Check remove_headers_on_redirect to avoid a potential network call within
452 # conn.is_same_host() which may use socket.gethostbyname() in the future.
453 if retries.remove_headers_on_redirect and not conn.is_same_host(
454 redirect_location
455 ):
456 new_headers = kw["headers"].copy()
457 for header in kw["headers"]:
458 if header.lower() in retries.remove_headers_on_redirect:
459 new_headers.pop(header, None)
460 kw["headers"] = new_headers
462 try:
463 retries = retries.increment(method, url, response=response, _pool=conn)
464 except MaxRetryError:
465 if retries.raise_on_redirect:
466 response.drain_conn()
467 raise
468 return response
470 kw["retries"] = retries
471 kw["redirect"] = redirect
473 log.info("Redirecting %s -> %s", url, redirect_location)
475 response.drain_conn()
476 return self.urlopen(method, redirect_location, **kw)
479class ProxyManager(PoolManager):
480 """
481 Behaves just like :class:`PoolManager`, but sends all requests through
482 the defined proxy, using the CONNECT method for HTTPS URLs.
484 :param proxy_url:
485 The URL of the proxy to be used.
487 :param proxy_headers:
488 A dictionary containing headers that will be sent to the proxy. In case
489 of HTTP they are being sent with each request, while in the
490 HTTPS/CONNECT case they are sent only once. Could be used for proxy
491 authentication.
493 :param proxy_ssl_context:
494 The proxy SSL context is used to establish the TLS connection to the
495 proxy when using HTTPS proxies.
497 :param use_forwarding_for_https:
498 (Defaults to False) If set to True will forward requests to the HTTPS
499 proxy to be made on behalf of the client instead of creating a TLS
500 tunnel via the CONNECT method. **Enabling this flag means that request
501 and response headers and content will be visible from the HTTPS proxy**
502 whereas tunneling keeps request and response headers and content
503 private. IP address, target hostname, SNI, and port are always visible
504 to an HTTPS proxy even when this flag is disabled.
506 :param proxy_assert_hostname:
507 The hostname of the certificate to verify against.
509 :param proxy_assert_fingerprint:
510 The fingerprint of the certificate to verify against.
512 Example:
514 .. code-block:: python
516 import urllib3
518 proxy = urllib3.ProxyManager("https://localhost:3128/")
520 resp1 = proxy.request("GET", "https://google.com/")
521 resp2 = proxy.request("GET", "https://httpbin.org/")
523 print(len(proxy.pools))
524 # 1
526 resp3 = proxy.request("GET", "https://httpbin.org/")
527 resp4 = proxy.request("GET", "https://twitter.com/")
529 print(len(proxy.pools))
530 # 3
532 """
534 def __init__(
535 self,
536 proxy_url: str,
537 num_pools: int = 10,
538 headers: typing.Mapping[str, str] | None = None,
539 proxy_headers: typing.Mapping[str, str] | None = None,
540 proxy_ssl_context: ssl.SSLContext | None = None,
541 use_forwarding_for_https: bool = False,
542 proxy_assert_hostname: None | str | Literal[False] = None,
543 proxy_assert_fingerprint: str | None = None,
544 **connection_pool_kw: typing.Any,
545 ) -> None:
546 if isinstance(proxy_url, HTTPConnectionPool):
547 str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
548 else:
549 str_proxy_url = proxy_url
550 proxy = parse_url(str_proxy_url)
552 if proxy.scheme not in ("http", "https"):
553 raise ProxySchemeUnknown(proxy.scheme)
555 if not proxy.port:
556 port = port_by_scheme.get(proxy.scheme, 80)
557 proxy = proxy._replace(port=port)
559 self.proxy = proxy
560 self.proxy_headers = proxy_headers or {}
561 self.proxy_ssl_context = proxy_ssl_context
562 self.proxy_config = ProxyConfig(
563 proxy_ssl_context,
564 use_forwarding_for_https,
565 proxy_assert_hostname,
566 proxy_assert_fingerprint,
567 )
569 connection_pool_kw["_proxy"] = self.proxy
570 connection_pool_kw["_proxy_headers"] = self.proxy_headers
571 connection_pool_kw["_proxy_config"] = self.proxy_config
573 super().__init__(num_pools, headers, **connection_pool_kw)
575 def connection_from_host(
576 self,
577 host: str | None,
578 port: int | None = None,
579 scheme: str | None = "http",
580 pool_kwargs: dict[str, typing.Any] | None = None,
581 ) -> HTTPConnectionPool:
582 if scheme == "https":
583 return super().connection_from_host(
584 host, port, scheme, pool_kwargs=pool_kwargs
585 )
587 return super().connection_from_host(
588 self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
589 )
591 def _set_proxy_headers(
592 self, url: str, headers: typing.Mapping[str, str] | None = None
593 ) -> typing.Mapping[str, str]:
594 """
595 Sets headers needed by proxies: specifically, the Accept and Host
596 headers. Only sets headers not provided by the user.
597 """
598 headers_ = {"Accept": "*/*"}
600 netloc = parse_url(url).netloc
601 if netloc:
602 headers_["Host"] = netloc
604 if headers:
605 headers_.update(headers)
606 return headers_
608 def urlopen( # type: ignore[override]
609 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
610 ) -> BaseHTTPResponse:
611 "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
612 u = parse_url(url)
613 if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
614 # For connections using HTTP CONNECT, httplib sets the necessary
615 # headers on the CONNECT to the proxy. If we're not using CONNECT,
616 # we'll definitely need to set 'Host' at the very least.
617 headers = kw.get("headers", self.headers)
618 kw["headers"] = self._set_proxy_headers(url, headers)
620 return super().urlopen(method, url, redirect=redirect, **kw)
623def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
624 return ProxyManager(proxy_url=url, **kw)