Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/requests/adapters.py: 22%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2requests.adapters
3~~~~~~~~~~~~~~~~~
5This module contains the transport adapters that Requests uses to define
6and maintain connections.
7"""
9import os.path
10import socket # noqa: F401
11import typing
12import warnings
14from urllib3.exceptions import (
15 ClosedPoolError,
16 ConnectTimeoutError,
17 LocationValueError,
18 MaxRetryError,
19 NewConnectionError,
20 ProtocolError,
21 ReadTimeoutError,
22 ResponseError,
23)
24from urllib3.exceptions import HTTPError as _HTTPError
25from urllib3.exceptions import InvalidHeader as _InvalidHeader
26from urllib3.exceptions import ProxyError as _ProxyError
27from urllib3.exceptions import SSLError as _SSLError
28from urllib3.poolmanager import PoolManager, proxy_from_url
29from urllib3.util import Timeout as TimeoutSauce
30from urllib3.util import parse_url
31from urllib3.util.retry import Retry
33from .auth import _basic_auth_str
34from .compat import basestring, urlparse
35from .cookies import extract_cookies_to_jar
36from .exceptions import (
37 ConnectionError,
38 ConnectTimeout,
39 InvalidHeader,
40 InvalidProxyURL,
41 InvalidSchema,
42 InvalidURL,
43 ProxyError,
44 ReadTimeout,
45 RetryError,
46 SSLError,
47)
48from .models import Response
49from .structures import CaseInsensitiveDict
50from .utils import (
51 DEFAULT_CA_BUNDLE_PATH,
52 get_auth_from_url,
53 get_encoding_from_headers,
54 prepend_scheme_if_needed,
55 select_proxy,
56 urldefragauth,
57)
59try:
60 from urllib3.contrib.socks import SOCKSProxyManager
61except ImportError:
63 def SOCKSProxyManager(*args, **kwargs):
64 raise InvalidSchema("Missing dependencies for SOCKS support.")
67if typing.TYPE_CHECKING:
68 from .models import PreparedRequest
71DEFAULT_POOLBLOCK = False
72DEFAULT_POOLSIZE = 10
73DEFAULT_RETRIES = 0
74DEFAULT_POOL_TIMEOUT = None
77def _urllib3_request_context(
78 request: "PreparedRequest",
79 verify: "bool | str | None",
80 client_cert: "tuple[str, str] | str | None",
81 poolmanager: "PoolManager",
82) -> "(dict[str, typing.Any], dict[str, typing.Any])":
83 host_params = {}
84 pool_kwargs = {}
85 parsed_request_url = urlparse(request.url)
86 scheme = parsed_request_url.scheme.lower()
87 port = parsed_request_url.port
89 cert_reqs = "CERT_REQUIRED"
90 if verify is False:
91 cert_reqs = "CERT_NONE"
92 elif isinstance(verify, str):
93 if not os.path.isdir(verify):
94 pool_kwargs["ca_certs"] = verify
95 else:
96 pool_kwargs["ca_cert_dir"] = verify
97 pool_kwargs["cert_reqs"] = cert_reqs
98 if client_cert is not None:
99 if isinstance(client_cert, tuple) and len(client_cert) == 2:
100 pool_kwargs["cert_file"] = client_cert[0]
101 pool_kwargs["key_file"] = client_cert[1]
102 else:
103 # According to our docs, we allow users to specify just the client
104 # cert path
105 pool_kwargs["cert_file"] = client_cert
106 host_params = {
107 "scheme": scheme,
108 "host": parsed_request_url.hostname,
109 "port": port,
110 }
111 return host_params, pool_kwargs
114class BaseAdapter:
115 """The Base Transport Adapter"""
117 def __init__(self):
118 super().__init__()
120 def send(
121 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
122 ):
123 """Sends PreparedRequest object. Returns Response object.
125 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
126 :param stream: (optional) Whether to stream the request content.
127 :param timeout: (optional) How long to wait for the server to send
128 data before giving up, as a float, or a :ref:`(connect timeout,
129 read timeout) <timeouts>` tuple.
130 :type timeout: float or tuple
131 :param verify: (optional) Either a boolean, in which case it controls whether we verify
132 the server's TLS certificate, or a string, in which case it must be a path
133 to a CA bundle to use
134 :param cert: (optional) Any user-provided SSL certificate to be trusted.
135 :param proxies: (optional) The proxies dictionary to apply to the request.
136 """
137 raise NotImplementedError
139 def close(self):
140 """Cleans up adapter specific items."""
141 raise NotImplementedError
144class HTTPAdapter(BaseAdapter):
145 """The built-in HTTP Adapter for urllib3.
147 Provides a general-case interface for Requests sessions to contact HTTP and
148 HTTPS urls by implementing the Transport Adapter interface. This class will
149 usually be created by the :class:`Session <Session>` class under the
150 covers.
152 :param pool_connections: The number of urllib3 connection pools to cache.
153 :param pool_maxsize: The maximum number of connections to save in the pool.
154 :param max_retries: The maximum number of retries each connection
155 should attempt. Note, this applies only to failed DNS lookups, socket
156 connections and connection timeouts, never to requests where data has
157 made it to the server. By default, Requests does not retry failed
158 connections. If you need granular control over the conditions under
159 which we retry a request, import urllib3's ``Retry`` class and pass
160 that instead.
161 :param pool_block: Whether the connection pool should block for connections.
163 Usage::
165 >>> import requests
166 >>> s = requests.Session()
167 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
168 >>> s.mount('http://', a)
169 """
171 __attrs__ = [
172 "max_retries",
173 "config",
174 "_pool_connections",
175 "_pool_maxsize",
176 "_pool_block",
177 ]
179 def __init__(
180 self,
181 pool_connections=DEFAULT_POOLSIZE,
182 pool_maxsize=DEFAULT_POOLSIZE,
183 max_retries=DEFAULT_RETRIES,
184 pool_block=DEFAULT_POOLBLOCK,
185 ):
186 if max_retries == DEFAULT_RETRIES:
187 self.max_retries = Retry(0, read=False)
188 else:
189 self.max_retries = Retry.from_int(max_retries)
190 self.config = {}
191 self.proxy_manager = {}
193 super().__init__()
195 self._pool_connections = pool_connections
196 self._pool_maxsize = pool_maxsize
197 self._pool_block = pool_block
199 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
201 def __getstate__(self):
202 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
204 def __setstate__(self, state):
205 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
206 # self.poolmanager uses a lambda function, which isn't pickleable.
207 self.proxy_manager = {}
208 self.config = {}
210 for attr, value in state.items():
211 setattr(self, attr, value)
213 self.init_poolmanager(
214 self._pool_connections, self._pool_maxsize, block=self._pool_block
215 )
217 def init_poolmanager(
218 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
219 ):
220 """Initializes a urllib3 PoolManager.
222 This method should not be called from user code, and is only
223 exposed for use when subclassing the
224 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
226 :param connections: The number of urllib3 connection pools to cache.
227 :param maxsize: The maximum number of connections to save in the pool.
228 :param block: Block when no free connections are available.
229 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
230 """
231 # save these values for pickling
232 self._pool_connections = connections
233 self._pool_maxsize = maxsize
234 self._pool_block = block
236 self.poolmanager = PoolManager(
237 num_pools=connections,
238 maxsize=maxsize,
239 block=block,
240 **pool_kwargs,
241 )
243 def proxy_manager_for(self, proxy, **proxy_kwargs):
244 """Return urllib3 ProxyManager for the given proxy.
246 This method should not be called from user code, and is only
247 exposed for use when subclassing the
248 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
250 :param proxy: The proxy to return a urllib3 ProxyManager for.
251 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
252 :returns: ProxyManager
253 :rtype: urllib3.ProxyManager
254 """
255 if proxy in self.proxy_manager:
256 manager = self.proxy_manager[proxy]
257 elif proxy.lower().startswith("socks"):
258 username, password = get_auth_from_url(proxy)
259 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
260 proxy,
261 username=username,
262 password=password,
263 num_pools=self._pool_connections,
264 maxsize=self._pool_maxsize,
265 block=self._pool_block,
266 **proxy_kwargs,
267 )
268 else:
269 proxy_headers = self.proxy_headers(proxy)
270 manager = self.proxy_manager[proxy] = proxy_from_url(
271 proxy,
272 proxy_headers=proxy_headers,
273 num_pools=self._pool_connections,
274 maxsize=self._pool_maxsize,
275 block=self._pool_block,
276 **proxy_kwargs,
277 )
279 return manager
281 def cert_verify(self, conn, url, verify, cert):
282 """Verify a SSL certificate. This method should not be called from user
283 code, and is only exposed for use when subclassing the
284 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
286 :param conn: The urllib3 connection object associated with the cert.
287 :param url: The requested URL.
288 :param verify: Either a boolean, in which case it controls whether we verify
289 the server's TLS certificate, or a string, in which case it must be a path
290 to a CA bundle to use
291 :param cert: The SSL certificate to verify.
292 """
293 if url.lower().startswith("https") and verify:
294 cert_loc = None
296 # Allow self-specified cert location.
297 if verify is not True:
298 cert_loc = verify
300 if not cert_loc:
301 cert_loc = DEFAULT_CA_BUNDLE_PATH
303 if not cert_loc or not os.path.exists(cert_loc):
304 raise OSError(
305 f"Could not find a suitable TLS CA certificate bundle, "
306 f"invalid path: {cert_loc}"
307 )
309 conn.cert_reqs = "CERT_REQUIRED"
311 if not os.path.isdir(cert_loc):
312 conn.ca_certs = cert_loc
313 else:
314 conn.ca_cert_dir = cert_loc
315 else:
316 conn.cert_reqs = "CERT_NONE"
317 conn.ca_certs = None
318 conn.ca_cert_dir = None
320 if cert:
321 if not isinstance(cert, basestring):
322 conn.cert_file = cert[0]
323 conn.key_file = cert[1]
324 else:
325 conn.cert_file = cert
326 conn.key_file = None
327 if conn.cert_file and not os.path.exists(conn.cert_file):
328 raise OSError(
329 f"Could not find the TLS certificate file, "
330 f"invalid path: {conn.cert_file}"
331 )
332 if conn.key_file and not os.path.exists(conn.key_file):
333 raise OSError(
334 f"Could not find the TLS key file, invalid path: {conn.key_file}"
335 )
337 def build_response(self, req, resp):
338 """Builds a :class:`Response <requests.Response>` object from a urllib3
339 response. This should not be called from user code, and is only exposed
340 for use when subclassing the
341 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
343 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
344 :param resp: The urllib3 response object.
345 :rtype: requests.Response
346 """
347 response = Response()
349 # Fallback to None if there's no status_code, for whatever reason.
350 response.status_code = getattr(resp, "status", None)
352 # Make headers case-insensitive.
353 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
355 # Set encoding.
356 response.encoding = get_encoding_from_headers(response.headers)
357 response.raw = resp
358 response.reason = response.raw.reason
360 if isinstance(req.url, bytes):
361 response.url = req.url.decode("utf-8")
362 else:
363 response.url = req.url
365 # Add new cookies from the server.
366 extract_cookies_to_jar(response.cookies, req, resp)
368 # Give the Response some context.
369 response.request = req
370 response.connection = self
372 return response
374 def build_connection_pool_key_attributes(self, request, verify, cert=None):
375 """Build the PoolKey attributes used by urllib3 to return a connection.
377 This looks at the PreparedRequest, the user-specified verify value,
378 and the value of the cert parameter to determine what PoolKey values
379 to use to select a connection from a given urllib3 Connection Pool.
381 The SSL related pool key arguments are not consistently set. As of
382 this writing, use the following to determine what keys may be in that
383 dictionary:
385 * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
386 default Requests SSL Context
387 * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
388 ``"cert_reqs"`` will be set
389 * If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
390 ``"ca_certs"`` will be set if the string is not a directory recognized
391 by :py:func:`os.path.isdir`, otherwise ``"ca_cert_dir"`` will be
392 set.
393 * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
394 ``"cert"`` is a tuple with a second item, ``"key_file"`` will also
395 be present
397 To override these settings, one may subclass this class, call this
398 method and use the above logic to change parameters as desired. For
399 example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
400 must both set ``"ssl_context"`` and based on what else they require,
401 alter the other keys to ensure the desired behaviour.
403 :param request:
404 The PreparedReqest being sent over the connection.
405 :type request:
406 :class:`~requests.models.PreparedRequest`
407 :param verify:
408 Either a boolean, in which case it controls whether
409 we verify the server's TLS certificate, or a string, in which case it
410 must be a path to a CA bundle to use.
411 :param cert:
412 (optional) Any user-provided SSL certificate for client
413 authentication (a.k.a., mTLS). This may be a string (i.e., just
414 the path to a file which holds both certificate and key) or a
415 tuple of length 2 with the certificate file path and key file
416 path.
417 :returns:
418 A tuple of two dictionaries. The first is the "host parameters"
419 portion of the Pool Key including scheme, hostname, and port. The
420 second is a dictionary of SSLContext related parameters.
421 """
422 return _urllib3_request_context(request, verify, cert, self.poolmanager)
424 def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
425 """Returns a urllib3 connection for the given request and TLS settings.
426 This should not be called from user code, and is only exposed for use
427 when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
429 :param request:
430 The :class:`PreparedRequest <PreparedRequest>` object to be sent
431 over the connection.
432 :param verify:
433 Either a boolean, in which case it controls whether we verify the
434 server's TLS certificate, or a string, in which case it must be a
435 path to a CA bundle to use.
436 :param proxies:
437 (optional) The proxies dictionary to apply to the request.
438 :param cert:
439 (optional) Any user-provided SSL certificate to be used for client
440 authentication (a.k.a., mTLS).
441 :rtype:
442 urllib3.ConnectionPool
443 """
444 proxy = select_proxy(request.url, proxies)
445 try:
446 host_params, pool_kwargs = self.build_connection_pool_key_attributes(
447 request,
448 verify,
449 cert,
450 )
451 except ValueError as e:
452 raise InvalidURL(e, request=request)
453 if proxy:
454 proxy = prepend_scheme_if_needed(proxy, "http")
455 proxy_url = parse_url(proxy)
456 if not proxy_url.host:
457 raise InvalidProxyURL(
458 "Please check proxy URL. It is malformed "
459 "and could be missing the host."
460 )
461 proxy_manager = self.proxy_manager_for(proxy)
462 conn = proxy_manager.connection_from_host(
463 **host_params, pool_kwargs=pool_kwargs
464 )
465 else:
466 # Only scheme should be lower case
467 conn = self.poolmanager.connection_from_host(
468 **host_params, pool_kwargs=pool_kwargs
469 )
471 return conn
473 def get_connection(self, url, proxies=None):
474 """DEPRECATED: Users should move to `get_connection_with_tls_context`
475 for all subclasses of HTTPAdapter using Requests>=2.32.2.
477 Returns a urllib3 connection for the given URL. This should not be
478 called from user code, and is only exposed for use when subclassing the
479 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
481 :param url: The URL to connect to.
482 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
483 :rtype: urllib3.ConnectionPool
484 """
485 warnings.warn(
486 (
487 "`get_connection` has been deprecated in favor of "
488 "`get_connection_with_tls_context`. Custom HTTPAdapter subclasses "
489 "will need to migrate for Requests>=2.32.2. Please see "
490 "https://github.com/psf/requests/pull/6710 for more details."
491 ),
492 DeprecationWarning,
493 )
494 proxy = select_proxy(url, proxies)
496 if proxy:
497 proxy = prepend_scheme_if_needed(proxy, "http")
498 proxy_url = parse_url(proxy)
499 if not proxy_url.host:
500 raise InvalidProxyURL(
501 "Please check proxy URL. It is malformed "
502 "and could be missing the host."
503 )
504 proxy_manager = self.proxy_manager_for(proxy)
505 conn = proxy_manager.connection_from_url(url)
506 else:
507 # Only scheme should be lower case
508 parsed = urlparse(url)
509 url = parsed.geturl()
510 conn = self.poolmanager.connection_from_url(url)
512 return conn
514 def close(self):
515 """Disposes of any internal state.
517 Currently, this closes the PoolManager and any active ProxyManager,
518 which closes any pooled connections.
519 """
520 self.poolmanager.clear()
521 for proxy in self.proxy_manager.values():
522 proxy.clear()
524 def request_url(self, request, proxies):
525 """Obtain the url to use when making the final request.
527 If the message is being sent through a HTTP proxy, the full URL has to
528 be used. Otherwise, we should only use the path portion of the URL.
530 This should not be called from user code, and is only exposed for use
531 when subclassing the
532 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
534 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
535 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
536 :rtype: str
537 """
538 proxy = select_proxy(request.url, proxies)
539 scheme = urlparse(request.url).scheme
541 is_proxied_http_request = proxy and scheme != "https"
542 using_socks_proxy = False
543 if proxy:
544 proxy_scheme = urlparse(proxy).scheme.lower()
545 using_socks_proxy = proxy_scheme.startswith("socks")
547 url = request.path_url
548 if url.startswith("//"): # Don't confuse urllib3
549 url = f"/{url.lstrip('/')}"
551 if is_proxied_http_request and not using_socks_proxy:
552 url = urldefragauth(request.url)
554 return url
556 def add_headers(self, request, **kwargs):
557 """Add any headers needed by the connection. As of v2.0 this does
558 nothing by default, but is left for overriding by users that subclass
559 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
561 This should not be called from user code, and is only exposed for use
562 when subclassing the
563 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
565 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
566 :param kwargs: The keyword arguments from the call to send().
567 """
568 pass
570 def proxy_headers(self, proxy):
571 """Returns a dictionary of the headers to add to any request sent
572 through a proxy. This works with urllib3 magic to ensure that they are
573 correctly sent to the proxy, rather than in a tunnelled request if
574 CONNECT is being used.
576 This should not be called from user code, and is only exposed for use
577 when subclassing the
578 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
580 :param proxy: The url of the proxy being used for this request.
581 :rtype: dict
582 """
583 headers = {}
584 username, password = get_auth_from_url(proxy)
586 if username:
587 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
589 return headers
591 def send(
592 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
593 ):
594 """Sends PreparedRequest object. Returns Response object.
596 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
597 :param stream: (optional) Whether to stream the request content.
598 :param timeout: (optional) How long to wait for the server to send
599 data before giving up, as a float, or a :ref:`(connect timeout,
600 read timeout) <timeouts>` tuple.
601 :type timeout: float or tuple or urllib3 Timeout object
602 :param verify: (optional) Either a boolean, in which case it controls whether
603 we verify the server's TLS certificate, or a string, in which case it
604 must be a path to a CA bundle to use
605 :param cert: (optional) Any user-provided SSL certificate to be trusted.
606 :param proxies: (optional) The proxies dictionary to apply to the request.
607 :rtype: requests.Response
608 """
610 try:
611 conn = self.get_connection_with_tls_context(
612 request, verify, proxies=proxies, cert=cert
613 )
614 except LocationValueError as e:
615 raise InvalidURL(e, request=request)
617 self.cert_verify(conn, request.url, verify, cert)
618 url = self.request_url(request, proxies)
619 self.add_headers(
620 request,
621 stream=stream,
622 timeout=timeout,
623 verify=verify,
624 cert=cert,
625 proxies=proxies,
626 )
628 chunked = not (request.body is None or "Content-Length" in request.headers)
630 if isinstance(timeout, tuple):
631 try:
632 connect, read = timeout
633 timeout = TimeoutSauce(connect=connect, read=read)
634 except ValueError:
635 raise ValueError(
636 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
637 f"or a single float to set both timeouts to the same value."
638 )
639 elif isinstance(timeout, TimeoutSauce):
640 pass
641 else:
642 timeout = TimeoutSauce(connect=timeout, read=timeout)
644 try:
645 resp = conn.urlopen(
646 method=request.method,
647 url=url,
648 body=request.body,
649 headers=request.headers,
650 redirect=False,
651 assert_same_host=False,
652 preload_content=False,
653 decode_content=False,
654 retries=self.max_retries,
655 timeout=timeout,
656 chunked=chunked,
657 )
659 except (ProtocolError, OSError) as err:
660 raise ConnectionError(err, request=request)
662 except MaxRetryError as e:
663 if isinstance(e.reason, ConnectTimeoutError):
664 # TODO: Remove this in 3.0.0: see #2811
665 if not isinstance(e.reason, NewConnectionError):
666 raise ConnectTimeout(e, request=request)
668 if isinstance(e.reason, ResponseError):
669 raise RetryError(e, request=request)
671 if isinstance(e.reason, _ProxyError):
672 raise ProxyError(e, request=request)
674 if isinstance(e.reason, _SSLError):
675 # This branch is for urllib3 v1.22 and later.
676 raise SSLError(e, request=request)
678 raise ConnectionError(e, request=request)
680 except ClosedPoolError as e:
681 raise ConnectionError(e, request=request)
683 except _ProxyError as e:
684 raise ProxyError(e)
686 except (_SSLError, _HTTPError) as e:
687 if isinstance(e, _SSLError):
688 # This branch is for urllib3 versions earlier than v1.22
689 raise SSLError(e, request=request)
690 elif isinstance(e, ReadTimeoutError):
691 raise ReadTimeout(e, request=request)
692 elif isinstance(e, _InvalidHeader):
693 raise InvalidHeader(e, request=request)
694 else:
695 raise
697 return self.build_response(request, resp)