Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/requests/adapters.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2requests.adapters
3~~~~~~~~~~~~~~~~~
5This module contains the transport adapters that Requests uses to define
6and maintain connections.
7"""
9import os.path
10import socket # noqa: F401
11import typing
12import warnings
14from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
15from urllib3.exceptions import HTTPError as _HTTPError
16from urllib3.exceptions import InvalidHeader as _InvalidHeader
17from urllib3.exceptions import (
18 LocationValueError,
19 MaxRetryError,
20 NewConnectionError,
21 ProtocolError,
22)
23from urllib3.exceptions import ProxyError as _ProxyError
24from urllib3.exceptions import ReadTimeoutError, ResponseError
25from urllib3.exceptions import SSLError as _SSLError
26from urllib3.poolmanager import PoolManager, proxy_from_url
27from urllib3.util import Timeout as TimeoutSauce
28from urllib3.util import parse_url
29from urllib3.util.retry import Retry
31from .auth import _basic_auth_str
32from .compat import basestring, urlparse
33from .cookies import extract_cookies_to_jar
34from .exceptions import (
35 ConnectionError,
36 ConnectTimeout,
37 InvalidHeader,
38 InvalidProxyURL,
39 InvalidSchema,
40 InvalidURL,
41 ProxyError,
42 ReadTimeout,
43 RetryError,
44 SSLError,
45)
46from .models import Response
47from .structures import CaseInsensitiveDict
48from .utils import (
49 DEFAULT_CA_BUNDLE_PATH,
50 extract_zipped_paths,
51 get_auth_from_url,
52 get_encoding_from_headers,
53 prepend_scheme_if_needed,
54 select_proxy,
55 urldefragauth,
56)
58try:
59 from urllib3.contrib.socks import SOCKSProxyManager
60except ImportError:
62 def SOCKSProxyManager(*args, **kwargs):
63 raise InvalidSchema("Missing dependencies for SOCKS support.")
66if typing.TYPE_CHECKING:
67 from .models import PreparedRequest
70DEFAULT_POOLBLOCK = False
71DEFAULT_POOLSIZE = 10
72DEFAULT_RETRIES = 0
73DEFAULT_POOL_TIMEOUT = None
76def _urllib3_request_context(
77 request: "PreparedRequest",
78 verify: "bool | str | None",
79 client_cert: "typing.Tuple[str, str] | str | None",
80 poolmanager: "PoolManager",
81) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])":
82 host_params = {}
83 pool_kwargs = {}
84 parsed_request_url = urlparse(request.url)
85 scheme = parsed_request_url.scheme.lower()
86 port = parsed_request_url.port
88 cert_reqs = "CERT_REQUIRED"
89 if verify is False:
90 cert_reqs = "CERT_NONE"
91 elif isinstance(verify, str):
92 if not os.path.isdir(verify):
93 pool_kwargs["ca_certs"] = verify
94 else:
95 pool_kwargs["ca_cert_dir"] = verify
96 pool_kwargs["cert_reqs"] = cert_reqs
97 if client_cert is not None:
98 if isinstance(client_cert, tuple) and len(client_cert) == 2:
99 pool_kwargs["cert_file"] = client_cert[0]
100 pool_kwargs["key_file"] = client_cert[1]
101 else:
102 # According to our docs, we allow users to specify just the client
103 # cert path
104 pool_kwargs["cert_file"] = client_cert
105 host_params = {
106 "scheme": scheme,
107 "host": parsed_request_url.hostname,
108 "port": port,
109 }
110 return host_params, pool_kwargs
113class BaseAdapter:
114 """The Base Transport Adapter"""
116 def __init__(self):
117 super().__init__()
119 def send(
120 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
121 ):
122 """Sends PreparedRequest object. Returns Response object.
124 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
125 :param stream: (optional) Whether to stream the request content.
126 :param timeout: (optional) How long to wait for the server to send
127 data before giving up, as a float, or a :ref:`(connect timeout,
128 read timeout) <timeouts>` tuple.
129 :type timeout: float or tuple
130 :param verify: (optional) Either a boolean, in which case it controls whether we verify
131 the server's TLS certificate, or a string, in which case it must be a path
132 to a CA bundle to use
133 :param cert: (optional) Any user-provided SSL certificate to be trusted.
134 :param proxies: (optional) The proxies dictionary to apply to the request.
135 """
136 raise NotImplementedError
138 def close(self):
139 """Cleans up adapter specific items."""
140 raise NotImplementedError
143class HTTPAdapter(BaseAdapter):
144 """The built-in HTTP Adapter for urllib3.
146 Provides a general-case interface for Requests sessions to contact HTTP and
147 HTTPS urls by implementing the Transport Adapter interface. This class will
148 usually be created by the :class:`Session <Session>` class under the
149 covers.
151 :param pool_connections: The number of urllib3 connection pools to cache.
152 :param pool_maxsize: The maximum number of connections to save in the pool.
153 :param max_retries: The maximum number of retries each connection
154 should attempt. Note, this applies only to failed DNS lookups, socket
155 connections and connection timeouts, never to requests where data has
156 made it to the server. By default, Requests does not retry failed
157 connections. If you need granular control over the conditions under
158 which we retry a request, import urllib3's ``Retry`` class and pass
159 that instead.
160 :param pool_block: Whether the connection pool should block for connections.
162 Usage::
164 >>> import requests
165 >>> s = requests.Session()
166 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
167 >>> s.mount('http://', a)
168 """
170 __attrs__ = [
171 "max_retries",
172 "config",
173 "_pool_connections",
174 "_pool_maxsize",
175 "_pool_block",
176 ]
178 def __init__(
179 self,
180 pool_connections=DEFAULT_POOLSIZE,
181 pool_maxsize=DEFAULT_POOLSIZE,
182 max_retries=DEFAULT_RETRIES,
183 pool_block=DEFAULT_POOLBLOCK,
184 ):
185 if max_retries == DEFAULT_RETRIES:
186 self.max_retries = Retry(0, read=False)
187 else:
188 self.max_retries = Retry.from_int(max_retries)
189 self.config = {}
190 self.proxy_manager = {}
192 super().__init__()
194 self._pool_connections = pool_connections
195 self._pool_maxsize = pool_maxsize
196 self._pool_block = pool_block
198 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
200 def __getstate__(self):
201 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
203 def __setstate__(self, state):
204 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
205 # self.poolmanager uses a lambda function, which isn't pickleable.
206 self.proxy_manager = {}
207 self.config = {}
209 for attr, value in state.items():
210 setattr(self, attr, value)
212 self.init_poolmanager(
213 self._pool_connections, self._pool_maxsize, block=self._pool_block
214 )
216 def init_poolmanager(
217 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
218 ):
219 """Initializes a urllib3 PoolManager.
221 This method should not be called from user code, and is only
222 exposed for use when subclassing the
223 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
225 :param connections: The number of urllib3 connection pools to cache.
226 :param maxsize: The maximum number of connections to save in the pool.
227 :param block: Block when no free connections are available.
228 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
229 """
230 # save these values for pickling
231 self._pool_connections = connections
232 self._pool_maxsize = maxsize
233 self._pool_block = block
235 self.poolmanager = PoolManager(
236 num_pools=connections,
237 maxsize=maxsize,
238 block=block,
239 **pool_kwargs,
240 )
242 def proxy_manager_for(self, proxy, **proxy_kwargs):
243 """Return urllib3 ProxyManager for the given proxy.
245 This method should not be called from user code, and is only
246 exposed for use when subclassing the
247 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
249 :param proxy: The proxy to return a urllib3 ProxyManager for.
250 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
251 :returns: ProxyManager
252 :rtype: urllib3.ProxyManager
253 """
254 if proxy in self.proxy_manager:
255 manager = self.proxy_manager[proxy]
256 elif proxy.lower().startswith("socks"):
257 username, password = get_auth_from_url(proxy)
258 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
259 proxy,
260 username=username,
261 password=password,
262 num_pools=self._pool_connections,
263 maxsize=self._pool_maxsize,
264 block=self._pool_block,
265 **proxy_kwargs,
266 )
267 else:
268 proxy_headers = self.proxy_headers(proxy)
269 manager = self.proxy_manager[proxy] = proxy_from_url(
270 proxy,
271 proxy_headers=proxy_headers,
272 num_pools=self._pool_connections,
273 maxsize=self._pool_maxsize,
274 block=self._pool_block,
275 **proxy_kwargs,
276 )
278 return manager
280 def cert_verify(self, conn, url, verify, cert):
281 """Verify a SSL certificate. This method should not be called from user
282 code, and is only exposed for use when subclassing the
283 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
285 :param conn: The urllib3 connection object associated with the cert.
286 :param url: The requested URL.
287 :param verify: Either a boolean, in which case it controls whether we verify
288 the server's TLS certificate, or a string, in which case it must be a path
289 to a CA bundle to use
290 :param cert: The SSL certificate to verify.
291 """
292 if url.lower().startswith("https") and verify:
293 cert_loc = None
295 # Allow self-specified cert location.
296 if verify is not True:
297 cert_loc = verify
299 if not cert_loc:
300 cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
302 if not cert_loc or not os.path.exists(cert_loc):
303 raise OSError(
304 f"Could not find a suitable TLS CA certificate bundle, "
305 f"invalid path: {cert_loc}"
306 )
308 conn.cert_reqs = "CERT_REQUIRED"
310 if not os.path.isdir(cert_loc):
311 conn.ca_certs = cert_loc
312 else:
313 conn.ca_cert_dir = cert_loc
314 else:
315 conn.cert_reqs = "CERT_NONE"
316 conn.ca_certs = None
317 conn.ca_cert_dir = None
319 if cert:
320 if not isinstance(cert, basestring):
321 conn.cert_file = cert[0]
322 conn.key_file = cert[1]
323 else:
324 conn.cert_file = cert
325 conn.key_file = None
326 if conn.cert_file and not os.path.exists(conn.cert_file):
327 raise OSError(
328 f"Could not find the TLS certificate file, "
329 f"invalid path: {conn.cert_file}"
330 )
331 if conn.key_file and not os.path.exists(conn.key_file):
332 raise OSError(
333 f"Could not find the TLS key file, invalid path: {conn.key_file}"
334 )
336 def build_response(self, req, resp):
337 """Builds a :class:`Response <requests.Response>` object from a urllib3
338 response. This should not be called from user code, and is only exposed
339 for use when subclassing the
340 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
342 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
343 :param resp: The urllib3 response object.
344 :rtype: requests.Response
345 """
346 response = Response()
348 # Fallback to None if there's no status_code, for whatever reason.
349 response.status_code = getattr(resp, "status", None)
351 # Make headers case-insensitive.
352 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
354 # Set encoding.
355 response.encoding = get_encoding_from_headers(response.headers)
356 response.raw = resp
357 response.reason = response.raw.reason
359 if isinstance(req.url, bytes):
360 response.url = req.url.decode("utf-8")
361 else:
362 response.url = req.url
364 # Add new cookies from the server.
365 extract_cookies_to_jar(response.cookies, req, resp)
367 # Give the Response some context.
368 response.request = req
369 response.connection = self
371 return response
373 def build_connection_pool_key_attributes(self, request, verify, cert=None):
374 """Build the PoolKey attributes used by urllib3 to return a connection.
376 This looks at the PreparedRequest, the user-specified verify value,
377 and the value of the cert parameter to determine what PoolKey values
378 to use to select a connection from a given urllib3 Connection Pool.
380 The SSL related pool key arguments are not consistently set. As of
381 this writing, use the following to determine what keys may be in that
382 dictionary:
384 * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
385 default Requests SSL Context
386 * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
387 ``"cert_reqs"`` will be set
388 * If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
389 ``"ca_certs"`` will be set if the string is not a directory recognized
390 by :py:func:`os.path.isdir`, otherwise ``"ca_cert_dir"`` will be
391 set.
392 * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
393 ``"cert"`` is a tuple with a second item, ``"key_file"`` will also
394 be present
396 To override these settings, one may subclass this class, call this
397 method and use the above logic to change parameters as desired. For
398 example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
399 must both set ``"ssl_context"`` and based on what else they require,
400 alter the other keys to ensure the desired behaviour.
402 :param request:
403 The PreparedReqest being sent over the connection.
404 :type request:
405 :class:`~requests.models.PreparedRequest`
406 :param verify:
407 Either a boolean, in which case it controls whether
408 we verify the server's TLS certificate, or a string, in which case it
409 must be a path to a CA bundle to use.
410 :param cert:
411 (optional) Any user-provided SSL certificate for client
412 authentication (a.k.a., mTLS). This may be a string (i.e., just
413 the path to a file which holds both certificate and key) or a
414 tuple of length 2 with the certificate file path and key file
415 path.
416 :returns:
417 A tuple of two dictionaries. The first is the "host parameters"
418 portion of the Pool Key including scheme, hostname, and port. The
419 second is a dictionary of SSLContext related parameters.
420 """
421 return _urllib3_request_context(request, verify, cert, self.poolmanager)
423 def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
424 """Returns a urllib3 connection for the given request and TLS settings.
425 This should not be called from user code, and is only exposed for use
426 when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
428 :param request:
429 The :class:`PreparedRequest <PreparedRequest>` object to be sent
430 over the connection.
431 :param verify:
432 Either a boolean, in which case it controls whether we verify the
433 server's TLS certificate, or a string, in which case it must be a
434 path to a CA bundle to use.
435 :param proxies:
436 (optional) The proxies dictionary to apply to the request.
437 :param cert:
438 (optional) Any user-provided SSL certificate to be used for client
439 authentication (a.k.a., mTLS).
440 :rtype:
441 urllib3.ConnectionPool
442 """
443 proxy = select_proxy(request.url, proxies)
444 try:
445 host_params, pool_kwargs = self.build_connection_pool_key_attributes(
446 request,
447 verify,
448 cert,
449 )
450 except ValueError as e:
451 raise InvalidURL(e, request=request)
452 if proxy:
453 proxy = prepend_scheme_if_needed(proxy, "http")
454 proxy_url = parse_url(proxy)
455 if not proxy_url.host:
456 raise InvalidProxyURL(
457 "Please check proxy URL. It is malformed "
458 "and could be missing the host."
459 )
460 proxy_manager = self.proxy_manager_for(proxy)
461 conn = proxy_manager.connection_from_host(
462 **host_params, pool_kwargs=pool_kwargs
463 )
464 else:
465 # Only scheme should be lower case
466 conn = self.poolmanager.connection_from_host(
467 **host_params, pool_kwargs=pool_kwargs
468 )
470 return conn
472 def get_connection(self, url, proxies=None):
473 """DEPRECATED: Users should move to `get_connection_with_tls_context`
474 for all subclasses of HTTPAdapter using Requests>=2.32.2.
476 Returns a urllib3 connection for the given URL. This should not be
477 called from user code, and is only exposed for use when subclassing the
478 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
480 :param url: The URL to connect to.
481 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
482 :rtype: urllib3.ConnectionPool
483 """
484 warnings.warn(
485 (
486 "`get_connection` has been deprecated in favor of "
487 "`get_connection_with_tls_context`. Custom HTTPAdapter subclasses "
488 "will need to migrate for Requests>=2.32.2. Please see "
489 "https://github.com/psf/requests/pull/6710 for more details."
490 ),
491 DeprecationWarning,
492 )
493 proxy = select_proxy(url, proxies)
495 if proxy:
496 proxy = prepend_scheme_if_needed(proxy, "http")
497 proxy_url = parse_url(proxy)
498 if not proxy_url.host:
499 raise InvalidProxyURL(
500 "Please check proxy URL. It is malformed "
501 "and could be missing the host."
502 )
503 proxy_manager = self.proxy_manager_for(proxy)
504 conn = proxy_manager.connection_from_url(url)
505 else:
506 # Only scheme should be lower case
507 parsed = urlparse(url)
508 url = parsed.geturl()
509 conn = self.poolmanager.connection_from_url(url)
511 return conn
513 def close(self):
514 """Disposes of any internal state.
516 Currently, this closes the PoolManager and any active ProxyManager,
517 which closes any pooled connections.
518 """
519 self.poolmanager.clear()
520 for proxy in self.proxy_manager.values():
521 proxy.clear()
523 def request_url(self, request, proxies):
524 """Obtain the url to use when making the final request.
526 If the message is being sent through a HTTP proxy, the full URL has to
527 be used. Otherwise, we should only use the path portion of the URL.
529 This should not be called from user code, and is only exposed for use
530 when subclassing the
531 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
533 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
534 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
535 :rtype: str
536 """
537 proxy = select_proxy(request.url, proxies)
538 scheme = urlparse(request.url).scheme
540 is_proxied_http_request = proxy and scheme != "https"
541 using_socks_proxy = False
542 if proxy:
543 proxy_scheme = urlparse(proxy).scheme.lower()
544 using_socks_proxy = proxy_scheme.startswith("socks")
546 url = request.path_url
547 if url.startswith("//"): # Don't confuse urllib3
548 url = f"/{url.lstrip('/')}"
550 if is_proxied_http_request and not using_socks_proxy:
551 url = urldefragauth(request.url)
553 return url
555 def add_headers(self, request, **kwargs):
556 """Add any headers needed by the connection. As of v2.0 this does
557 nothing by default, but is left for overriding by users that subclass
558 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
560 This should not be called from user code, and is only exposed for use
561 when subclassing the
562 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
564 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
565 :param kwargs: The keyword arguments from the call to send().
566 """
567 pass
569 def proxy_headers(self, proxy):
570 """Returns a dictionary of the headers to add to any request sent
571 through a proxy. This works with urllib3 magic to ensure that they are
572 correctly sent to the proxy, rather than in a tunnelled request if
573 CONNECT is being used.
575 This should not be called from user code, and is only exposed for use
576 when subclassing the
577 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
579 :param proxy: The url of the proxy being used for this request.
580 :rtype: dict
581 """
582 headers = {}
583 username, password = get_auth_from_url(proxy)
585 if username:
586 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
588 return headers
590 def send(
591 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
592 ):
593 """Sends PreparedRequest object. Returns Response object.
595 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
596 :param stream: (optional) Whether to stream the request content.
597 :param timeout: (optional) How long to wait for the server to send
598 data before giving up, as a float, or a :ref:`(connect timeout,
599 read timeout) <timeouts>` tuple.
600 :type timeout: float or tuple or urllib3 Timeout object
601 :param verify: (optional) Either a boolean, in which case it controls whether
602 we verify the server's TLS certificate, or a string, in which case it
603 must be a path to a CA bundle to use
604 :param cert: (optional) Any user-provided SSL certificate to be trusted.
605 :param proxies: (optional) The proxies dictionary to apply to the request.
606 :rtype: requests.Response
607 """
609 try:
610 conn = self.get_connection_with_tls_context(
611 request, verify, proxies=proxies, cert=cert
612 )
613 except LocationValueError as e:
614 raise InvalidURL(e, request=request)
616 self.cert_verify(conn, request.url, verify, cert)
617 url = self.request_url(request, proxies)
618 self.add_headers(
619 request,
620 stream=stream,
621 timeout=timeout,
622 verify=verify,
623 cert=cert,
624 proxies=proxies,
625 )
627 chunked = not (request.body is None or "Content-Length" in request.headers)
629 if isinstance(timeout, tuple):
630 try:
631 connect, read = timeout
632 timeout = TimeoutSauce(connect=connect, read=read)
633 except ValueError:
634 raise ValueError(
635 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
636 f"or a single float to set both timeouts to the same value."
637 )
638 elif isinstance(timeout, TimeoutSauce):
639 pass
640 else:
641 timeout = TimeoutSauce(connect=timeout, read=timeout)
643 try:
644 resp = conn.urlopen(
645 method=request.method,
646 url=url,
647 body=request.body,
648 headers=request.headers,
649 redirect=False,
650 assert_same_host=False,
651 preload_content=False,
652 decode_content=False,
653 retries=self.max_retries,
654 timeout=timeout,
655 chunked=chunked,
656 )
658 except (ProtocolError, OSError) as err:
659 raise ConnectionError(err, request=request)
661 except MaxRetryError as e:
662 if isinstance(e.reason, ConnectTimeoutError):
663 # TODO: Remove this in 3.0.0: see #2811
664 if not isinstance(e.reason, NewConnectionError):
665 raise ConnectTimeout(e, request=request)
667 if isinstance(e.reason, ResponseError):
668 raise RetryError(e, request=request)
670 if isinstance(e.reason, _ProxyError):
671 raise ProxyError(e, request=request)
673 if isinstance(e.reason, _SSLError):
674 # This branch is for urllib3 v1.22 and later.
675 raise SSLError(e, request=request)
677 raise ConnectionError(e, request=request)
679 except ClosedPoolError as e:
680 raise ConnectionError(e, request=request)
682 except _ProxyError as e:
683 raise ProxyError(e)
685 except (_SSLError, _HTTPError) as e:
686 if isinstance(e, _SSLError):
687 # This branch is for urllib3 versions earlier than v1.22
688 raise SSLError(e, request=request)
689 elif isinstance(e, ReadTimeoutError):
690 raise ReadTimeout(e, request=request)
691 elif isinstance(e, _InvalidHeader):
692 raise InvalidHeader(e, request=request)
693 else:
694 raise
696 return self.build_response(request, resp)