Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/requests/adapters.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2requests.adapters
3~~~~~~~~~~~~~~~~~
5This module contains the transport adapters that Requests uses to define
6and maintain connections.
7"""
9import os.path
10import socket # noqa: F401
11import typing
12import warnings
14from urllib3.exceptions import (
15 ClosedPoolError,
16 ConnectTimeoutError,
17 LocationValueError,
18 MaxRetryError,
19 NewConnectionError,
20 ProtocolError,
21 ReadTimeoutError,
22 ResponseError,
23)
24from urllib3.exceptions import HTTPError as _HTTPError
25from urllib3.exceptions import InvalidHeader as _InvalidHeader
26from urllib3.exceptions import ProxyError as _ProxyError
27from urllib3.exceptions import SSLError as _SSLError
28from urllib3.poolmanager import PoolManager, proxy_from_url
29from urllib3.util import Timeout as TimeoutSauce
30from urllib3.util import parse_url
31from urllib3.util.retry import Retry
33from .auth import _basic_auth_str
34from .compat import basestring, urlparse
35from .cookies import extract_cookies_to_jar
36from .exceptions import (
37 ConnectionError,
38 ConnectTimeout,
39 InvalidHeader,
40 InvalidProxyURL,
41 InvalidSchema,
42 InvalidURL,
43 ProxyError,
44 ReadTimeout,
45 RetryError,
46 SSLError,
47)
48from .models import Response
49from .structures import CaseInsensitiveDict
50from .utils import (
51 DEFAULT_CA_BUNDLE_PATH,
52 extract_zipped_paths,
53 get_auth_from_url,
54 get_encoding_from_headers,
55 prepend_scheme_if_needed,
56 select_proxy,
57 urldefragauth,
58)
60try:
61 from urllib3.contrib.socks import SOCKSProxyManager
62except ImportError:
64 def SOCKSProxyManager(*args, **kwargs):
65 raise InvalidSchema("Missing dependencies for SOCKS support.")
68if typing.TYPE_CHECKING:
69 from .models import PreparedRequest
72DEFAULT_POOLBLOCK = False
73DEFAULT_POOLSIZE = 10
74DEFAULT_RETRIES = 0
75DEFAULT_POOL_TIMEOUT = None
78def _urllib3_request_context(
79 request: "PreparedRequest",
80 verify: "bool | str | None",
81 client_cert: "tuple[str, str] | str | None",
82 poolmanager: "PoolManager",
83) -> "(dict[str, typing.Any], dict[str, typing.Any])":
84 host_params = {}
85 pool_kwargs = {}
86 parsed_request_url = urlparse(request.url)
87 scheme = parsed_request_url.scheme.lower()
88 port = parsed_request_url.port
90 cert_reqs = "CERT_REQUIRED"
91 if verify is False:
92 cert_reqs = "CERT_NONE"
93 elif isinstance(verify, str):
94 if not os.path.isdir(verify):
95 pool_kwargs["ca_certs"] = verify
96 else:
97 pool_kwargs["ca_cert_dir"] = verify
98 pool_kwargs["cert_reqs"] = cert_reqs
99 if client_cert is not None:
100 if isinstance(client_cert, tuple) and len(client_cert) == 2:
101 pool_kwargs["cert_file"] = client_cert[0]
102 pool_kwargs["key_file"] = client_cert[1]
103 else:
104 # According to our docs, we allow users to specify just the client
105 # cert path
106 pool_kwargs["cert_file"] = client_cert
107 host_params = {
108 "scheme": scheme,
109 "host": parsed_request_url.hostname,
110 "port": port,
111 }
112 return host_params, pool_kwargs
115class BaseAdapter:
116 """The Base Transport Adapter"""
118 def __init__(self):
119 super().__init__()
121 def send(
122 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
123 ):
124 """Sends PreparedRequest object. Returns Response object.
126 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
127 :param stream: (optional) Whether to stream the request content.
128 :param timeout: (optional) How long to wait for the server to send
129 data before giving up, as a float, or a :ref:`(connect timeout,
130 read timeout) <timeouts>` tuple.
131 :type timeout: float or tuple
132 :param verify: (optional) Either a boolean, in which case it controls whether we verify
133 the server's TLS certificate, or a string, in which case it must be a path
134 to a CA bundle to use
135 :param cert: (optional) Any user-provided SSL certificate to be trusted.
136 :param proxies: (optional) The proxies dictionary to apply to the request.
137 """
138 raise NotImplementedError
140 def close(self):
141 """Cleans up adapter specific items."""
142 raise NotImplementedError
145class HTTPAdapter(BaseAdapter):
146 """The built-in HTTP Adapter for urllib3.
148 Provides a general-case interface for Requests sessions to contact HTTP and
149 HTTPS urls by implementing the Transport Adapter interface. This class will
150 usually be created by the :class:`Session <Session>` class under the
151 covers.
153 :param pool_connections: The number of urllib3 connection pools to cache.
154 :param pool_maxsize: The maximum number of connections to save in the pool.
155 :param max_retries: The maximum number of retries each connection
156 should attempt. Note, this applies only to failed DNS lookups, socket
157 connections and connection timeouts, never to requests where data has
158 made it to the server. By default, Requests does not retry failed
159 connections. If you need granular control over the conditions under
160 which we retry a request, import urllib3's ``Retry`` class and pass
161 that instead.
162 :param pool_block: Whether the connection pool should block for connections.
164 Usage::
166 >>> import requests
167 >>> s = requests.Session()
168 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
169 >>> s.mount('http://', a)
170 """
172 __attrs__ = [
173 "max_retries",
174 "config",
175 "_pool_connections",
176 "_pool_maxsize",
177 "_pool_block",
178 ]
180 def __init__(
181 self,
182 pool_connections=DEFAULT_POOLSIZE,
183 pool_maxsize=DEFAULT_POOLSIZE,
184 max_retries=DEFAULT_RETRIES,
185 pool_block=DEFAULT_POOLBLOCK,
186 ):
187 if max_retries == DEFAULT_RETRIES:
188 self.max_retries = Retry(0, read=False)
189 else:
190 self.max_retries = Retry.from_int(max_retries)
191 self.config = {}
192 self.proxy_manager = {}
194 super().__init__()
196 self._pool_connections = pool_connections
197 self._pool_maxsize = pool_maxsize
198 self._pool_block = pool_block
200 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
202 def __getstate__(self):
203 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
205 def __setstate__(self, state):
206 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
207 # self.poolmanager uses a lambda function, which isn't pickleable.
208 self.proxy_manager = {}
209 self.config = {}
211 for attr, value in state.items():
212 setattr(self, attr, value)
214 self.init_poolmanager(
215 self._pool_connections, self._pool_maxsize, block=self._pool_block
216 )
218 def init_poolmanager(
219 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
220 ):
221 """Initializes a urllib3 PoolManager.
223 This method should not be called from user code, and is only
224 exposed for use when subclassing the
225 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
227 :param connections: The number of urllib3 connection pools to cache.
228 :param maxsize: The maximum number of connections to save in the pool.
229 :param block: Block when no free connections are available.
230 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
231 """
232 # save these values for pickling
233 self._pool_connections = connections
234 self._pool_maxsize = maxsize
235 self._pool_block = block
237 self.poolmanager = PoolManager(
238 num_pools=connections,
239 maxsize=maxsize,
240 block=block,
241 **pool_kwargs,
242 )
244 def proxy_manager_for(self, proxy, **proxy_kwargs):
245 """Return urllib3 ProxyManager for the given proxy.
247 This method should not be called from user code, and is only
248 exposed for use when subclassing the
249 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
251 :param proxy: The proxy to return a urllib3 ProxyManager for.
252 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
253 :returns: ProxyManager
254 :rtype: urllib3.ProxyManager
255 """
256 if proxy in self.proxy_manager:
257 manager = self.proxy_manager[proxy]
258 elif proxy.lower().startswith("socks"):
259 username, password = get_auth_from_url(proxy)
260 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
261 proxy,
262 username=username,
263 password=password,
264 num_pools=self._pool_connections,
265 maxsize=self._pool_maxsize,
266 block=self._pool_block,
267 **proxy_kwargs,
268 )
269 else:
270 proxy_headers = self.proxy_headers(proxy)
271 manager = self.proxy_manager[proxy] = proxy_from_url(
272 proxy,
273 proxy_headers=proxy_headers,
274 num_pools=self._pool_connections,
275 maxsize=self._pool_maxsize,
276 block=self._pool_block,
277 **proxy_kwargs,
278 )
280 return manager
282 def cert_verify(self, conn, url, verify, cert):
283 """Verify a SSL certificate. This method should not be called from user
284 code, and is only exposed for use when subclassing the
285 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
287 :param conn: The urllib3 connection object associated with the cert.
288 :param url: The requested URL.
289 :param verify: Either a boolean, in which case it controls whether we verify
290 the server's TLS certificate, or a string, in which case it must be a path
291 to a CA bundle to use
292 :param cert: The SSL certificate to verify.
293 """
294 if url.lower().startswith("https") and verify:
295 cert_loc = None
297 # Allow self-specified cert location.
298 if verify is not True:
299 cert_loc = verify
301 if not cert_loc:
302 cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
304 if not cert_loc or not os.path.exists(cert_loc):
305 raise OSError(
306 f"Could not find a suitable TLS CA certificate bundle, "
307 f"invalid path: {cert_loc}"
308 )
310 conn.cert_reqs = "CERT_REQUIRED"
312 if not os.path.isdir(cert_loc):
313 conn.ca_certs = cert_loc
314 else:
315 conn.ca_cert_dir = cert_loc
316 else:
317 conn.cert_reqs = "CERT_NONE"
318 conn.ca_certs = None
319 conn.ca_cert_dir = None
321 if cert:
322 if not isinstance(cert, basestring):
323 conn.cert_file = cert[0]
324 conn.key_file = cert[1]
325 else:
326 conn.cert_file = cert
327 conn.key_file = None
328 if conn.cert_file and not os.path.exists(conn.cert_file):
329 raise OSError(
330 f"Could not find the TLS certificate file, "
331 f"invalid path: {conn.cert_file}"
332 )
333 if conn.key_file and not os.path.exists(conn.key_file):
334 raise OSError(
335 f"Could not find the TLS key file, invalid path: {conn.key_file}"
336 )
338 def build_response(self, req, resp):
339 """Builds a :class:`Response <requests.Response>` object from a urllib3
340 response. This should not be called from user code, and is only exposed
341 for use when subclassing the
342 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
344 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
345 :param resp: The urllib3 response object.
346 :rtype: requests.Response
347 """
348 response = Response()
350 # Fallback to None if there's no status_code, for whatever reason.
351 response.status_code = getattr(resp, "status", None)
353 # Make headers case-insensitive.
354 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
356 # Set encoding.
357 response.encoding = get_encoding_from_headers(response.headers)
358 response.raw = resp
359 response.reason = response.raw.reason
361 if isinstance(req.url, bytes):
362 response.url = req.url.decode("utf-8")
363 else:
364 response.url = req.url
366 # Add new cookies from the server.
367 extract_cookies_to_jar(response.cookies, req, resp)
369 # Give the Response some context.
370 response.request = req
371 response.connection = self
373 return response
375 def build_connection_pool_key_attributes(self, request, verify, cert=None):
376 """Build the PoolKey attributes used by urllib3 to return a connection.
378 This looks at the PreparedRequest, the user-specified verify value,
379 and the value of the cert parameter to determine what PoolKey values
380 to use to select a connection from a given urllib3 Connection Pool.
382 The SSL related pool key arguments are not consistently set. As of
383 this writing, use the following to determine what keys may be in that
384 dictionary:
386 * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
387 default Requests SSL Context
388 * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
389 ``"cert_reqs"`` will be set
390 * If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
391 ``"ca_certs"`` will be set if the string is not a directory recognized
392 by :py:func:`os.path.isdir`, otherwise ``"ca_cert_dir"`` will be
393 set.
394 * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
395 ``"cert"`` is a tuple with a second item, ``"key_file"`` will also
396 be present
398 To override these settings, one may subclass this class, call this
399 method and use the above logic to change parameters as desired. For
400 example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
401 must both set ``"ssl_context"`` and based on what else they require,
402 alter the other keys to ensure the desired behaviour.
404 :param request:
405 The PreparedReqest being sent over the connection.
406 :type request:
407 :class:`~requests.models.PreparedRequest`
408 :param verify:
409 Either a boolean, in which case it controls whether
410 we verify the server's TLS certificate, or a string, in which case it
411 must be a path to a CA bundle to use.
412 :param cert:
413 (optional) Any user-provided SSL certificate for client
414 authentication (a.k.a., mTLS). This may be a string (i.e., just
415 the path to a file which holds both certificate and key) or a
416 tuple of length 2 with the certificate file path and key file
417 path.
418 :returns:
419 A tuple of two dictionaries. The first is the "host parameters"
420 portion of the Pool Key including scheme, hostname, and port. The
421 second is a dictionary of SSLContext related parameters.
422 """
423 return _urllib3_request_context(request, verify, cert, self.poolmanager)
425 def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
426 """Returns a urllib3 connection for the given request and TLS settings.
427 This should not be called from user code, and is only exposed for use
428 when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
430 :param request:
431 The :class:`PreparedRequest <PreparedRequest>` object to be sent
432 over the connection.
433 :param verify:
434 Either a boolean, in which case it controls whether we verify the
435 server's TLS certificate, or a string, in which case it must be a
436 path to a CA bundle to use.
437 :param proxies:
438 (optional) The proxies dictionary to apply to the request.
439 :param cert:
440 (optional) Any user-provided SSL certificate to be used for client
441 authentication (a.k.a., mTLS).
442 :rtype:
443 urllib3.ConnectionPool
444 """
445 proxy = select_proxy(request.url, proxies)
446 try:
447 host_params, pool_kwargs = self.build_connection_pool_key_attributes(
448 request,
449 verify,
450 cert,
451 )
452 except ValueError as e:
453 raise InvalidURL(e, request=request)
454 if proxy:
455 proxy = prepend_scheme_if_needed(proxy, "http")
456 proxy_url = parse_url(proxy)
457 if not proxy_url.host:
458 raise InvalidProxyURL(
459 "Please check proxy URL. It is malformed "
460 "and could be missing the host."
461 )
462 proxy_manager = self.proxy_manager_for(proxy)
463 conn = proxy_manager.connection_from_host(
464 **host_params, pool_kwargs=pool_kwargs
465 )
466 else:
467 # Only scheme should be lower case
468 conn = self.poolmanager.connection_from_host(
469 **host_params, pool_kwargs=pool_kwargs
470 )
472 return conn
474 def get_connection(self, url, proxies=None):
475 """DEPRECATED: Users should move to `get_connection_with_tls_context`
476 for all subclasses of HTTPAdapter using Requests>=2.32.2.
478 Returns a urllib3 connection for the given URL. This should not be
479 called from user code, and is only exposed for use when subclassing the
480 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
482 :param url: The URL to connect to.
483 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
484 :rtype: urllib3.ConnectionPool
485 """
486 warnings.warn(
487 (
488 "`get_connection` has been deprecated in favor of "
489 "`get_connection_with_tls_context`. Custom HTTPAdapter subclasses "
490 "will need to migrate for Requests>=2.32.2. Please see "
491 "https://github.com/psf/requests/pull/6710 for more details."
492 ),
493 DeprecationWarning,
494 )
495 proxy = select_proxy(url, proxies)
497 if proxy:
498 proxy = prepend_scheme_if_needed(proxy, "http")
499 proxy_url = parse_url(proxy)
500 if not proxy_url.host:
501 raise InvalidProxyURL(
502 "Please check proxy URL. It is malformed "
503 "and could be missing the host."
504 )
505 proxy_manager = self.proxy_manager_for(proxy)
506 conn = proxy_manager.connection_from_url(url)
507 else:
508 # Only scheme should be lower case
509 parsed = urlparse(url)
510 url = parsed.geturl()
511 conn = self.poolmanager.connection_from_url(url)
513 return conn
515 def close(self):
516 """Disposes of any internal state.
518 Currently, this closes the PoolManager and any active ProxyManager,
519 which closes any pooled connections.
520 """
521 self.poolmanager.clear()
522 for proxy in self.proxy_manager.values():
523 proxy.clear()
525 def request_url(self, request, proxies):
526 """Obtain the url to use when making the final request.
528 If the message is being sent through a HTTP proxy, the full URL has to
529 be used. Otherwise, we should only use the path portion of the URL.
531 This should not be called from user code, and is only exposed for use
532 when subclassing the
533 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
535 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
536 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
537 :rtype: str
538 """
539 proxy = select_proxy(request.url, proxies)
540 scheme = urlparse(request.url).scheme
542 is_proxied_http_request = proxy and scheme != "https"
543 using_socks_proxy = False
544 if proxy:
545 proxy_scheme = urlparse(proxy).scheme.lower()
546 using_socks_proxy = proxy_scheme.startswith("socks")
548 url = request.path_url
549 if url.startswith("//"): # Don't confuse urllib3
550 url = f"/{url.lstrip('/')}"
552 if is_proxied_http_request and not using_socks_proxy:
553 url = urldefragauth(request.url)
555 return url
557 def add_headers(self, request, **kwargs):
558 """Add any headers needed by the connection. As of v2.0 this does
559 nothing by default, but is left for overriding by users that subclass
560 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
562 This should not be called from user code, and is only exposed for use
563 when subclassing the
564 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
566 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
567 :param kwargs: The keyword arguments from the call to send().
568 """
569 pass
571 def proxy_headers(self, proxy):
572 """Returns a dictionary of the headers to add to any request sent
573 through a proxy. This works with urllib3 magic to ensure that they are
574 correctly sent to the proxy, rather than in a tunnelled request if
575 CONNECT is being used.
577 This should not be called from user code, and is only exposed for use
578 when subclassing the
579 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
581 :param proxy: The url of the proxy being used for this request.
582 :rtype: dict
583 """
584 headers = {}
585 username, password = get_auth_from_url(proxy)
587 if username:
588 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
590 return headers
592 def send(
593 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
594 ):
595 """Sends PreparedRequest object. Returns Response object.
597 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
598 :param stream: (optional) Whether to stream the request content.
599 :param timeout: (optional) How long to wait for the server to send
600 data before giving up, as a float, or a :ref:`(connect timeout,
601 read timeout) <timeouts>` tuple.
602 :type timeout: float or tuple or urllib3 Timeout object
603 :param verify: (optional) Either a boolean, in which case it controls whether
604 we verify the server's TLS certificate, or a string, in which case it
605 must be a path to a CA bundle to use
606 :param cert: (optional) Any user-provided SSL certificate to be trusted.
607 :param proxies: (optional) The proxies dictionary to apply to the request.
608 :rtype: requests.Response
609 """
611 try:
612 conn = self.get_connection_with_tls_context(
613 request, verify, proxies=proxies, cert=cert
614 )
615 except LocationValueError as e:
616 raise InvalidURL(e, request=request)
618 self.cert_verify(conn, request.url, verify, cert)
619 url = self.request_url(request, proxies)
620 self.add_headers(
621 request,
622 stream=stream,
623 timeout=timeout,
624 verify=verify,
625 cert=cert,
626 proxies=proxies,
627 )
629 chunked = not (request.body is None or "Content-Length" in request.headers)
631 if isinstance(timeout, tuple):
632 try:
633 connect, read = timeout
634 timeout = TimeoutSauce(connect=connect, read=read)
635 except ValueError:
636 raise ValueError(
637 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
638 f"or a single float to set both timeouts to the same value."
639 )
640 elif isinstance(timeout, TimeoutSauce):
641 pass
642 else:
643 timeout = TimeoutSauce(connect=timeout, read=timeout)
645 try:
646 resp = conn.urlopen(
647 method=request.method,
648 url=url,
649 body=request.body,
650 headers=request.headers,
651 redirect=False,
652 assert_same_host=False,
653 preload_content=False,
654 decode_content=False,
655 retries=self.max_retries,
656 timeout=timeout,
657 chunked=chunked,
658 )
660 except (ProtocolError, OSError) as err:
661 raise ConnectionError(err, request=request)
663 except MaxRetryError as e:
664 if isinstance(e.reason, ConnectTimeoutError):
665 # TODO: Remove this in 3.0.0: see #2811
666 if not isinstance(e.reason, NewConnectionError):
667 raise ConnectTimeout(e, request=request)
669 if isinstance(e.reason, ResponseError):
670 raise RetryError(e, request=request)
672 if isinstance(e.reason, _ProxyError):
673 raise ProxyError(e, request=request)
675 if isinstance(e.reason, _SSLError):
676 # This branch is for urllib3 v1.22 and later.
677 raise SSLError(e, request=request)
679 raise ConnectionError(e, request=request)
681 except ClosedPoolError as e:
682 raise ConnectionError(e, request=request)
684 except _ProxyError as e:
685 raise ProxyError(e)
687 except (_SSLError, _HTTPError) as e:
688 if isinstance(e, _SSLError):
689 # This branch is for urllib3 versions earlier than v1.22
690 raise SSLError(e, request=request)
691 elif isinstance(e, ReadTimeoutError):
692 raise ReadTimeout(e, request=request)
693 elif isinstance(e, _InvalidHeader):
694 raise InvalidHeader(e, request=request)
695 else:
696 raise
698 return self.build_response(request, resp)