Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/requests/adapters.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2requests.adapters
3~~~~~~~~~~~~~~~~~
5This module contains the transport adapters that Requests uses to define
6and maintain connections.
7"""
9import os.path
10import socket # noqa: F401
11import typing
13from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
14from urllib3.exceptions import HTTPError as _HTTPError
15from urllib3.exceptions import InvalidHeader as _InvalidHeader
16from urllib3.exceptions import (
17 LocationValueError,
18 MaxRetryError,
19 NewConnectionError,
20 ProtocolError,
21)
22from urllib3.exceptions import ProxyError as _ProxyError
23from urllib3.exceptions import ReadTimeoutError, ResponseError
24from urllib3.exceptions import SSLError as _SSLError
25from urllib3.poolmanager import PoolManager, proxy_from_url
26from urllib3.util import Timeout as TimeoutSauce
27from urllib3.util import parse_url
28from urllib3.util.retry import Retry
29from urllib3.util.ssl_ import create_urllib3_context
31from .auth import _basic_auth_str
32from .compat import basestring, urlparse
33from .cookies import extract_cookies_to_jar
34from .exceptions import (
35 ConnectionError,
36 ConnectTimeout,
37 InvalidHeader,
38 InvalidProxyURL,
39 InvalidSchema,
40 InvalidURL,
41 ProxyError,
42 ReadTimeout,
43 RetryError,
44 SSLError,
45)
46from .models import Response
47from .structures import CaseInsensitiveDict
48from .utils import (
49 DEFAULT_CA_BUNDLE_PATH,
50 extract_zipped_paths,
51 get_auth_from_url,
52 get_encoding_from_headers,
53 prepend_scheme_if_needed,
54 select_proxy,
55 urldefragauth,
56)
58try:
59 from urllib3.contrib.socks import SOCKSProxyManager
60except ImportError:
62 def SOCKSProxyManager(*args, **kwargs):
63 raise InvalidSchema("Missing dependencies for SOCKS support.")
66if typing.TYPE_CHECKING:
67 from .models import PreparedRequest
70DEFAULT_POOLBLOCK = False
71DEFAULT_POOLSIZE = 10
72DEFAULT_RETRIES = 0
73DEFAULT_POOL_TIMEOUT = None
75_preloaded_ssl_context = create_urllib3_context()
76_preloaded_ssl_context.load_verify_locations(
77 extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
78)
81def _urllib3_request_context(
82 request: "PreparedRequest",
83 verify: "bool | str | None",
84 client_cert: "typing.Tuple[str, str] | str | None",
85) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])":
86 host_params = {}
87 pool_kwargs = {}
88 parsed_request_url = urlparse(request.url)
89 scheme = parsed_request_url.scheme.lower()
90 port = parsed_request_url.port
91 cert_reqs = "CERT_REQUIRED"
92 if verify is False:
93 cert_reqs = "CERT_NONE"
94 elif verify is True:
95 pool_kwargs["ssl_context"] = _preloaded_ssl_context
96 elif isinstance(verify, str):
97 if not os.path.isdir(verify):
98 pool_kwargs["ca_certs"] = verify
99 else:
100 pool_kwargs["ca_cert_dir"] = verify
101 pool_kwargs["cert_reqs"] = cert_reqs
102 if client_cert is not None:
103 if isinstance(client_cert, tuple) and len(client_cert) == 2:
104 pool_kwargs["cert_file"] = client_cert[0]
105 pool_kwargs["key_file"] = client_cert[1]
106 else:
107 # According to our docs, we allow users to specify just the client
108 # cert path
109 pool_kwargs["cert_file"] = client_cert
110 host_params = {
111 "scheme": scheme,
112 "host": parsed_request_url.hostname,
113 "port": port,
114 }
115 return host_params, pool_kwargs
118class BaseAdapter:
119 """The Base Transport Adapter"""
121 def __init__(self):
122 super().__init__()
124 def send(
125 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
126 ):
127 """Sends PreparedRequest object. Returns Response object.
129 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
130 :param stream: (optional) Whether to stream the request content.
131 :param timeout: (optional) How long to wait for the server to send
132 data before giving up, as a float, or a :ref:`(connect timeout,
133 read timeout) <timeouts>` tuple.
134 :type timeout: float or tuple
135 :param verify: (optional) Either a boolean, in which case it controls whether we verify
136 the server's TLS certificate, or a string, in which case it must be a path
137 to a CA bundle to use
138 :param cert: (optional) Any user-provided SSL certificate to be trusted.
139 :param proxies: (optional) The proxies dictionary to apply to the request.
140 """
141 raise NotImplementedError
143 def close(self):
144 """Cleans up adapter specific items."""
145 raise NotImplementedError
148class HTTPAdapter(BaseAdapter):
149 """The built-in HTTP Adapter for urllib3.
151 Provides a general-case interface for Requests sessions to contact HTTP and
152 HTTPS urls by implementing the Transport Adapter interface. This class will
153 usually be created by the :class:`Session <Session>` class under the
154 covers.
156 :param pool_connections: The number of urllib3 connection pools to cache.
157 :param pool_maxsize: The maximum number of connections to save in the pool.
158 :param max_retries: The maximum number of retries each connection
159 should attempt. Note, this applies only to failed DNS lookups, socket
160 connections and connection timeouts, never to requests where data has
161 made it to the server. By default, Requests does not retry failed
162 connections. If you need granular control over the conditions under
163 which we retry a request, import urllib3's ``Retry`` class and pass
164 that instead.
165 :param pool_block: Whether the connection pool should block for connections.
167 Usage::
169 >>> import requests
170 >>> s = requests.Session()
171 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
172 >>> s.mount('http://', a)
173 """
175 __attrs__ = [
176 "max_retries",
177 "config",
178 "_pool_connections",
179 "_pool_maxsize",
180 "_pool_block",
181 ]
183 def __init__(
184 self,
185 pool_connections=DEFAULT_POOLSIZE,
186 pool_maxsize=DEFAULT_POOLSIZE,
187 max_retries=DEFAULT_RETRIES,
188 pool_block=DEFAULT_POOLBLOCK,
189 ):
190 if max_retries == DEFAULT_RETRIES:
191 self.max_retries = Retry(0, read=False)
192 else:
193 self.max_retries = Retry.from_int(max_retries)
194 self.config = {}
195 self.proxy_manager = {}
197 super().__init__()
199 self._pool_connections = pool_connections
200 self._pool_maxsize = pool_maxsize
201 self._pool_block = pool_block
203 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
205 def __getstate__(self):
206 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
208 def __setstate__(self, state):
209 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
210 # self.poolmanager uses a lambda function, which isn't pickleable.
211 self.proxy_manager = {}
212 self.config = {}
214 for attr, value in state.items():
215 setattr(self, attr, value)
217 self.init_poolmanager(
218 self._pool_connections, self._pool_maxsize, block=self._pool_block
219 )
221 def init_poolmanager(
222 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
223 ):
224 """Initializes a urllib3 PoolManager.
226 This method should not be called from user code, and is only
227 exposed for use when subclassing the
228 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
230 :param connections: The number of urllib3 connection pools to cache.
231 :param maxsize: The maximum number of connections to save in the pool.
232 :param block: Block when no free connections are available.
233 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
234 """
235 # save these values for pickling
236 self._pool_connections = connections
237 self._pool_maxsize = maxsize
238 self._pool_block = block
240 self.poolmanager = PoolManager(
241 num_pools=connections,
242 maxsize=maxsize,
243 block=block,
244 **pool_kwargs,
245 )
247 def proxy_manager_for(self, proxy, **proxy_kwargs):
248 """Return urllib3 ProxyManager for the given proxy.
250 This method should not be called from user code, and is only
251 exposed for use when subclassing the
252 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
254 :param proxy: The proxy to return a urllib3 ProxyManager for.
255 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
256 :returns: ProxyManager
257 :rtype: urllib3.ProxyManager
258 """
259 if proxy in self.proxy_manager:
260 manager = self.proxy_manager[proxy]
261 elif proxy.lower().startswith("socks"):
262 username, password = get_auth_from_url(proxy)
263 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
264 proxy,
265 username=username,
266 password=password,
267 num_pools=self._pool_connections,
268 maxsize=self._pool_maxsize,
269 block=self._pool_block,
270 **proxy_kwargs,
271 )
272 else:
273 proxy_headers = self.proxy_headers(proxy)
274 manager = self.proxy_manager[proxy] = proxy_from_url(
275 proxy,
276 proxy_headers=proxy_headers,
277 num_pools=self._pool_connections,
278 maxsize=self._pool_maxsize,
279 block=self._pool_block,
280 **proxy_kwargs,
281 )
283 return manager
285 def cert_verify(self, conn, url, verify, cert):
286 """Verify a SSL certificate. This method should not be called from user
287 code, and is only exposed for use when subclassing the
288 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
290 :param conn: The urllib3 connection object associated with the cert.
291 :param url: The requested URL.
292 :param verify: Either a boolean, in which case it controls whether we verify
293 the server's TLS certificate, or a string, in which case it must be a path
294 to a CA bundle to use
295 :param cert: The SSL certificate to verify.
296 """
297 if url.lower().startswith("https") and verify:
298 conn.cert_reqs = "CERT_REQUIRED"
300 # Only load the CA certificates if 'verify' is a string indicating the CA bundle to use.
301 # Otherwise, if verify is a boolean, we don't load anything since
302 # the connection will be using a context with the default certificates already loaded,
303 # and this avoids a call to the slow load_verify_locations()
304 if verify is not True:
305 # `verify` must be a str with a path then
306 cert_loc = verify
308 if not os.path.exists(cert_loc):
309 raise OSError(
310 f"Could not find a suitable TLS CA certificate bundle, "
311 f"invalid path: {cert_loc}"
312 )
314 if not os.path.isdir(cert_loc):
315 conn.ca_certs = cert_loc
316 else:
317 conn.ca_cert_dir = cert_loc
318 else:
319 conn.cert_reqs = "CERT_NONE"
320 conn.ca_certs = None
321 conn.ca_cert_dir = None
323 if cert:
324 if not isinstance(cert, basestring):
325 conn.cert_file = cert[0]
326 conn.key_file = cert[1]
327 else:
328 conn.cert_file = cert
329 conn.key_file = None
330 if conn.cert_file and not os.path.exists(conn.cert_file):
331 raise OSError(
332 f"Could not find the TLS certificate file, "
333 f"invalid path: {conn.cert_file}"
334 )
335 if conn.key_file and not os.path.exists(conn.key_file):
336 raise OSError(
337 f"Could not find the TLS key file, invalid path: {conn.key_file}"
338 )
340 def build_response(self, req, resp):
341 """Builds a :class:`Response <requests.Response>` object from a urllib3
342 response. This should not be called from user code, and is only exposed
343 for use when subclassing the
344 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
346 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
347 :param resp: The urllib3 response object.
348 :rtype: requests.Response
349 """
350 response = Response()
352 # Fallback to None if there's no status_code, for whatever reason.
353 response.status_code = getattr(resp, "status", None)
355 # Make headers case-insensitive.
356 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
358 # Set encoding.
359 response.encoding = get_encoding_from_headers(response.headers)
360 response.raw = resp
361 response.reason = response.raw.reason
363 if isinstance(req.url, bytes):
364 response.url = req.url.decode("utf-8")
365 else:
366 response.url = req.url
368 # Add new cookies from the server.
369 extract_cookies_to_jar(response.cookies, req, resp)
371 # Give the Response some context.
372 response.request = req
373 response.connection = self
375 return response
377 def _get_connection(self, request, verify, proxies=None, cert=None):
378 # Replace the existing get_connection without breaking things and
379 # ensure that TLS settings are considered when we interact with
380 # urllib3 HTTP Pools
381 proxy = select_proxy(request.url, proxies)
382 try:
383 host_params, pool_kwargs = _urllib3_request_context(request, verify, cert)
384 except ValueError as e:
385 raise InvalidURL(e, request=request)
386 if proxy:
387 proxy = prepend_scheme_if_needed(proxy, "http")
388 proxy_url = parse_url(proxy)
389 if not proxy_url.host:
390 raise InvalidProxyURL(
391 "Please check proxy URL. It is malformed "
392 "and could be missing the host."
393 )
394 proxy_manager = self.proxy_manager_for(proxy)
395 conn = proxy_manager.connection_from_host(
396 **host_params, pool_kwargs=pool_kwargs
397 )
398 else:
399 # Only scheme should be lower case
400 conn = self.poolmanager.connection_from_host(
401 **host_params, pool_kwargs=pool_kwargs
402 )
404 return conn
406 def get_connection(self, url, proxies=None):
407 """Returns a urllib3 connection for the given URL. This should not be
408 called from user code, and is only exposed for use when subclassing the
409 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
411 :param url: The URL to connect to.
412 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
413 :rtype: urllib3.ConnectionPool
414 """
415 proxy = select_proxy(url, proxies)
417 if proxy:
418 proxy = prepend_scheme_if_needed(proxy, "http")
419 proxy_url = parse_url(proxy)
420 if not proxy_url.host:
421 raise InvalidProxyURL(
422 "Please check proxy URL. It is malformed "
423 "and could be missing the host."
424 )
425 proxy_manager = self.proxy_manager_for(proxy)
426 conn = proxy_manager.connection_from_url(url)
427 else:
428 # Only scheme should be lower case
429 parsed = urlparse(url)
430 url = parsed.geturl()
431 conn = self.poolmanager.connection_from_url(url)
433 return conn
435 def close(self):
436 """Disposes of any internal state.
438 Currently, this closes the PoolManager and any active ProxyManager,
439 which closes any pooled connections.
440 """
441 self.poolmanager.clear()
442 for proxy in self.proxy_manager.values():
443 proxy.clear()
445 def request_url(self, request, proxies):
446 """Obtain the url to use when making the final request.
448 If the message is being sent through a HTTP proxy, the full URL has to
449 be used. Otherwise, we should only use the path portion of the URL.
451 This should not be called from user code, and is only exposed for use
452 when subclassing the
453 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
455 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
456 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
457 :rtype: str
458 """
459 proxy = select_proxy(request.url, proxies)
460 scheme = urlparse(request.url).scheme
462 is_proxied_http_request = proxy and scheme != "https"
463 using_socks_proxy = False
464 if proxy:
465 proxy_scheme = urlparse(proxy).scheme.lower()
466 using_socks_proxy = proxy_scheme.startswith("socks")
468 url = request.path_url
469 if url.startswith("//"): # Don't confuse urllib3
470 url = f"/{url.lstrip('/')}"
472 if is_proxied_http_request and not using_socks_proxy:
473 url = urldefragauth(request.url)
475 return url
477 def add_headers(self, request, **kwargs):
478 """Add any headers needed by the connection. As of v2.0 this does
479 nothing by default, but is left for overriding by users that subclass
480 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
482 This should not be called from user code, and is only exposed for use
483 when subclassing the
484 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
486 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
487 :param kwargs: The keyword arguments from the call to send().
488 """
489 pass
491 def proxy_headers(self, proxy):
492 """Returns a dictionary of the headers to add to any request sent
493 through a proxy. This works with urllib3 magic to ensure that they are
494 correctly sent to the proxy, rather than in a tunnelled request if
495 CONNECT is being used.
497 This should not be called from user code, and is only exposed for use
498 when subclassing the
499 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
501 :param proxy: The url of the proxy being used for this request.
502 :rtype: dict
503 """
504 headers = {}
505 username, password = get_auth_from_url(proxy)
507 if username:
508 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
510 return headers
512 def send(
513 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
514 ):
515 """Sends PreparedRequest object. Returns Response object.
517 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
518 :param stream: (optional) Whether to stream the request content.
519 :param timeout: (optional) How long to wait for the server to send
520 data before giving up, as a float, or a :ref:`(connect timeout,
521 read timeout) <timeouts>` tuple.
522 :type timeout: float or tuple or urllib3 Timeout object
523 :param verify: (optional) Either a boolean, in which case it controls whether
524 we verify the server's TLS certificate, or a string, in which case it
525 must be a path to a CA bundle to use
526 :param cert: (optional) Any user-provided SSL certificate to be trusted.
527 :param proxies: (optional) The proxies dictionary to apply to the request.
528 :rtype: requests.Response
529 """
531 try:
532 conn = self._get_connection(request, verify, proxies=proxies, cert=cert)
533 except LocationValueError as e:
534 raise InvalidURL(e, request=request)
536 self.cert_verify(conn, request.url, verify, cert)
537 url = self.request_url(request, proxies)
538 self.add_headers(
539 request,
540 stream=stream,
541 timeout=timeout,
542 verify=verify,
543 cert=cert,
544 proxies=proxies,
545 )
547 chunked = not (request.body is None or "Content-Length" in request.headers)
549 if isinstance(timeout, tuple):
550 try:
551 connect, read = timeout
552 timeout = TimeoutSauce(connect=connect, read=read)
553 except ValueError:
554 raise ValueError(
555 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
556 f"or a single float to set both timeouts to the same value."
557 )
558 elif isinstance(timeout, TimeoutSauce):
559 pass
560 else:
561 timeout = TimeoutSauce(connect=timeout, read=timeout)
563 try:
564 resp = conn.urlopen(
565 method=request.method,
566 url=url,
567 body=request.body,
568 headers=request.headers,
569 redirect=False,
570 assert_same_host=False,
571 preload_content=False,
572 decode_content=False,
573 retries=self.max_retries,
574 timeout=timeout,
575 chunked=chunked,
576 )
578 except (ProtocolError, OSError) as err:
579 raise ConnectionError(err, request=request)
581 except MaxRetryError as e:
582 if isinstance(e.reason, ConnectTimeoutError):
583 # TODO: Remove this in 3.0.0: see #2811
584 if not isinstance(e.reason, NewConnectionError):
585 raise ConnectTimeout(e, request=request)
587 if isinstance(e.reason, ResponseError):
588 raise RetryError(e, request=request)
590 if isinstance(e.reason, _ProxyError):
591 raise ProxyError(e, request=request)
593 if isinstance(e.reason, _SSLError):
594 # This branch is for urllib3 v1.22 and later.
595 raise SSLError(e, request=request)
597 raise ConnectionError(e, request=request)
599 except ClosedPoolError as e:
600 raise ConnectionError(e, request=request)
602 except _ProxyError as e:
603 raise ProxyError(e)
605 except (_SSLError, _HTTPError) as e:
606 if isinstance(e, _SSLError):
607 # This branch is for urllib3 versions earlier than v1.22
608 raise SSLError(e, request=request)
609 elif isinstance(e, ReadTimeoutError):
610 raise ReadTimeout(e, request=request)
611 elif isinstance(e, _InvalidHeader):
612 raise InvalidHeader(e, request=request)
613 else:
614 raise
616 return self.build_response(request, resp)