Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/requests/adapters.py: 24%
194 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:40 +0000
1"""
2requests.adapters
3~~~~~~~~~~~~~~~~~
5This module contains the transport adapters that Requests uses to define
6and maintain connections.
7"""
9import os.path
10import socket # noqa: F401
12from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
13from urllib3.exceptions import HTTPError as _HTTPError
14from urllib3.exceptions import InvalidHeader as _InvalidHeader
15from urllib3.exceptions import (
16 LocationValueError,
17 MaxRetryError,
18 NewConnectionError,
19 ProtocolError,
20)
21from urllib3.exceptions import ProxyError as _ProxyError
22from urllib3.exceptions import ReadTimeoutError, ResponseError
23from urllib3.exceptions import SSLError as _SSLError
24from urllib3.poolmanager import PoolManager, proxy_from_url
25from urllib3.util import Timeout as TimeoutSauce
26from urllib3.util import parse_url
27from urllib3.util.retry import Retry
29from .auth import _basic_auth_str
30from .compat import basestring, urlparse
31from .cookies import extract_cookies_to_jar
32from .exceptions import (
33 ConnectionError,
34 ConnectTimeout,
35 InvalidHeader,
36 InvalidProxyURL,
37 InvalidSchema,
38 InvalidURL,
39 ProxyError,
40 ReadTimeout,
41 RetryError,
42 SSLError,
43)
44from .models import Response
45from .structures import CaseInsensitiveDict
46from .utils import (
47 DEFAULT_CA_BUNDLE_PATH,
48 extract_zipped_paths,
49 get_auth_from_url,
50 get_encoding_from_headers,
51 prepend_scheme_if_needed,
52 select_proxy,
53 urldefragauth,
54)
56try:
57 from urllib3.contrib.socks import SOCKSProxyManager
58except ImportError:
60 def SOCKSProxyManager(*args, **kwargs):
61 raise InvalidSchema("Missing dependencies for SOCKS support.")
64DEFAULT_POOLBLOCK = False
65DEFAULT_POOLSIZE = 10
66DEFAULT_RETRIES = 0
67DEFAULT_POOL_TIMEOUT = None
70class BaseAdapter:
71 """The Base Transport Adapter"""
73 def __init__(self):
74 super().__init__()
76 def send(
77 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
78 ):
79 """Sends PreparedRequest object. Returns Response object.
81 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
82 :param stream: (optional) Whether to stream the request content.
83 :param timeout: (optional) How long to wait for the server to send
84 data before giving up, as a float, or a :ref:`(connect timeout,
85 read timeout) <timeouts>` tuple.
86 :type timeout: float or tuple
87 :param verify: (optional) Either a boolean, in which case it controls whether we verify
88 the server's TLS certificate, or a string, in which case it must be a path
89 to a CA bundle to use
90 :param cert: (optional) Any user-provided SSL certificate to be trusted.
91 :param proxies: (optional) The proxies dictionary to apply to the request.
92 """
93 raise NotImplementedError
95 def close(self):
96 """Cleans up adapter specific items."""
97 raise NotImplementedError
100class HTTPAdapter(BaseAdapter):
101 """The built-in HTTP Adapter for urllib3.
103 Provides a general-case interface for Requests sessions to contact HTTP and
104 HTTPS urls by implementing the Transport Adapter interface. This class will
105 usually be created by the :class:`Session <Session>` class under the
106 covers.
108 :param pool_connections: The number of urllib3 connection pools to cache.
109 :param pool_maxsize: The maximum number of connections to save in the pool.
110 :param max_retries: The maximum number of retries each connection
111 should attempt. Note, this applies only to failed DNS lookups, socket
112 connections and connection timeouts, never to requests where data has
113 made it to the server. By default, Requests does not retry failed
114 connections. If you need granular control over the conditions under
115 which we retry a request, import urllib3's ``Retry`` class and pass
116 that instead.
117 :param pool_block: Whether the connection pool should block for connections.
119 Usage::
121 >>> import requests
122 >>> s = requests.Session()
123 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
124 >>> s.mount('http://', a)
125 """
127 __attrs__ = [
128 "max_retries",
129 "config",
130 "_pool_connections",
131 "_pool_maxsize",
132 "_pool_block",
133 ]
135 def __init__(
136 self,
137 pool_connections=DEFAULT_POOLSIZE,
138 pool_maxsize=DEFAULT_POOLSIZE,
139 max_retries=DEFAULT_RETRIES,
140 pool_block=DEFAULT_POOLBLOCK,
141 ):
142 if max_retries == DEFAULT_RETRIES:
143 self.max_retries = Retry(0, read=False)
144 else:
145 self.max_retries = Retry.from_int(max_retries)
146 self.config = {}
147 self.proxy_manager = {}
149 super().__init__()
151 self._pool_connections = pool_connections
152 self._pool_maxsize = pool_maxsize
153 self._pool_block = pool_block
155 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
157 def __getstate__(self):
158 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
160 def __setstate__(self, state):
161 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
162 # self.poolmanager uses a lambda function, which isn't pickleable.
163 self.proxy_manager = {}
164 self.config = {}
166 for attr, value in state.items():
167 setattr(self, attr, value)
169 self.init_poolmanager(
170 self._pool_connections, self._pool_maxsize, block=self._pool_block
171 )
173 def init_poolmanager(
174 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
175 ):
176 """Initializes a urllib3 PoolManager.
178 This method should not be called from user code, and is only
179 exposed for use when subclassing the
180 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
182 :param connections: The number of urllib3 connection pools to cache.
183 :param maxsize: The maximum number of connections to save in the pool.
184 :param block: Block when no free connections are available.
185 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
186 """
187 # save these values for pickling
188 self._pool_connections = connections
189 self._pool_maxsize = maxsize
190 self._pool_block = block
192 self.poolmanager = PoolManager(
193 num_pools=connections,
194 maxsize=maxsize,
195 block=block,
196 **pool_kwargs,
197 )
199 def proxy_manager_for(self, proxy, **proxy_kwargs):
200 """Return urllib3 ProxyManager for the given proxy.
202 This method should not be called from user code, and is only
203 exposed for use when subclassing the
204 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
206 :param proxy: The proxy to return a urllib3 ProxyManager for.
207 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
208 :returns: ProxyManager
209 :rtype: urllib3.ProxyManager
210 """
211 if proxy in self.proxy_manager:
212 manager = self.proxy_manager[proxy]
213 elif proxy.lower().startswith("socks"):
214 username, password = get_auth_from_url(proxy)
215 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
216 proxy,
217 username=username,
218 password=password,
219 num_pools=self._pool_connections,
220 maxsize=self._pool_maxsize,
221 block=self._pool_block,
222 **proxy_kwargs,
223 )
224 else:
225 proxy_headers = self.proxy_headers(proxy)
226 manager = self.proxy_manager[proxy] = proxy_from_url(
227 proxy,
228 proxy_headers=proxy_headers,
229 num_pools=self._pool_connections,
230 maxsize=self._pool_maxsize,
231 block=self._pool_block,
232 **proxy_kwargs,
233 )
235 return manager
237 def cert_verify(self, conn, url, verify, cert):
238 """Verify a SSL certificate. This method should not be called from user
239 code, and is only exposed for use when subclassing the
240 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
242 :param conn: The urllib3 connection object associated with the cert.
243 :param url: The requested URL.
244 :param verify: Either a boolean, in which case it controls whether we verify
245 the server's TLS certificate, or a string, in which case it must be a path
246 to a CA bundle to use
247 :param cert: The SSL certificate to verify.
248 """
249 if url.lower().startswith("https") and verify:
251 cert_loc = None
253 # Allow self-specified cert location.
254 if verify is not True:
255 cert_loc = verify
257 if not cert_loc:
258 cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
260 if not cert_loc or not os.path.exists(cert_loc):
261 raise OSError(
262 f"Could not find a suitable TLS CA certificate bundle, "
263 f"invalid path: {cert_loc}"
264 )
266 conn.cert_reqs = "CERT_REQUIRED"
268 if not os.path.isdir(cert_loc):
269 conn.ca_certs = cert_loc
270 else:
271 conn.ca_cert_dir = cert_loc
272 else:
273 conn.cert_reqs = "CERT_NONE"
274 conn.ca_certs = None
275 conn.ca_cert_dir = None
277 if cert:
278 if not isinstance(cert, basestring):
279 conn.cert_file = cert[0]
280 conn.key_file = cert[1]
281 else:
282 conn.cert_file = cert
283 conn.key_file = None
284 if conn.cert_file and not os.path.exists(conn.cert_file):
285 raise OSError(
286 f"Could not find the TLS certificate file, "
287 f"invalid path: {conn.cert_file}"
288 )
289 if conn.key_file and not os.path.exists(conn.key_file):
290 raise OSError(
291 f"Could not find the TLS key file, invalid path: {conn.key_file}"
292 )
294 def build_response(self, req, resp):
295 """Builds a :class:`Response <requests.Response>` object from a urllib3
296 response. This should not be called from user code, and is only exposed
297 for use when subclassing the
298 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
300 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
301 :param resp: The urllib3 response object.
302 :rtype: requests.Response
303 """
304 response = Response()
306 # Fallback to None if there's no status_code, for whatever reason.
307 response.status_code = getattr(resp, "status", None)
309 # Make headers case-insensitive.
310 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
312 # Set encoding.
313 response.encoding = get_encoding_from_headers(response.headers)
314 response.raw = resp
315 response.reason = response.raw.reason
317 if isinstance(req.url, bytes):
318 response.url = req.url.decode("utf-8")
319 else:
320 response.url = req.url
322 # Add new cookies from the server.
323 extract_cookies_to_jar(response.cookies, req, resp)
325 # Give the Response some context.
326 response.request = req
327 response.connection = self
329 return response
331 def get_connection(self, url, proxies=None):
332 """Returns a urllib3 connection for the given URL. This should not be
333 called from user code, and is only exposed for use when subclassing the
334 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
336 :param url: The URL to connect to.
337 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
338 :rtype: urllib3.ConnectionPool
339 """
340 proxy = select_proxy(url, proxies)
342 if proxy:
343 proxy = prepend_scheme_if_needed(proxy, "http")
344 proxy_url = parse_url(proxy)
345 if not proxy_url.host:
346 raise InvalidProxyURL(
347 "Please check proxy URL. It is malformed "
348 "and could be missing the host."
349 )
350 proxy_manager = self.proxy_manager_for(proxy)
351 conn = proxy_manager.connection_from_url(url)
352 else:
353 # Only scheme should be lower case
354 parsed = urlparse(url)
355 url = parsed.geturl()
356 conn = self.poolmanager.connection_from_url(url)
358 return conn
360 def close(self):
361 """Disposes of any internal state.
363 Currently, this closes the PoolManager and any active ProxyManager,
364 which closes any pooled connections.
365 """
366 self.poolmanager.clear()
367 for proxy in self.proxy_manager.values():
368 proxy.clear()
370 def request_url(self, request, proxies):
371 """Obtain the url to use when making the final request.
373 If the message is being sent through a HTTP proxy, the full URL has to
374 be used. Otherwise, we should only use the path portion of the URL.
376 This should not be called from user code, and is only exposed for use
377 when subclassing the
378 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
380 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
381 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
382 :rtype: str
383 """
384 proxy = select_proxy(request.url, proxies)
385 scheme = urlparse(request.url).scheme
387 is_proxied_http_request = proxy and scheme != "https"
388 using_socks_proxy = False
389 if proxy:
390 proxy_scheme = urlparse(proxy).scheme.lower()
391 using_socks_proxy = proxy_scheme.startswith("socks")
393 url = request.path_url
394 if is_proxied_http_request and not using_socks_proxy:
395 url = urldefragauth(request.url)
397 return url
399 def add_headers(self, request, **kwargs):
400 """Add any headers needed by the connection. As of v2.0 this does
401 nothing by default, but is left for overriding by users that subclass
402 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
404 This should not be called from user code, and is only exposed for use
405 when subclassing the
406 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
408 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
409 :param kwargs: The keyword arguments from the call to send().
410 """
411 pass
413 def proxy_headers(self, proxy):
414 """Returns a dictionary of the headers to add to any request sent
415 through a proxy. This works with urllib3 magic to ensure that they are
416 correctly sent to the proxy, rather than in a tunnelled request if
417 CONNECT is being used.
419 This should not be called from user code, and is only exposed for use
420 when subclassing the
421 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
423 :param proxy: The url of the proxy being used for this request.
424 :rtype: dict
425 """
426 headers = {}
427 username, password = get_auth_from_url(proxy)
429 if username:
430 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
432 return headers
434 def send(
435 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
436 ):
437 """Sends PreparedRequest object. Returns Response object.
439 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
440 :param stream: (optional) Whether to stream the request content.
441 :param timeout: (optional) How long to wait for the server to send
442 data before giving up, as a float, or a :ref:`(connect timeout,
443 read timeout) <timeouts>` tuple.
444 :type timeout: float or tuple or urllib3 Timeout object
445 :param verify: (optional) Either a boolean, in which case it controls whether
446 we verify the server's TLS certificate, or a string, in which case it
447 must be a path to a CA bundle to use
448 :param cert: (optional) Any user-provided SSL certificate to be trusted.
449 :param proxies: (optional) The proxies dictionary to apply to the request.
450 :rtype: requests.Response
451 """
453 try:
454 conn = self.get_connection(request.url, proxies)
455 except LocationValueError as e:
456 raise InvalidURL(e, request=request)
458 self.cert_verify(conn, request.url, verify, cert)
459 url = self.request_url(request, proxies)
460 self.add_headers(
461 request,
462 stream=stream,
463 timeout=timeout,
464 verify=verify,
465 cert=cert,
466 proxies=proxies,
467 )
469 chunked = not (request.body is None or "Content-Length" in request.headers)
471 if isinstance(timeout, tuple):
472 try:
473 connect, read = timeout
474 timeout = TimeoutSauce(connect=connect, read=read)
475 except ValueError:
476 raise ValueError(
477 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
478 f"or a single float to set both timeouts to the same value."
479 )
480 elif isinstance(timeout, TimeoutSauce):
481 pass
482 else:
483 timeout = TimeoutSauce(connect=timeout, read=timeout)
485 try:
486 resp = conn.urlopen(
487 method=request.method,
488 url=url,
489 body=request.body,
490 headers=request.headers,
491 redirect=False,
492 assert_same_host=False,
493 preload_content=False,
494 decode_content=False,
495 retries=self.max_retries,
496 timeout=timeout,
497 chunked=chunked,
498 )
500 except (ProtocolError, OSError) as err:
501 raise ConnectionError(err, request=request)
503 except MaxRetryError as e:
504 if isinstance(e.reason, ConnectTimeoutError):
505 # TODO: Remove this in 3.0.0: see #2811
506 if not isinstance(e.reason, NewConnectionError):
507 raise ConnectTimeout(e, request=request)
509 if isinstance(e.reason, ResponseError):
510 raise RetryError(e, request=request)
512 if isinstance(e.reason, _ProxyError):
513 raise ProxyError(e, request=request)
515 if isinstance(e.reason, _SSLError):
516 # This branch is for urllib3 v1.22 and later.
517 raise SSLError(e, request=request)
519 raise ConnectionError(e, request=request)
521 except ClosedPoolError as e:
522 raise ConnectionError(e, request=request)
524 except _ProxyError as e:
525 raise ProxyError(e)
527 except (_SSLError, _HTTPError) as e:
528 if isinstance(e, _SSLError):
529 # This branch is for urllib3 versions earlier than v1.22
530 raise SSLError(e, request=request)
531 elif isinstance(e, ReadTimeoutError):
532 raise ReadTimeout(e, request=request)
533 elif isinstance(e, _InvalidHeader):
534 raise InvalidHeader(e, request=request)
535 else:
536 raise
538 return self.build_response(request, resp)