Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/httpsession.py: 25%
256 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-26 06:03 +0000
1import logging
2import os
3import os.path
4import socket
5import sys
6import warnings
7from base64 import b64encode
9from urllib3 import PoolManager, Timeout, proxy_from_url
10from urllib3.exceptions import (
11 ConnectTimeoutError as URLLib3ConnectTimeoutError,
12)
13from urllib3.exceptions import (
14 LocationParseError,
15 NewConnectionError,
16 ProtocolError,
17 ProxyError,
18)
19from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
20from urllib3.exceptions import SSLError as URLLib3SSLError
21from urllib3.util.retry import Retry
22from urllib3.util.ssl_ import (
23 DEFAULT_CIPHERS,
24 OP_NO_COMPRESSION,
25 PROTOCOL_TLS,
26 OP_NO_SSLv2,
27 OP_NO_SSLv3,
28 is_ipaddress,
29 ssl,
30)
31from urllib3.util.url import parse_url
33try:
34 from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT
35except ImportError:
36 # Fallback directly to ssl for version of urllib3 before 1.26.
37 # They are available in the standard library starting in Python 3.6.
38 from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT
40try:
41 # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point.
42 # This can be removed once our urllib3 floor is raised to >= 2.0.
43 with warnings.catch_warnings():
44 warnings.simplefilter("ignore", category=DeprecationWarning)
45 # Always import the original SSLContext, even if it has been patched
46 from urllib3.contrib.pyopenssl import (
47 orig_util_SSLContext as SSLContext,
48 )
49except ImportError:
50 from urllib3.util.ssl_ import SSLContext
52import botocore.awsrequest
53from botocore.compat import (
54 IPV6_ADDRZ_RE,
55 ensure_bytes,
56 filter_ssl_warnings,
57 unquote,
58 urlparse,
59)
60from botocore.exceptions import (
61 ConnectionClosedError,
62 ConnectTimeoutError,
63 EndpointConnectionError,
64 HTTPClientError,
65 InvalidProxiesConfigError,
66 ProxyConnectionError,
67 ReadTimeoutError,
68 SSLError,
69)
71filter_ssl_warnings()
72logger = logging.getLogger(__name__)
73DEFAULT_TIMEOUT = 60
74MAX_POOL_CONNECTIONS = 10
75DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem')
77try:
78 from certifi import where
79except ImportError:
81 def where():
82 return DEFAULT_CA_BUNDLE
85def get_cert_path(verify):
86 if verify is not True:
87 return verify
89 cert_path = where()
90 logger.debug(f"Certificate path: {cert_path}")
92 return cert_path
95def create_urllib3_context(
96 ssl_version=None, cert_reqs=None, options=None, ciphers=None
97):
98 """This function is a vendored version of the same function in urllib3
100 We vendor this function to ensure that the SSL contexts we construct
101 always use the std lib SSLContext instead of pyopenssl.
102 """
103 # PROTOCOL_TLS is deprecated in Python 3.10
104 if not ssl_version or ssl_version == PROTOCOL_TLS:
105 ssl_version = PROTOCOL_TLS_CLIENT
107 context = SSLContext(ssl_version)
109 context.set_ciphers(ciphers or DEFAULT_CIPHERS)
111 # Setting the default here, as we may have no ssl module on import
112 cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
114 if options is None:
115 options = 0
116 # SSLv2 is easily broken and is considered harmful and dangerous
117 options |= OP_NO_SSLv2
118 # SSLv3 has several problems and is now dangerous
119 options |= OP_NO_SSLv3
120 # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
121 # (issue urllib3#309)
122 options |= OP_NO_COMPRESSION
123 # TLSv1.2 only. Unless set explicitly, do not request tickets.
124 # This may save some bandwidth on wire, and although the ticket is encrypted,
125 # there is a risk associated with it being on wire,
126 # if the server is not rotating its ticketing keys properly.
127 options |= OP_NO_TICKET
129 context.options |= options
131 # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
132 # necessary for conditional client cert authentication with TLS 1.3.
133 # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
134 # versions of Python. We only enable on Python 3.7.4+ or if certificate
135 # verification is enabled to work around Python issue #37428
136 # See: https://bugs.python.org/issue37428
137 if (
138 cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)
139 ) and getattr(context, "post_handshake_auth", None) is not None:
140 context.post_handshake_auth = True
142 def disable_check_hostname():
143 if (
144 getattr(context, "check_hostname", None) is not None
145 ): # Platform-specific: Python 3.2
146 # We do our own verification, including fingerprints and alternative
147 # hostnames. So disable it here
148 context.check_hostname = False
150 # The order of the below lines setting verify_mode and check_hostname
151 # matter due to safe-guards SSLContext has to prevent an SSLContext with
152 # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more
153 # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used
154 # or not so we don't know the initial state of the freshly created SSLContext.
155 if cert_reqs == ssl.CERT_REQUIRED:
156 context.verify_mode = cert_reqs
157 disable_check_hostname()
158 else:
159 disable_check_hostname()
160 context.verify_mode = cert_reqs
162 # Enable logging of TLS session keys via defacto standard environment variable
163 # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
164 if hasattr(context, "keylog_filename"):
165 sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
166 if sslkeylogfile and not sys.flags.ignore_environment:
167 context.keylog_filename = sslkeylogfile
169 return context
172def ensure_boolean(val):
173 """Ensures a boolean value if a string or boolean is provided
175 For strings, the value for True/False is case insensitive
176 """
177 if isinstance(val, bool):
178 return val
179 else:
180 return val.lower() == 'true'
183def mask_proxy_url(proxy_url):
184 """
185 Mask proxy url credentials.
187 :type proxy_url: str
188 :param proxy_url: The proxy url, i.e. https://username:password@proxy.com
190 :return: Masked proxy url, i.e. https://***:***@proxy.com
191 """
192 mask = '*' * 3
193 parsed_url = urlparse(proxy_url)
194 if parsed_url.username:
195 proxy_url = proxy_url.replace(parsed_url.username, mask, 1)
196 if parsed_url.password:
197 proxy_url = proxy_url.replace(parsed_url.password, mask, 1)
198 return proxy_url
201def _is_ipaddress(host):
202 """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses."""
203 return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host))
206class ProxyConfiguration:
207 """Represents a proxy configuration dictionary and additional settings.
209 This class represents a proxy configuration dictionary and provides utility
210 functions to retreive well structured proxy urls and proxy headers from the
211 proxy configuration dictionary.
212 """
214 def __init__(self, proxies=None, proxies_settings=None):
215 if proxies is None:
216 proxies = {}
217 if proxies_settings is None:
218 proxies_settings = {}
220 self._proxies = proxies
221 self._proxies_settings = proxies_settings
223 def proxy_url_for(self, url):
224 """Retrieves the corresponding proxy url for a given url."""
225 parsed_url = urlparse(url)
226 proxy = self._proxies.get(parsed_url.scheme)
227 if proxy:
228 proxy = self._fix_proxy_url(proxy)
229 return proxy
231 def proxy_headers_for(self, proxy_url):
232 """Retrieves the corresponding proxy headers for a given proxy url."""
233 headers = {}
234 username, password = self._get_auth_from_url(proxy_url)
235 if username and password:
236 basic_auth = self._construct_basic_auth(username, password)
237 headers['Proxy-Authorization'] = basic_auth
238 return headers
240 @property
241 def settings(self):
242 return self._proxies_settings
244 def _fix_proxy_url(self, proxy_url):
245 if proxy_url.startswith('http:') or proxy_url.startswith('https:'):
246 return proxy_url
247 elif proxy_url.startswith('//'):
248 return 'http:' + proxy_url
249 else:
250 return 'http://' + proxy_url
252 def _construct_basic_auth(self, username, password):
253 auth_str = f'{username}:{password}'
254 encoded_str = b64encode(auth_str.encode('ascii')).strip().decode()
255 return f'Basic {encoded_str}'
257 def _get_auth_from_url(self, url):
258 parsed_url = urlparse(url)
259 try:
260 return unquote(parsed_url.username), unquote(parsed_url.password)
261 except (AttributeError, TypeError):
262 return None, None
265class URLLib3Session:
266 """A basic HTTP client that supports connection pooling and proxies.
268 This class is inspired by requests.adapters.HTTPAdapter, but has been
269 boiled down to meet the use cases needed by botocore. For the most part
270 this classes matches the functionality of HTTPAdapter in requests v2.7.0
271 (the same as our vendored version). The only major difference of note is
272 that we currently do not support sending chunked requests. While requests
273 v2.7.0 implemented this themselves, later version urllib3 support this
274 directly via a flag to urlopen so enabling it if needed should be trivial.
275 """
277 def __init__(
278 self,
279 verify=True,
280 proxies=None,
281 timeout=None,
282 max_pool_connections=MAX_POOL_CONNECTIONS,
283 socket_options=None,
284 client_cert=None,
285 proxies_config=None,
286 ):
287 self._verify = verify
288 self._proxy_config = ProxyConfiguration(
289 proxies=proxies, proxies_settings=proxies_config
290 )
291 self._pool_classes_by_scheme = {
292 'http': botocore.awsrequest.AWSHTTPConnectionPool,
293 'https': botocore.awsrequest.AWSHTTPSConnectionPool,
294 }
295 if timeout is None:
296 timeout = DEFAULT_TIMEOUT
297 if not isinstance(timeout, (int, float)):
298 timeout = Timeout(connect=timeout[0], read=timeout[1])
300 self._cert_file = None
301 self._key_file = None
302 if isinstance(client_cert, str):
303 self._cert_file = client_cert
304 elif isinstance(client_cert, tuple):
305 self._cert_file, self._key_file = client_cert
307 self._timeout = timeout
308 self._max_pool_connections = max_pool_connections
309 self._socket_options = socket_options
310 if socket_options is None:
311 self._socket_options = []
312 self._proxy_managers = {}
313 self._manager = PoolManager(**self._get_pool_manager_kwargs())
314 self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme
316 def _proxies_kwargs(self, **kwargs):
317 proxies_settings = self._proxy_config.settings
318 proxies_kwargs = {
319 'use_forwarding_for_https': proxies_settings.get(
320 'proxy_use_forwarding_for_https'
321 ),
322 **kwargs,
323 }
324 return {k: v for k, v in proxies_kwargs.items() if v is not None}
326 def _get_pool_manager_kwargs(self, **extra_kwargs):
327 pool_manager_kwargs = {
328 'strict': True,
329 'timeout': self._timeout,
330 'maxsize': self._max_pool_connections,
331 'ssl_context': self._get_ssl_context(),
332 'socket_options': self._socket_options,
333 'cert_file': self._cert_file,
334 'key_file': self._key_file,
335 }
336 pool_manager_kwargs.update(**extra_kwargs)
337 return pool_manager_kwargs
339 def _get_ssl_context(self):
340 return create_urllib3_context()
342 def _get_proxy_manager(self, proxy_url):
343 if proxy_url not in self._proxy_managers:
344 proxy_headers = self._proxy_config.proxy_headers_for(proxy_url)
345 proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url)
346 proxy_manager_kwargs = self._get_pool_manager_kwargs(
347 proxy_headers=proxy_headers
348 )
349 proxy_manager_kwargs.update(
350 self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context)
351 )
352 proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs)
353 proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme
354 self._proxy_managers[proxy_url] = proxy_manager
356 return self._proxy_managers[proxy_url]
358 def _path_url(self, url):
359 parsed_url = urlparse(url)
360 path = parsed_url.path
361 if not path:
362 path = '/'
363 if parsed_url.query:
364 path = path + '?' + parsed_url.query
365 return path
367 def _setup_ssl_cert(self, conn, url, verify):
368 if url.lower().startswith('https') and verify:
369 conn.cert_reqs = 'CERT_REQUIRED'
370 conn.ca_certs = get_cert_path(verify)
371 else:
372 conn.cert_reqs = 'CERT_NONE'
373 conn.ca_certs = None
375 def _setup_proxy_ssl_context(self, proxy_url):
376 proxies_settings = self._proxy_config.settings
377 proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle')
378 proxy_cert = proxies_settings.get('proxy_client_cert')
379 if proxy_ca_bundle is None and proxy_cert is None:
380 return None
382 context = self._get_ssl_context()
383 try:
384 url = parse_url(proxy_url)
385 # urllib3 disables this by default but we need it for proper
386 # proxy tls negotiation when proxy_url is not an IP Address
387 if not _is_ipaddress(url.host):
388 context.check_hostname = True
389 if proxy_ca_bundle is not None:
390 context.load_verify_locations(cafile=proxy_ca_bundle)
392 if isinstance(proxy_cert, tuple):
393 context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1])
394 elif isinstance(proxy_cert, str):
395 context.load_cert_chain(proxy_cert)
397 return context
398 except (OSError, URLLib3SSLError, LocationParseError) as e:
399 raise InvalidProxiesConfigError(error=e)
401 def _get_connection_manager(self, url, proxy_url=None):
402 if proxy_url:
403 manager = self._get_proxy_manager(proxy_url)
404 else:
405 manager = self._manager
406 return manager
408 def _get_request_target(self, url, proxy_url):
409 has_proxy = proxy_url is not None
411 if not has_proxy:
412 return self._path_url(url)
414 # HTTP proxies expect the request_target to be the absolute url to know
415 # which host to establish a connection to. urllib3 also supports
416 # forwarding for HTTPS through the 'use_forwarding_for_https' parameter.
417 proxy_scheme = urlparse(proxy_url).scheme
418 using_https_forwarding_proxy = (
419 proxy_scheme == 'https'
420 and self._proxies_kwargs().get('use_forwarding_for_https', False)
421 )
423 if using_https_forwarding_proxy or url.startswith('http:'):
424 return url
425 else:
426 return self._path_url(url)
428 def _chunked(self, headers):
429 transfer_encoding = headers.get('Transfer-Encoding', b'')
430 transfer_encoding = ensure_bytes(transfer_encoding)
431 return transfer_encoding.lower() == b'chunked'
433 def close(self):
434 self._manager.clear()
435 for manager in self._proxy_managers.values():
436 manager.clear()
438 def send(self, request):
439 try:
440 proxy_url = self._proxy_config.proxy_url_for(request.url)
441 manager = self._get_connection_manager(request.url, proxy_url)
442 conn = manager.connection_from_url(request.url)
443 self._setup_ssl_cert(conn, request.url, self._verify)
444 if ensure_boolean(
445 os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')
446 ):
447 # This is currently an "experimental" feature which provides
448 # no guarantees of backwards compatibility. It may be subject
449 # to change or removal in any patch version. Anyone opting in
450 # to this feature should strictly pin botocore.
451 host = urlparse(request.url).hostname
452 conn.proxy_headers['host'] = host
454 request_target = self._get_request_target(request.url, proxy_url)
455 urllib_response = conn.urlopen(
456 method=request.method,
457 url=request_target,
458 body=request.body,
459 headers=request.headers,
460 retries=Retry(False),
461 assert_same_host=False,
462 preload_content=False,
463 decode_content=False,
464 chunked=self._chunked(request.headers),
465 )
467 http_response = botocore.awsrequest.AWSResponse(
468 request.url,
469 urllib_response.status,
470 urllib_response.headers,
471 urllib_response,
472 )
474 if not request.stream_output:
475 # Cause the raw stream to be exhausted immediately. We do it
476 # this way instead of using preload_content because
477 # preload_content will never buffer chunked responses
478 http_response.content
480 return http_response
481 except URLLib3SSLError as e:
482 raise SSLError(endpoint_url=request.url, error=e)
483 except (NewConnectionError, socket.gaierror) as e:
484 raise EndpointConnectionError(endpoint_url=request.url, error=e)
485 except ProxyError as e:
486 raise ProxyConnectionError(
487 proxy_url=mask_proxy_url(proxy_url), error=e
488 )
489 except URLLib3ConnectTimeoutError as e:
490 raise ConnectTimeoutError(endpoint_url=request.url, error=e)
491 except URLLib3ReadTimeoutError as e:
492 raise ReadTimeoutError(endpoint_url=request.url, error=e)
493 except ProtocolError as e:
494 raise ConnectionClosedError(
495 error=e, request=request, endpoint_url=request.url
496 )
497 except Exception as e:
498 message = 'Exception received when sending urllib3 HTTP request'
499 logger.debug(message, exc_info=True)
500 raise HTTPClientError(error=e)