Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/httpsession.py: 24%
263 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:51 +0000
1import logging
2import os
3import os.path
4import socket
5import sys
6import warnings
7from base64 import b64encode
9from urllib3 import PoolManager, Timeout, proxy_from_url
10from urllib3.exceptions import (
11 ConnectTimeoutError as URLLib3ConnectTimeoutError,
12)
13from urllib3.exceptions import (
14 LocationParseError,
15 NewConnectionError,
16 ProtocolError,
17 ProxyError,
18)
19from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
20from urllib3.exceptions import SSLError as URLLib3SSLError
21from urllib3.util.retry import Retry
22from urllib3.util.ssl_ import (
23 OP_NO_COMPRESSION,
24 PROTOCOL_TLS,
25 OP_NO_SSLv2,
26 OP_NO_SSLv3,
27 is_ipaddress,
28 ssl,
29)
30from urllib3.util.url import parse_url
32try:
33 from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT
34except ImportError:
35 # Fallback directly to ssl for version of urllib3 before 1.26.
36 # They are available in the standard library starting in Python 3.6.
37 from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT
39try:
40 # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point.
41 # This can be removed once our urllib3 floor is raised to >= 2.0.
42 with warnings.catch_warnings():
43 warnings.simplefilter("ignore", category=DeprecationWarning)
44 # Always import the original SSLContext, even if it has been patched
45 from urllib3.contrib.pyopenssl import (
46 orig_util_SSLContext as SSLContext,
47 )
48except ImportError:
49 from urllib3.util.ssl_ import SSLContext
51try:
52 from urllib3.util.ssl_ import DEFAULT_CIPHERS
53except ImportError:
54 # Defer to system configuration starting with
55 # urllib3 2.0. This will choose the ciphers provided by
56 # Openssl 1.1.1+ or secure system defaults.
57 DEFAULT_CIPHERS = None
59import botocore.awsrequest
60from botocore.compat import (
61 IPV6_ADDRZ_RE,
62 ensure_bytes,
63 filter_ssl_warnings,
64 unquote,
65 urlparse,
66)
67from botocore.exceptions import (
68 ConnectionClosedError,
69 ConnectTimeoutError,
70 EndpointConnectionError,
71 HTTPClientError,
72 InvalidProxiesConfigError,
73 ProxyConnectionError,
74 ReadTimeoutError,
75 SSLError,
76)
78filter_ssl_warnings()
79logger = logging.getLogger(__name__)
80DEFAULT_TIMEOUT = 60
81MAX_POOL_CONNECTIONS = 10
82DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem')
84try:
85 from certifi import where
86except ImportError:
88 def where():
89 return DEFAULT_CA_BUNDLE
92def get_cert_path(verify):
93 if verify is not True:
94 return verify
96 cert_path = where()
97 logger.debug(f"Certificate path: {cert_path}")
99 return cert_path
102def create_urllib3_context(
103 ssl_version=None, cert_reqs=None, options=None, ciphers=None
104):
105 """This function is a vendored version of the same function in urllib3
107 We vendor this function to ensure that the SSL contexts we construct
108 always use the std lib SSLContext instead of pyopenssl.
109 """
110 # PROTOCOL_TLS is deprecated in Python 3.10
111 if not ssl_version or ssl_version == PROTOCOL_TLS:
112 ssl_version = PROTOCOL_TLS_CLIENT
114 context = SSLContext(ssl_version)
116 if ciphers:
117 context.set_ciphers(ciphers)
118 elif DEFAULT_CIPHERS:
119 context.set_ciphers(DEFAULT_CIPHERS)
121 # Setting the default here, as we may have no ssl module on import
122 cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
124 if options is None:
125 options = 0
126 # SSLv2 is easily broken and is considered harmful and dangerous
127 options |= OP_NO_SSLv2
128 # SSLv3 has several problems and is now dangerous
129 options |= OP_NO_SSLv3
130 # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
131 # (issue urllib3#309)
132 options |= OP_NO_COMPRESSION
133 # TLSv1.2 only. Unless set explicitly, do not request tickets.
134 # This may save some bandwidth on wire, and although the ticket is encrypted,
135 # there is a risk associated with it being on wire,
136 # if the server is not rotating its ticketing keys properly.
137 options |= OP_NO_TICKET
139 context.options |= options
141 # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
142 # necessary for conditional client cert authentication with TLS 1.3.
143 # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
144 # versions of Python. We only enable on Python 3.7.4+ or if certificate
145 # verification is enabled to work around Python issue #37428
146 # See: https://bugs.python.org/issue37428
147 if (
148 cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)
149 ) and getattr(context, "post_handshake_auth", None) is not None:
150 context.post_handshake_auth = True
152 def disable_check_hostname():
153 if (
154 getattr(context, "check_hostname", None) is not None
155 ): # Platform-specific: Python 3.2
156 # We do our own verification, including fingerprints and alternative
157 # hostnames. So disable it here
158 context.check_hostname = False
160 # The order of the below lines setting verify_mode and check_hostname
161 # matter due to safe-guards SSLContext has to prevent an SSLContext with
162 # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more
163 # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used
164 # or not so we don't know the initial state of the freshly created SSLContext.
165 if cert_reqs == ssl.CERT_REQUIRED:
166 context.verify_mode = cert_reqs
167 disable_check_hostname()
168 else:
169 disable_check_hostname()
170 context.verify_mode = cert_reqs
172 # Enable logging of TLS session keys via defacto standard environment variable
173 # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
174 if hasattr(context, "keylog_filename"):
175 sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
176 if sslkeylogfile and not sys.flags.ignore_environment:
177 context.keylog_filename = sslkeylogfile
179 return context
182def ensure_boolean(val):
183 """Ensures a boolean value if a string or boolean is provided
185 For strings, the value for True/False is case insensitive
186 """
187 if isinstance(val, bool):
188 return val
189 else:
190 return val.lower() == 'true'
193def mask_proxy_url(proxy_url):
194 """
195 Mask proxy url credentials.
197 :type proxy_url: str
198 :param proxy_url: The proxy url, i.e. https://username:password@proxy.com
200 :return: Masked proxy url, i.e. https://***:***@proxy.com
201 """
202 mask = '*' * 3
203 parsed_url = urlparse(proxy_url)
204 if parsed_url.username:
205 proxy_url = proxy_url.replace(parsed_url.username, mask, 1)
206 if parsed_url.password:
207 proxy_url = proxy_url.replace(parsed_url.password, mask, 1)
208 return proxy_url
211def _is_ipaddress(host):
212 """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses."""
213 return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host))
216class ProxyConfiguration:
217 """Represents a proxy configuration dictionary and additional settings.
219 This class represents a proxy configuration dictionary and provides utility
220 functions to retrieve well structured proxy urls and proxy headers from the
221 proxy configuration dictionary.
222 """
224 def __init__(self, proxies=None, proxies_settings=None):
225 if proxies is None:
226 proxies = {}
227 if proxies_settings is None:
228 proxies_settings = {}
230 self._proxies = proxies
231 self._proxies_settings = proxies_settings
233 def proxy_url_for(self, url):
234 """Retrieves the corresponding proxy url for a given url."""
235 parsed_url = urlparse(url)
236 proxy = self._proxies.get(parsed_url.scheme)
237 if proxy:
238 proxy = self._fix_proxy_url(proxy)
239 return proxy
241 def proxy_headers_for(self, proxy_url):
242 """Retrieves the corresponding proxy headers for a given proxy url."""
243 headers = {}
244 username, password = self._get_auth_from_url(proxy_url)
245 if username and password:
246 basic_auth = self._construct_basic_auth(username, password)
247 headers['Proxy-Authorization'] = basic_auth
248 return headers
250 @property
251 def settings(self):
252 return self._proxies_settings
254 def _fix_proxy_url(self, proxy_url):
255 if proxy_url.startswith('http:') or proxy_url.startswith('https:'):
256 return proxy_url
257 elif proxy_url.startswith('//'):
258 return 'http:' + proxy_url
259 else:
260 return 'http://' + proxy_url
262 def _construct_basic_auth(self, username, password):
263 auth_str = f'{username}:{password}'
264 encoded_str = b64encode(auth_str.encode('ascii')).strip().decode()
265 return f'Basic {encoded_str}'
267 def _get_auth_from_url(self, url):
268 parsed_url = urlparse(url)
269 try:
270 return unquote(parsed_url.username), unquote(parsed_url.password)
271 except (AttributeError, TypeError):
272 return None, None
275class URLLib3Session:
276 """A basic HTTP client that supports connection pooling and proxies.
278 This class is inspired by requests.adapters.HTTPAdapter, but has been
279 boiled down to meet the use cases needed by botocore. For the most part
280 this classes matches the functionality of HTTPAdapter in requests v2.7.0
281 (the same as our vendored version). The only major difference of note is
282 that we currently do not support sending chunked requests. While requests
283 v2.7.0 implemented this themselves, later version urllib3 support this
284 directly via a flag to urlopen so enabling it if needed should be trivial.
285 """
287 def __init__(
288 self,
289 verify=True,
290 proxies=None,
291 timeout=None,
292 max_pool_connections=MAX_POOL_CONNECTIONS,
293 socket_options=None,
294 client_cert=None,
295 proxies_config=None,
296 ):
297 self._verify = verify
298 self._proxy_config = ProxyConfiguration(
299 proxies=proxies, proxies_settings=proxies_config
300 )
301 self._pool_classes_by_scheme = {
302 'http': botocore.awsrequest.AWSHTTPConnectionPool,
303 'https': botocore.awsrequest.AWSHTTPSConnectionPool,
304 }
305 if timeout is None:
306 timeout = DEFAULT_TIMEOUT
307 if not isinstance(timeout, (int, float)):
308 timeout = Timeout(connect=timeout[0], read=timeout[1])
310 self._cert_file = None
311 self._key_file = None
312 if isinstance(client_cert, str):
313 self._cert_file = client_cert
314 elif isinstance(client_cert, tuple):
315 self._cert_file, self._key_file = client_cert
317 self._timeout = timeout
318 self._max_pool_connections = max_pool_connections
319 self._socket_options = socket_options
320 if socket_options is None:
321 self._socket_options = []
322 self._proxy_managers = {}
323 self._manager = PoolManager(**self._get_pool_manager_kwargs())
324 self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme
326 def _proxies_kwargs(self, **kwargs):
327 proxies_settings = self._proxy_config.settings
328 proxies_kwargs = {
329 'use_forwarding_for_https': proxies_settings.get(
330 'proxy_use_forwarding_for_https'
331 ),
332 **kwargs,
333 }
334 return {k: v for k, v in proxies_kwargs.items() if v is not None}
336 def _get_pool_manager_kwargs(self, **extra_kwargs):
337 pool_manager_kwargs = {
338 'timeout': self._timeout,
339 'maxsize': self._max_pool_connections,
340 'ssl_context': self._get_ssl_context(),
341 'socket_options': self._socket_options,
342 'cert_file': self._cert_file,
343 'key_file': self._key_file,
344 }
345 pool_manager_kwargs.update(**extra_kwargs)
346 return pool_manager_kwargs
348 def _get_ssl_context(self):
349 return create_urllib3_context()
351 def _get_proxy_manager(self, proxy_url):
352 if proxy_url not in self._proxy_managers:
353 proxy_headers = self._proxy_config.proxy_headers_for(proxy_url)
354 proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url)
355 proxy_manager_kwargs = self._get_pool_manager_kwargs(
356 proxy_headers=proxy_headers
357 )
358 proxy_manager_kwargs.update(
359 self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context)
360 )
361 proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs)
362 proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme
363 self._proxy_managers[proxy_url] = proxy_manager
365 return self._proxy_managers[proxy_url]
367 def _path_url(self, url):
368 parsed_url = urlparse(url)
369 path = parsed_url.path
370 if not path:
371 path = '/'
372 if parsed_url.query:
373 path = path + '?' + parsed_url.query
374 return path
376 def _setup_ssl_cert(self, conn, url, verify):
377 if url.lower().startswith('https') and verify:
378 conn.cert_reqs = 'CERT_REQUIRED'
379 conn.ca_certs = get_cert_path(verify)
380 else:
381 conn.cert_reqs = 'CERT_NONE'
382 conn.ca_certs = None
384 def _setup_proxy_ssl_context(self, proxy_url):
385 proxies_settings = self._proxy_config.settings
386 proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle')
387 proxy_cert = proxies_settings.get('proxy_client_cert')
388 if proxy_ca_bundle is None and proxy_cert is None:
389 return None
391 context = self._get_ssl_context()
392 try:
393 url = parse_url(proxy_url)
394 # urllib3 disables this by default but we need it for proper
395 # proxy tls negotiation when proxy_url is not an IP Address
396 if not _is_ipaddress(url.host):
397 context.check_hostname = True
398 if proxy_ca_bundle is not None:
399 context.load_verify_locations(cafile=proxy_ca_bundle)
401 if isinstance(proxy_cert, tuple):
402 context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1])
403 elif isinstance(proxy_cert, str):
404 context.load_cert_chain(proxy_cert)
406 return context
407 except (OSError, URLLib3SSLError, LocationParseError) as e:
408 raise InvalidProxiesConfigError(error=e)
410 def _get_connection_manager(self, url, proxy_url=None):
411 if proxy_url:
412 manager = self._get_proxy_manager(proxy_url)
413 else:
414 manager = self._manager
415 return manager
417 def _get_request_target(self, url, proxy_url):
418 has_proxy = proxy_url is not None
420 if not has_proxy:
421 return self._path_url(url)
423 # HTTP proxies expect the request_target to be the absolute url to know
424 # which host to establish a connection to. urllib3 also supports
425 # forwarding for HTTPS through the 'use_forwarding_for_https' parameter.
426 proxy_scheme = urlparse(proxy_url).scheme
427 using_https_forwarding_proxy = (
428 proxy_scheme == 'https'
429 and self._proxies_kwargs().get('use_forwarding_for_https', False)
430 )
432 if using_https_forwarding_proxy or url.startswith('http:'):
433 return url
434 else:
435 return self._path_url(url)
437 def _chunked(self, headers):
438 transfer_encoding = headers.get('Transfer-Encoding', b'')
439 transfer_encoding = ensure_bytes(transfer_encoding)
440 return transfer_encoding.lower() == b'chunked'
442 def close(self):
443 self._manager.clear()
444 for manager in self._proxy_managers.values():
445 manager.clear()
447 def send(self, request):
448 try:
449 proxy_url = self._proxy_config.proxy_url_for(request.url)
450 manager = self._get_connection_manager(request.url, proxy_url)
451 conn = manager.connection_from_url(request.url)
452 self._setup_ssl_cert(conn, request.url, self._verify)
453 if ensure_boolean(
454 os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')
455 ):
456 # This is currently an "experimental" feature which provides
457 # no guarantees of backwards compatibility. It may be subject
458 # to change or removal in any patch version. Anyone opting in
459 # to this feature should strictly pin botocore.
460 host = urlparse(request.url).hostname
461 conn.proxy_headers['host'] = host
463 request_target = self._get_request_target(request.url, proxy_url)
464 urllib_response = conn.urlopen(
465 method=request.method,
466 url=request_target,
467 body=request.body,
468 headers=request.headers,
469 retries=Retry(False),
470 assert_same_host=False,
471 preload_content=False,
472 decode_content=False,
473 chunked=self._chunked(request.headers),
474 )
476 http_response = botocore.awsrequest.AWSResponse(
477 request.url,
478 urllib_response.status,
479 urllib_response.headers,
480 urllib_response,
481 )
483 if not request.stream_output:
484 # Cause the raw stream to be exhausted immediately. We do it
485 # this way instead of using preload_content because
486 # preload_content will never buffer chunked responses
487 http_response.content
489 return http_response
490 except URLLib3SSLError as e:
491 raise SSLError(endpoint_url=request.url, error=e)
492 except (NewConnectionError, socket.gaierror) as e:
493 raise EndpointConnectionError(endpoint_url=request.url, error=e)
494 except ProxyError as e:
495 raise ProxyConnectionError(
496 proxy_url=mask_proxy_url(proxy_url), error=e
497 )
498 except URLLib3ConnectTimeoutError as e:
499 raise ConnectTimeoutError(endpoint_url=request.url, error=e)
500 except URLLib3ReadTimeoutError as e:
501 raise ReadTimeoutError(endpoint_url=request.url, error=e)
502 except ProtocolError as e:
503 raise ConnectionClosedError(
504 error=e, request=request, endpoint_url=request.url
505 )
506 except Exception as e:
507 message = 'Exception received when sending urllib3 HTTP request'
508 logger.debug(message, exc_info=True)
509 raise HTTPClientError(error=e)