Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/botocore/httpsession.py: 26%

1import logging 

2import os 

3import os.path 

4import socket 

5import sys 

6import warnings 

7from base64 import b64encode 

8from concurrent.futures import CancelledError 

9 

10from urllib3 import PoolManager, Timeout, proxy_from_url 

11from urllib3.exceptions import ( 

12 ConnectTimeoutError as URLLib3ConnectTimeoutError, 

13) 

14from urllib3.exceptions import ( 

15 LocationParseError, 

16 NewConnectionError, 

17 ProtocolError, 

18 ProxyError, 

19) 

20from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError 

21from urllib3.exceptions import SSLError as URLLib3SSLError 

22from urllib3.poolmanager import PoolKey 

23from urllib3.util.retry import Retry 

24from urllib3.util.ssl_ import ( 

25 OP_NO_COMPRESSION, 

26 PROTOCOL_TLS, 

27 OP_NO_SSLv2, 

28 OP_NO_SSLv3, 

29 is_ipaddress, 

30 ssl, 

31) 

32from urllib3.util.url import parse_url 

33 

34try: 

35 from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

36except ImportError: 

37 # Fallback directly to ssl for version of urllib3 before 1.26. 

38 # They are available in the standard library starting in Python 3.6. 

39 from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

40 

41try: 

42 # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point. 

43 # This can be removed once our urllib3 floor is raised to >= 2.0. 

44 with warnings.catch_warnings(): 

45 warnings.simplefilter("ignore", category=DeprecationWarning) 

46 # Always import the original SSLContext, even if it has been patched 

47 from urllib3.contrib.pyopenssl import ( 

48 orig_util_SSLContext as SSLContext, 

49 ) 

50except (AttributeError, ImportError): 

51 from urllib3.util.ssl_ import SSLContext 

52 

53try: 

54 from urllib3.util.ssl_ import DEFAULT_CIPHERS 

55except ImportError: 

56 # Defer to system configuration starting with 

57 # urllib3 2.0. This will choose the ciphers provided by 

58 # Openssl 1.1.1+ or secure system defaults. 

59 DEFAULT_CIPHERS = None 

60 

61import botocore.awsrequest 

62from botocore.compat import ( 

63 IPV6_ADDRZ_RE, 

64 ensure_bytes, 

65 filter_ssl_warnings, 

66 unquote, 

67 urlparse, 

68) 

69from botocore.exceptions import ( 

70 ConnectionClosedError, 

71 ConnectTimeoutError, 

72 EndpointConnectionError, 

73 HTTPClientError, 

74 InvalidProxiesConfigError, 

75 ProxyConnectionError, 

76 ReadTimeoutError, 

77 SSLError, 

78) 

79 

80filter_ssl_warnings() 

81logger = logging.getLogger(__name__) 

82DEFAULT_TIMEOUT = 60 

83MAX_POOL_CONNECTIONS = 10 

84DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem') 

85BUFFER_SIZE = None 

86if hasattr(PoolKey, 'key_blocksize'): 

87 # urllib3 2.0 implemented its own chunking logic and set 

88 # a default blocksize of 16KB. This creates a noticeable 

89 # performance bottleneck when transferring objects 

90 # larger than 100MB. Based on experiments, a blocksize 

91 # of 128KB significantly improves throughput before 

92 # getting diminishing returns. 

93 BUFFER_SIZE = 1024 * 128 

94 

95try: 

96 from certifi import where 

97except ImportError: 

98 

99 def where(): 

100 return DEFAULT_CA_BUNDLE 

101 

102 

103def get_cert_path(verify): 

104 if verify is not True: 

105 return verify 

106 

107 cert_path = where() 

108 logger.debug("Certificate path: %s", cert_path) 

109 

110 return cert_path 

111 

112 

113def create_urllib3_context( 

114 ssl_version=None, cert_reqs=None, options=None, ciphers=None 

115): 

116 """This function is a vendored version of the same function in urllib3 

117 

118 We vendor this function to ensure that the SSL contexts we construct 

119 always use the std lib SSLContext instead of pyopenssl. 

120 """ 

121 # PROTOCOL_TLS is deprecated in Python 3.10 

122 if not ssl_version or ssl_version == PROTOCOL_TLS: 

123 ssl_version = PROTOCOL_TLS_CLIENT 

124 

125 context = SSLContext(ssl_version) 

126 

127 if ciphers: 

128 context.set_ciphers(ciphers) 

129 elif DEFAULT_CIPHERS: 

130 context.set_ciphers(DEFAULT_CIPHERS) 

131 

132 # Setting the default here, as we may have no ssl module on import 

133 cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs 

134 

135 if options is None: 

136 options = 0 

137 # SSLv2 is easily broken and is considered harmful and dangerous 

138 options |= OP_NO_SSLv2 

139 # SSLv3 has several problems and is now dangerous 

140 options |= OP_NO_SSLv3 

141 # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ 

142 # (issue urllib3#309) 

143 options |= OP_NO_COMPRESSION 

144 # TLSv1.2 only. Unless set explicitly, do not request tickets. 

145 # This may save some bandwidth on wire, and although the ticket is encrypted, 

146 # there is a risk associated with it being on wire, 

147 # if the server is not rotating its ticketing keys properly. 

148 options |= OP_NO_TICKET 

149 

150 context.options |= options 

151 

152 # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is 

153 # necessary for conditional client cert authentication with TLS 1.3. 

154 # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older 

155 # versions of Python. We only enable on Python 3.7.4+ or if certificate 

156 # verification is enabled to work around Python issue #37428 

157 # See: https://bugs.python.org/issue37428 

158 if ( 

159 cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4) 

160 ) and getattr(context, "post_handshake_auth", None) is not None: 

161 context.post_handshake_auth = True 

162 

163 def disable_check_hostname(): 

164 if ( 

165 getattr(context, "check_hostname", None) is not None 

166 ): # Platform-specific: Python 3.2 

167 # We do our own verification, including fingerprints and alternative 

168 # hostnames. So disable it here 

169 context.check_hostname = False 

170 

171 # The order of the below lines setting verify_mode and check_hostname 

172 # matter due to safe-guards SSLContext has to prevent an SSLContext with 

173 # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more 

174 # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used 

175 # or not so we don't know the initial state of the freshly created SSLContext. 

176 if cert_reqs == ssl.CERT_REQUIRED: 

177 context.verify_mode = cert_reqs 

178 disable_check_hostname() 

179 else: 

180 disable_check_hostname() 

181 context.verify_mode = cert_reqs 

182 

183 # Enable logging of TLS session keys via defacto standard environment variable 

184 # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. 

185 if hasattr(context, "keylog_filename"): 

186 sslkeylogfile = os.environ.get("SSLKEYLOGFILE") 

187 if sslkeylogfile and not sys.flags.ignore_environment: 

188 context.keylog_filename = sslkeylogfile 

189 

190 return context 

191 

192 

193def ensure_boolean(val): 

194 """Ensures a boolean value if a string or boolean is provided 

195 

196 For strings, the value for True/False is case insensitive 

197 """ 

198 if isinstance(val, bool): 

199 return val 

200 else: 

201 return val.lower() == 'true' 

202 

203 

204def mask_proxy_url(proxy_url): 

205 """ 

206 Mask proxy url credentials. 

207 

208 :type proxy_url: str 

209 :param proxy_url: The proxy url, i.e. https://username:password@proxy.com 

210 

211 :return: Masked proxy url, i.e. https://***:***@proxy.com 

212 """ 

213 mask = '*' * 3 

214 parsed_url = urlparse(proxy_url) 

215 if parsed_url.username: 

216 proxy_url = proxy_url.replace(parsed_url.username, mask, 1) 

217 if parsed_url.password: 

218 proxy_url = proxy_url.replace(parsed_url.password, mask, 1) 

219 return proxy_url 

220 

221 

222def _is_ipaddress(host): 

223 """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses.""" 

224 return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host)) 

225 

226 

227class ProxyConfiguration: 

228 """Represents a proxy configuration dictionary and additional settings. 

229 

230 This class represents a proxy configuration dictionary and provides utility 

231 functions to retrieve well structured proxy urls and proxy headers from the 

232 proxy configuration dictionary. 

233 """ 

234 

235 def __init__(self, proxies=None, proxies_settings=None): 

236 if proxies is None: 

237 proxies = {} 

238 if proxies_settings is None: 

239 proxies_settings = {} 

240 

241 self._proxies = proxies 

242 self._proxies_settings = proxies_settings 

243 

244 def proxy_url_for(self, url): 

245 """Retrieves the corresponding proxy url for a given url.""" 

246 parsed_url = urlparse(url) 

247 proxy = self._proxies.get(parsed_url.scheme) 

248 if proxy: 

249 proxy = self._fix_proxy_url(proxy) 

250 return proxy 

251 

252 def proxy_headers_for(self, proxy_url): 

253 """Retrieves the corresponding proxy headers for a given proxy url.""" 

254 headers = {} 

255 username, password = self._get_auth_from_url(proxy_url) 

256 if username and password: 

257 basic_auth = self._construct_basic_auth(username, password) 

258 headers['Proxy-Authorization'] = basic_auth 

259 return headers 

260 

261 @property 

262 def settings(self): 

263 return self._proxies_settings 

264 

265 def _fix_proxy_url(self, proxy_url): 

266 if proxy_url.startswith('http:') or proxy_url.startswith('https:'): 

267 return proxy_url 

268 elif proxy_url.startswith('//'): 

269 return 'http:' + proxy_url 

270 else: 

271 return 'http://' + proxy_url 

272 

273 def _construct_basic_auth(self, username, password): 

274 auth_str = f'{username}:{password}' 

275 encoded_str = b64encode(auth_str.encode('ascii')).strip().decode() 

276 return f'Basic {encoded_str}' 

277 

278 def _get_auth_from_url(self, url): 

279 parsed_url = urlparse(url) 

280 try: 

281 return unquote(parsed_url.username), unquote(parsed_url.password) 

282 except (AttributeError, TypeError): 

283 return None, None 

284 

285 

286class URLLib3Session: 

287 """A basic HTTP client that supports connection pooling and proxies. 

288 

289 This class is inspired by requests.adapters.HTTPAdapter, but has been 

290 boiled down to meet the use cases needed by botocore. For the most part 

291 this classes matches the functionality of HTTPAdapter in requests v2.7.0 

292 (the same as our vendored version). The only major difference of note is 

293 that we currently do not support sending chunked requests. While requests 

294 v2.7.0 implemented this themselves, later version urllib3 support this 

295 directly via a flag to urlopen so enabling it if needed should be trivial. 

296 """ 

297 

298 def __init__( 

299 self, 

300 verify=True, 

301 proxies=None, 

302 timeout=None, 

303 max_pool_connections=MAX_POOL_CONNECTIONS, 

304 socket_options=None, 

305 client_cert=None, 

306 proxies_config=None, 

307 ): 

308 self._verify = verify 

309 self._proxy_config = ProxyConfiguration( 

310 proxies=proxies, proxies_settings=proxies_config 

311 ) 

312 self._pool_classes_by_scheme = { 

313 'http': botocore.awsrequest.AWSHTTPConnectionPool, 

314 'https': botocore.awsrequest.AWSHTTPSConnectionPool, 

315 } 

316 if timeout is None: 

317 timeout = DEFAULT_TIMEOUT 

318 if not isinstance(timeout, (int, float)): 

319 timeout = Timeout(connect=timeout[0], read=timeout[1]) 

320 

321 self._cert_file = None 

322 self._key_file = None 

323 if isinstance(client_cert, str): 

324 self._cert_file = client_cert 

325 elif isinstance(client_cert, tuple): 

326 self._cert_file, self._key_file = client_cert 

327 

328 self._timeout = timeout 

329 self._max_pool_connections = max_pool_connections 

330 self._socket_options = socket_options 

331 if socket_options is None: 

332 self._socket_options = [] 

333 self._proxy_managers = {} 

334 self._manager = PoolManager(**self._get_pool_manager_kwargs()) 

335 self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

336 

337 def _proxies_kwargs(self, **kwargs): 

338 proxies_settings = self._proxy_config.settings 

339 proxies_kwargs = { 

340 'use_forwarding_for_https': proxies_settings.get( 

341 'proxy_use_forwarding_for_https' 

342 ), 

343 **kwargs, 

344 } 

345 return {k: v for k, v in proxies_kwargs.items() if v is not None} 

346 

347 def _get_pool_manager_kwargs(self, **extra_kwargs): 

348 pool_manager_kwargs = { 

349 'timeout': self._timeout, 

350 'maxsize': self._max_pool_connections, 

351 'ssl_context': self._get_ssl_context(), 

352 'socket_options': self._socket_options, 

353 'cert_file': self._cert_file, 

354 'key_file': self._key_file, 

355 } 

356 if BUFFER_SIZE: 

357 pool_manager_kwargs['blocksize'] = BUFFER_SIZE 

358 pool_manager_kwargs.update(**extra_kwargs) 

359 return pool_manager_kwargs 

360 

361 def _get_ssl_context(self): 

362 return create_urllib3_context() 

363 

364 def _get_proxy_manager(self, proxy_url): 

365 if proxy_url not in self._proxy_managers: 

366 proxy_headers = self._proxy_config.proxy_headers_for(proxy_url) 

367 proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url) 

368 proxy_manager_kwargs = self._get_pool_manager_kwargs( 

369 proxy_headers=proxy_headers 

370 ) 

371 proxy_manager_kwargs.update( 

372 self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context) 

373 ) 

374 proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs) 

375 proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

376 self._proxy_managers[proxy_url] = proxy_manager 

377 

378 return self._proxy_managers[proxy_url] 

379 

380 def _path_url(self, url): 

381 parsed_url = urlparse(url) 

382 path = parsed_url.path 

383 if not path: 

384 path = '/' 

385 if parsed_url.query: 

386 path = path + '?' + parsed_url.query 

387 return path 

388 

389 def _setup_ssl_cert(self, conn, url, verify): 

390 if url.lower().startswith('https') and verify: 

391 conn.cert_reqs = 'CERT_REQUIRED' 

392 conn.ca_certs = get_cert_path(verify) 

393 else: 

394 conn.cert_reqs = 'CERT_NONE' 

395 conn.ca_certs = None 

396 

397 def _setup_proxy_ssl_context(self, proxy_url): 

398 proxies_settings = self._proxy_config.settings 

399 proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') 

400 proxy_cert = proxies_settings.get('proxy_client_cert') 

401 if proxy_ca_bundle is None and proxy_cert is None: 

402 return None 

403 

404 context = self._get_ssl_context() 

405 try: 

406 url = parse_url(proxy_url) 

407 # urllib3 disables this by default but we need it for proper 

408 # proxy tls negotiation when proxy_url is not an IP Address 

409 if not _is_ipaddress(url.host): 

410 context.check_hostname = True 

411 if proxy_ca_bundle is not None: 

412 context.load_verify_locations(cafile=proxy_ca_bundle) 

413 

414 if isinstance(proxy_cert, tuple): 

415 context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1]) 

416 elif isinstance(proxy_cert, str): 

417 context.load_cert_chain(proxy_cert) 

418 

419 return context 

420 except (OSError, URLLib3SSLError, LocationParseError) as e: 

421 raise InvalidProxiesConfigError(error=e) 

422 

423 def _get_connection_manager(self, url, proxy_url=None): 

424 if proxy_url: 

425 manager = self._get_proxy_manager(proxy_url) 

426 else: 

427 manager = self._manager 

428 return manager 

429 

430 def _get_request_target(self, url, proxy_url): 

431 has_proxy = proxy_url is not None 

432 

433 if not has_proxy: 

434 return self._path_url(url) 

435 

436 # HTTP proxies expect the request_target to be the absolute url to know 

437 # which host to establish a connection to. urllib3 also supports 

438 # forwarding for HTTPS through the 'use_forwarding_for_https' parameter. 

439 proxy_scheme = urlparse(proxy_url).scheme 

440 using_https_forwarding_proxy = ( 

441 proxy_scheme == 'https' 

442 and self._proxies_kwargs().get('use_forwarding_for_https', False) 

443 ) 

444 

445 if using_https_forwarding_proxy or url.startswith('http:'): 

446 return url 

447 else: 

448 return self._path_url(url) 

449 

450 def _chunked(self, headers): 

451 transfer_encoding = headers.get('Transfer-Encoding', b'') 

452 transfer_encoding = ensure_bytes(transfer_encoding) 

453 return transfer_encoding.lower() == b'chunked' 

454 

455 def close(self): 

456 self._manager.clear() 

457 for manager in self._proxy_managers.values(): 

458 manager.clear() 

459 

460 def send(self, request): 

461 try: 

462 proxy_url = self._proxy_config.proxy_url_for(request.url) 

463 manager = self._get_connection_manager(request.url, proxy_url) 

464 conn = manager.connection_from_url(request.url) 

465 self._setup_ssl_cert(conn, request.url, self._verify) 

466 if ensure_boolean( 

467 os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '') 

468 ): 

469 # This is currently an "experimental" feature which provides 

470 # no guarantees of backwards compatibility. It may be subject 

471 # to change or removal in any patch version. Anyone opting in 

472 # to this feature should strictly pin botocore. 

473 host = urlparse(request.url).hostname 

474 conn.proxy_headers['host'] = host 

475 

476 request_target = self._get_request_target(request.url, proxy_url) 

477 urllib_response = conn.urlopen( 

478 method=request.method, 

479 url=request_target, 

480 body=request.body, 

481 headers=request.headers, 

482 retries=Retry(False), 

483 assert_same_host=False, 

484 preload_content=False, 

485 decode_content=False, 

486 chunked=self._chunked(request.headers), 

487 ) 

488 

489 http_response = botocore.awsrequest.AWSResponse( 

490 request.url, 

491 urllib_response.status, 

492 urllib_response.headers, 

493 urllib_response, 

494 ) 

495 

496 if not request.stream_output: 

497 # Cause the raw stream to be exhausted immediately. We do it 

498 # this way instead of using preload_content because 

499 # preload_content will never buffer chunked responses 

500 http_response.content 

501 

502 return http_response 

503 except URLLib3SSLError as e: 

504 raise SSLError(endpoint_url=request.url, error=e) 

505 except (NewConnectionError, socket.gaierror) as e: 

506 raise EndpointConnectionError(endpoint_url=request.url, error=e) 

507 except ProxyError as e: 

508 raise ProxyConnectionError( 

509 proxy_url=mask_proxy_url(proxy_url), error=e 

510 ) 

511 except URLLib3ConnectTimeoutError as e: 

512 raise ConnectTimeoutError(endpoint_url=request.url, error=e) 

513 except URLLib3ReadTimeoutError as e: 

514 raise ReadTimeoutError(endpoint_url=request.url, error=e) 

515 except ProtocolError as e: 

516 raise ConnectionClosedError( 

517 error=e, request=request, endpoint_url=request.url 

518 ) 

519 except CancelledError: 

520 raise 

521 except Exception as e: 

522 message = 'Exception received when sending urllib3 HTTP request' 

523 logger.debug(message, exc_info=True) 

524 raise HTTPClientError(error=e)