Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/botocore/httpsession.py: 26%

1import logging 

2import os 

3import os.path 

4import socket 

5import sys 

6import warnings 

7from base64 import b64encode 

8from concurrent.futures import CancelledError 

9 

10from urllib3 import PoolManager, Timeout, proxy_from_url 

11from urllib3.exceptions import ( 

12 ConnectTimeoutError as URLLib3ConnectTimeoutError, 

13) 

14from urllib3.exceptions import ( 

15 LocationParseError, 

16 NewConnectionError, 

17 ProtocolError, 

18 ProxyError, 

19) 

20from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError 

21from urllib3.exceptions import SSLError as URLLib3SSLError 

22from urllib3.util.retry import Retry 

23from urllib3.util.ssl_ import ( 

24 OP_NO_COMPRESSION, 

25 PROTOCOL_TLS, 

26 OP_NO_SSLv2, 

27 OP_NO_SSLv3, 

28 is_ipaddress, 

29 ssl, 

30) 

31from urllib3.util.url import parse_url 

32 

33try: 

34 from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

35except ImportError: 

36 # Fallback directly to ssl for version of urllib3 before 1.26. 

37 # They are available in the standard library starting in Python 3.6. 

38 from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

39 

40try: 

41 # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point. 

42 # This can be removed once our urllib3 floor is raised to >= 2.0. 

43 with warnings.catch_warnings(): 

44 warnings.simplefilter("ignore", category=DeprecationWarning) 

45 # Always import the original SSLContext, even if it has been patched 

46 from urllib3.contrib.pyopenssl import ( 

47 orig_util_SSLContext as SSLContext, 

48 ) 

49except (AttributeError, ImportError): 

50 from urllib3.util.ssl_ import SSLContext 

51 

52try: 

53 from urllib3.util.ssl_ import DEFAULT_CIPHERS 

54except ImportError: 

55 # Defer to system configuration starting with 

56 # urllib3 2.0. This will choose the ciphers provided by 

57 # Openssl 1.1.1+ or secure system defaults. 

58 DEFAULT_CIPHERS = None 

59 

60import botocore.awsrequest 

61from botocore.compat import ( 

62 IPV6_ADDRZ_RE, 

63 ensure_bytes, 

64 filter_ssl_warnings, 

65 unquote, 

66 urlparse, 

67) 

68from botocore.exceptions import ( 

69 ConnectionClosedError, 

70 ConnectTimeoutError, 

71 EndpointConnectionError, 

72 HTTPClientError, 

73 InvalidProxiesConfigError, 

74 ProxyConnectionError, 

75 ReadTimeoutError, 

76 SSLError, 

77) 

78 

79filter_ssl_warnings() 

80logger = logging.getLogger(__name__) 

81DEFAULT_TIMEOUT = 60 

82MAX_POOL_CONNECTIONS = 10 

83DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem') 

84 

85try: 

86 from certifi import where 

87except ImportError: 

88 

89 def where(): 

90 return DEFAULT_CA_BUNDLE 

91 

92 

93def get_cert_path(verify): 

94 if verify is not True: 

95 return verify 

96 

97 cert_path = where() 

98 logger.debug("Certificate path: %s", cert_path) 

99 

100 return cert_path 

101 

102 

103def create_urllib3_context( 

104 ssl_version=None, cert_reqs=None, options=None, ciphers=None 

105): 

106 """This function is a vendored version of the same function in urllib3 

107 

108 We vendor this function to ensure that the SSL contexts we construct 

109 always use the std lib SSLContext instead of pyopenssl. 

110 """ 

111 # PROTOCOL_TLS is deprecated in Python 3.10 

112 if not ssl_version or ssl_version == PROTOCOL_TLS: 

113 ssl_version = PROTOCOL_TLS_CLIENT 

114 

115 context = SSLContext(ssl_version) 

116 

117 if ciphers: 

118 context.set_ciphers(ciphers) 

119 elif DEFAULT_CIPHERS: 

120 context.set_ciphers(DEFAULT_CIPHERS) 

121 

122 # Setting the default here, as we may have no ssl module on import 

123 cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs 

124 

125 if options is None: 

126 options = 0 

127 # SSLv2 is easily broken and is considered harmful and dangerous 

128 options |= OP_NO_SSLv2 

129 # SSLv3 has several problems and is now dangerous 

130 options |= OP_NO_SSLv3 

131 # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ 

132 # (issue urllib3#309) 

133 options |= OP_NO_COMPRESSION 

134 # TLSv1.2 only. Unless set explicitly, do not request tickets. 

135 # This may save some bandwidth on wire, and although the ticket is encrypted, 

136 # there is a risk associated with it being on wire, 

137 # if the server is not rotating its ticketing keys properly. 

138 options |= OP_NO_TICKET 

139 

140 context.options |= options 

141 

142 # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is 

143 # necessary for conditional client cert authentication with TLS 1.3. 

144 # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older 

145 # versions of Python. We only enable on Python 3.7.4+ or if certificate 

146 # verification is enabled to work around Python issue #37428 

147 # See: https://bugs.python.org/issue37428 

148 if ( 

149 cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4) 

150 ) and getattr(context, "post_handshake_auth", None) is not None: 

151 context.post_handshake_auth = True 

152 

153 def disable_check_hostname(): 

154 if ( 

155 getattr(context, "check_hostname", None) is not None 

156 ): # Platform-specific: Python 3.2 

157 # We do our own verification, including fingerprints and alternative 

158 # hostnames. So disable it here 

159 context.check_hostname = False 

160 

161 # The order of the below lines setting verify_mode and check_hostname 

162 # matter due to safe-guards SSLContext has to prevent an SSLContext with 

163 # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more 

164 # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used 

165 # or not so we don't know the initial state of the freshly created SSLContext. 

166 if cert_reqs == ssl.CERT_REQUIRED: 

167 context.verify_mode = cert_reqs 

168 disable_check_hostname() 

169 else: 

170 disable_check_hostname() 

171 context.verify_mode = cert_reqs 

172 

173 # Enable logging of TLS session keys via defacto standard environment variable 

174 # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. 

175 if hasattr(context, "keylog_filename"): 

176 sslkeylogfile = os.environ.get("SSLKEYLOGFILE") 

177 if sslkeylogfile and not sys.flags.ignore_environment: 

178 context.keylog_filename = sslkeylogfile 

179 

180 return context 

181 

182 

183def ensure_boolean(val): 

184 """Ensures a boolean value if a string or boolean is provided 

185 

186 For strings, the value for True/False is case insensitive 

187 """ 

188 if isinstance(val, bool): 

189 return val 

190 else: 

191 return val.lower() == 'true' 

192 

193 

194def mask_proxy_url(proxy_url): 

195 """ 

196 Mask proxy url credentials. 

197 

198 :type proxy_url: str 

199 :param proxy_url: The proxy url, i.e. https://username:password@proxy.com 

200 

201 :return: Masked proxy url, i.e. https://***:***@proxy.com 

202 """ 

203 mask = '*' * 3 

204 parsed_url = urlparse(proxy_url) 

205 if parsed_url.username: 

206 proxy_url = proxy_url.replace(parsed_url.username, mask, 1) 

207 if parsed_url.password: 

208 proxy_url = proxy_url.replace(parsed_url.password, mask, 1) 

209 return proxy_url 

210 

211 

212def _is_ipaddress(host): 

213 """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses.""" 

214 return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host)) 

215 

216 

217class ProxyConfiguration: 

218 """Represents a proxy configuration dictionary and additional settings. 

219 

220 This class represents a proxy configuration dictionary and provides utility 

221 functions to retrieve well structured proxy urls and proxy headers from the 

222 proxy configuration dictionary. 

223 """ 

224 

225 def __init__(self, proxies=None, proxies_settings=None): 

226 if proxies is None: 

227 proxies = {} 

228 if proxies_settings is None: 

229 proxies_settings = {} 

230 

231 self._proxies = proxies 

232 self._proxies_settings = proxies_settings 

233 

234 def proxy_url_for(self, url): 

235 """Retrieves the corresponding proxy url for a given url.""" 

236 parsed_url = urlparse(url) 

237 proxy = self._proxies.get(parsed_url.scheme) 

238 if proxy: 

239 proxy = self._fix_proxy_url(proxy) 

240 return proxy 

241 

242 def proxy_headers_for(self, proxy_url): 

243 """Retrieves the corresponding proxy headers for a given proxy url.""" 

244 headers = {} 

245 username, password = self._get_auth_from_url(proxy_url) 

246 if username and password: 

247 basic_auth = self._construct_basic_auth(username, password) 

248 headers['Proxy-Authorization'] = basic_auth 

249 return headers 

250 

251 @property 

252 def settings(self): 

253 return self._proxies_settings 

254 

255 def _fix_proxy_url(self, proxy_url): 

256 if proxy_url.startswith('http:') or proxy_url.startswith('https:'): 

257 return proxy_url 

258 elif proxy_url.startswith('//'): 

259 return 'http:' + proxy_url 

260 else: 

261 return 'http://' + proxy_url 

262 

263 def _construct_basic_auth(self, username, password): 

264 auth_str = f'{username}:{password}' 

265 encoded_str = b64encode(auth_str.encode('ascii')).strip().decode() 

266 return f'Basic {encoded_str}' 

267 

268 def _get_auth_from_url(self, url): 

269 parsed_url = urlparse(url) 

270 try: 

271 return unquote(parsed_url.username), unquote(parsed_url.password) 

272 except (AttributeError, TypeError): 

273 return None, None 

274 

275 

276class URLLib3Session: 

277 """A basic HTTP client that supports connection pooling and proxies. 

278 

279 This class is inspired by requests.adapters.HTTPAdapter, but has been 

280 boiled down to meet the use cases needed by botocore. For the most part 

281 this classes matches the functionality of HTTPAdapter in requests v2.7.0 

282 (the same as our vendored version). The only major difference of note is 

283 that we currently do not support sending chunked requests. While requests 

284 v2.7.0 implemented this themselves, later version urllib3 support this 

285 directly via a flag to urlopen so enabling it if needed should be trivial. 

286 """ 

287 

288 def __init__( 

289 self, 

290 verify=True, 

291 proxies=None, 

292 timeout=None, 

293 max_pool_connections=MAX_POOL_CONNECTIONS, 

294 socket_options=None, 

295 client_cert=None, 

296 proxies_config=None, 

297 ): 

298 self._verify = verify 

299 self._proxy_config = ProxyConfiguration( 

300 proxies=proxies, proxies_settings=proxies_config 

301 ) 

302 self._pool_classes_by_scheme = { 

303 'http': botocore.awsrequest.AWSHTTPConnectionPool, 

304 'https': botocore.awsrequest.AWSHTTPSConnectionPool, 

305 } 

306 if timeout is None: 

307 timeout = DEFAULT_TIMEOUT 

308 if not isinstance(timeout, (int, float)): 

309 timeout = Timeout(connect=timeout[0], read=timeout[1]) 

310 

311 self._cert_file = None 

312 self._key_file = None 

313 if isinstance(client_cert, str): 

314 self._cert_file = client_cert 

315 elif isinstance(client_cert, tuple): 

316 self._cert_file, self._key_file = client_cert 

317 

318 self._timeout = timeout 

319 self._max_pool_connections = max_pool_connections 

320 self._socket_options = socket_options 

321 if socket_options is None: 

322 self._socket_options = [] 

323 self._proxy_managers = {} 

324 self._manager = PoolManager(**self._get_pool_manager_kwargs()) 

325 self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

326 

327 def _proxies_kwargs(self, **kwargs): 

328 proxies_settings = self._proxy_config.settings 

329 proxies_kwargs = { 

330 'use_forwarding_for_https': proxies_settings.get( 

331 'proxy_use_forwarding_for_https' 

332 ), 

333 **kwargs, 

334 } 

335 return {k: v for k, v in proxies_kwargs.items() if v is not None} 

336 

337 def _get_pool_manager_kwargs(self, **extra_kwargs): 

338 pool_manager_kwargs = { 

339 'timeout': self._timeout, 

340 'maxsize': self._max_pool_connections, 

341 'ssl_context': self._get_ssl_context(), 

342 'socket_options': self._socket_options, 

343 'cert_file': self._cert_file, 

344 'key_file': self._key_file, 

345 } 

346 pool_manager_kwargs.update(**extra_kwargs) 

347 return pool_manager_kwargs 

348 

349 def _get_ssl_context(self): 

350 return create_urllib3_context() 

351 

352 def _get_proxy_manager(self, proxy_url): 

353 if proxy_url not in self._proxy_managers: 

354 proxy_headers = self._proxy_config.proxy_headers_for(proxy_url) 

355 proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url) 

356 proxy_manager_kwargs = self._get_pool_manager_kwargs( 

357 proxy_headers=proxy_headers 

358 ) 

359 proxy_manager_kwargs.update( 

360 self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context) 

361 ) 

362 proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs) 

363 proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

364 self._proxy_managers[proxy_url] = proxy_manager 

365 

366 return self._proxy_managers[proxy_url] 

367 

368 def _path_url(self, url): 

369 parsed_url = urlparse(url) 

370 path = parsed_url.path 

371 if not path: 

372 path = '/' 

373 if parsed_url.query: 

374 path = path + '?' + parsed_url.query 

375 return path 

376 

377 def _setup_ssl_cert(self, conn, url, verify): 

378 if url.lower().startswith('https') and verify: 

379 conn.cert_reqs = 'CERT_REQUIRED' 

380 conn.ca_certs = get_cert_path(verify) 

381 else: 

382 conn.cert_reqs = 'CERT_NONE' 

383 conn.ca_certs = None 

384 

385 def _setup_proxy_ssl_context(self, proxy_url): 

386 proxies_settings = self._proxy_config.settings 

387 proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') 

388 proxy_cert = proxies_settings.get('proxy_client_cert') 

389 if proxy_ca_bundle is None and proxy_cert is None: 

390 return None 

391 

392 context = self._get_ssl_context() 

393 try: 

394 url = parse_url(proxy_url) 

395 # urllib3 disables this by default but we need it for proper 

396 # proxy tls negotiation when proxy_url is not an IP Address 

397 if not _is_ipaddress(url.host): 

398 context.check_hostname = True 

399 if proxy_ca_bundle is not None: 

400 context.load_verify_locations(cafile=proxy_ca_bundle) 

401 

402 if isinstance(proxy_cert, tuple): 

403 context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1]) 

404 elif isinstance(proxy_cert, str): 

405 context.load_cert_chain(proxy_cert) 

406 

407 return context 

408 except (OSError, URLLib3SSLError, LocationParseError) as e: 

409 raise InvalidProxiesConfigError(error=e) 

410 

411 def _get_connection_manager(self, url, proxy_url=None): 

412 if proxy_url: 

413 manager = self._get_proxy_manager(proxy_url) 

414 else: 

415 manager = self._manager 

416 return manager 

417 

418 def _get_request_target(self, url, proxy_url): 

419 has_proxy = proxy_url is not None 

420 

421 if not has_proxy: 

422 return self._path_url(url) 

423 

424 # HTTP proxies expect the request_target to be the absolute url to know 

425 # which host to establish a connection to. urllib3 also supports 

426 # forwarding for HTTPS through the 'use_forwarding_for_https' parameter. 

427 proxy_scheme = urlparse(proxy_url).scheme 

428 using_https_forwarding_proxy = ( 

429 proxy_scheme == 'https' 

430 and self._proxies_kwargs().get('use_forwarding_for_https', False) 

431 ) 

432 

433 if using_https_forwarding_proxy or url.startswith('http:'): 

434 return url 

435 else: 

436 return self._path_url(url) 

437 

438 def _chunked(self, headers): 

439 transfer_encoding = headers.get('Transfer-Encoding', b'') 

440 transfer_encoding = ensure_bytes(transfer_encoding) 

441 return transfer_encoding.lower() == b'chunked' 

442 

443 def close(self): 

444 self._manager.clear() 

445 for manager in self._proxy_managers.values(): 

446 manager.clear() 

447 

448 def send(self, request): 

449 try: 

450 proxy_url = self._proxy_config.proxy_url_for(request.url) 

451 manager = self._get_connection_manager(request.url, proxy_url) 

452 conn = manager.connection_from_url(request.url) 

453 self._setup_ssl_cert(conn, request.url, self._verify) 

454 if ensure_boolean( 

455 os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '') 

456 ): 

457 # This is currently an "experimental" feature which provides 

458 # no guarantees of backwards compatibility. It may be subject 

459 # to change or removal in any patch version. Anyone opting in 

460 # to this feature should strictly pin botocore. 

461 host = urlparse(request.url).hostname 

462 conn.proxy_headers['host'] = host 

463 

464 request_target = self._get_request_target(request.url, proxy_url) 

465 urllib_response = conn.urlopen( 

466 method=request.method, 

467 url=request_target, 

468 body=request.body, 

469 headers=request.headers, 

470 retries=Retry(False), 

471 assert_same_host=False, 

472 preload_content=False, 

473 decode_content=False, 

474 chunked=self._chunked(request.headers), 

475 ) 

476 

477 http_response = botocore.awsrequest.AWSResponse( 

478 request.url, 

479 urllib_response.status, 

480 urllib_response.headers, 

481 urllib_response, 

482 ) 

483 

484 if not request.stream_output: 

485 # Cause the raw stream to be exhausted immediately. We do it 

486 # this way instead of using preload_content because 

487 # preload_content will never buffer chunked responses 

488 http_response.content 

489 

490 return http_response 

491 except URLLib3SSLError as e: 

492 raise SSLError(endpoint_url=request.url, error=e) 

493 except (NewConnectionError, socket.gaierror) as e: 

494 raise EndpointConnectionError(endpoint_url=request.url, error=e) 

495 except ProxyError as e: 

496 raise ProxyConnectionError( 

497 proxy_url=mask_proxy_url(proxy_url), error=e 

498 ) 

499 except URLLib3ConnectTimeoutError as e: 

500 raise ConnectTimeoutError(endpoint_url=request.url, error=e) 

501 except URLLib3ReadTimeoutError as e: 

502 raise ReadTimeoutError(endpoint_url=request.url, error=e) 

503 except ProtocolError as e: 

504 raise ConnectionClosedError( 

505 error=e, request=request, endpoint_url=request.url 

506 ) 

507 except CancelledError: 

508 raise 

509 except Exception as e: 

510 message = 'Exception received when sending urllib3 HTTP request' 

511 logger.debug(message, exc_info=True) 

512 raise HTTPClientError(error=e)