Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/httpsession.py: 25%

1import logging 

2import os 

3import os.path 

4import socket 

5import sys 

6import warnings 

7from base64 import b64encode 

8 

9from urllib3 import PoolManager, Timeout, proxy_from_url 

10from urllib3.exceptions import ( 

11 ConnectTimeoutError as URLLib3ConnectTimeoutError, 

12) 

13from urllib3.exceptions import ( 

14 LocationParseError, 

15 NewConnectionError, 

16 ProtocolError, 

17 ProxyError, 

18) 

19from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError 

20from urllib3.exceptions import SSLError as URLLib3SSLError 

21from urllib3.util.retry import Retry 

22from urllib3.util.ssl_ import ( 

23 OP_NO_COMPRESSION, 

24 PROTOCOL_TLS, 

25 OP_NO_SSLv2, 

26 OP_NO_SSLv3, 

27 is_ipaddress, 

28 ssl, 

29) 

30from urllib3.util.url import parse_url 

31 

32try: 

33 from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

34except ImportError: 

35 # Fallback directly to ssl for version of urllib3 before 1.26. 

36 # They are available in the standard library starting in Python 3.6. 

37 from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT 

38 

39try: 

40 # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point. 

41 # This can be removed once our urllib3 floor is raised to >= 2.0. 

42 with warnings.catch_warnings(): 

43 warnings.simplefilter("ignore", category=DeprecationWarning) 

44 # Always import the original SSLContext, even if it has been patched 

45 from urllib3.contrib.pyopenssl import ( 

46 orig_util_SSLContext as SSLContext, 

47 ) 

48except ImportError: 

49 from urllib3.util.ssl_ import SSLContext 

50 

51try: 

52 from urllib3.util.ssl_ import DEFAULT_CIPHERS 

53except ImportError: 

54 # Defer to system configuration starting with 

55 # urllib3 2.0. This will choose the ciphers provided by 

56 # Openssl 1.1.1+ or secure system defaults. 

57 DEFAULT_CIPHERS = None 

58 

59import botocore.awsrequest 

60from botocore.compat import ( 

61 IPV6_ADDRZ_RE, 

62 ensure_bytes, 

63 filter_ssl_warnings, 

64 unquote, 

65 urlparse, 

66) 

67from botocore.exceptions import ( 

68 ConnectionClosedError, 

69 ConnectTimeoutError, 

70 EndpointConnectionError, 

71 HTTPClientError, 

72 InvalidProxiesConfigError, 

73 ProxyConnectionError, 

74 ReadTimeoutError, 

75 SSLError, 

76) 

77 

78filter_ssl_warnings() 

79logger = logging.getLogger(__name__) 

80DEFAULT_TIMEOUT = 60 

81MAX_POOL_CONNECTIONS = 10 

82DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem') 

83 

84try: 

85 from certifi import where 

86except ImportError: 

87 

88 def where(): 

89 return DEFAULT_CA_BUNDLE 

90 

91 

92def get_cert_path(verify): 

93 if verify is not True: 

94 return verify 

95 

96 cert_path = where() 

97 logger.debug(f"Certificate path: {cert_path}") 

98 

99 return cert_path 

100 

101 

102def create_urllib3_context( 

103 ssl_version=None, cert_reqs=None, options=None, ciphers=None 

104): 

105 """This function is a vendored version of the same function in urllib3 

106 

107 We vendor this function to ensure that the SSL contexts we construct 

108 always use the std lib SSLContext instead of pyopenssl. 

109 """ 

110 # PROTOCOL_TLS is deprecated in Python 3.10 

111 if not ssl_version or ssl_version == PROTOCOL_TLS: 

112 ssl_version = PROTOCOL_TLS_CLIENT 

113 

114 context = SSLContext(ssl_version) 

115 

116 if ciphers: 

117 context.set_ciphers(ciphers) 

118 elif DEFAULT_CIPHERS: 

119 context.set_ciphers(DEFAULT_CIPHERS) 

120 

121 # Setting the default here, as we may have no ssl module on import 

122 cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs 

123 

124 if options is None: 

125 options = 0 

126 # SSLv2 is easily broken and is considered harmful and dangerous 

127 options |= OP_NO_SSLv2 

128 # SSLv3 has several problems and is now dangerous 

129 options |= OP_NO_SSLv3 

130 # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ 

131 # (issue urllib3#309) 

132 options |= OP_NO_COMPRESSION 

133 # TLSv1.2 only. Unless set explicitly, do not request tickets. 

134 # This may save some bandwidth on wire, and although the ticket is encrypted, 

135 # there is a risk associated with it being on wire, 

136 # if the server is not rotating its ticketing keys properly. 

137 options |= OP_NO_TICKET 

138 

139 context.options |= options 

140 

141 # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is 

142 # necessary for conditional client cert authentication with TLS 1.3. 

143 # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older 

144 # versions of Python. We only enable on Python 3.7.4+ or if certificate 

145 # verification is enabled to work around Python issue #37428 

146 # See: https://bugs.python.org/issue37428 

147 if ( 

148 cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4) 

149 ) and getattr(context, "post_handshake_auth", None) is not None: 

150 context.post_handshake_auth = True 

151 

152 def disable_check_hostname(): 

153 if ( 

154 getattr(context, "check_hostname", None) is not None 

155 ): # Platform-specific: Python 3.2 

156 # We do our own verification, including fingerprints and alternative 

157 # hostnames. So disable it here 

158 context.check_hostname = False 

159 

160 # The order of the below lines setting verify_mode and check_hostname 

161 # matter due to safe-guards SSLContext has to prevent an SSLContext with 

162 # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more 

163 # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used 

164 # or not so we don't know the initial state of the freshly created SSLContext. 

165 if cert_reqs == ssl.CERT_REQUIRED: 

166 context.verify_mode = cert_reqs 

167 disable_check_hostname() 

168 else: 

169 disable_check_hostname() 

170 context.verify_mode = cert_reqs 

171 

172 # Enable logging of TLS session keys via defacto standard environment variable 

173 # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. 

174 if hasattr(context, "keylog_filename"): 

175 sslkeylogfile = os.environ.get("SSLKEYLOGFILE") 

176 if sslkeylogfile and not sys.flags.ignore_environment: 

177 context.keylog_filename = sslkeylogfile 

178 

179 return context 

180 

181 

182def ensure_boolean(val): 

183 """Ensures a boolean value if a string or boolean is provided 

184 

185 For strings, the value for True/False is case insensitive 

186 """ 

187 if isinstance(val, bool): 

188 return val 

189 else: 

190 return val.lower() == 'true' 

191 

192 

193def mask_proxy_url(proxy_url): 

194 """ 

195 Mask proxy url credentials. 

196 

197 :type proxy_url: str 

198 :param proxy_url: The proxy url, i.e. https://username:password@proxy.com 

199 

200 :return: Masked proxy url, i.e. https://***:***@proxy.com 

201 """ 

202 mask = '*' * 3 

203 parsed_url = urlparse(proxy_url) 

204 if parsed_url.username: 

205 proxy_url = proxy_url.replace(parsed_url.username, mask, 1) 

206 if parsed_url.password: 

207 proxy_url = proxy_url.replace(parsed_url.password, mask, 1) 

208 return proxy_url 

209 

210 

211def _is_ipaddress(host): 

212 """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses.""" 

213 return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host)) 

214 

215 

216class ProxyConfiguration: 

217 """Represents a proxy configuration dictionary and additional settings. 

218 

219 This class represents a proxy configuration dictionary and provides utility 

220 functions to retrieve well structured proxy urls and proxy headers from the 

221 proxy configuration dictionary. 

222 """ 

223 

224 def __init__(self, proxies=None, proxies_settings=None): 

225 if proxies is None: 

226 proxies = {} 

227 if proxies_settings is None: 

228 proxies_settings = {} 

229 

230 self._proxies = proxies 

231 self._proxies_settings = proxies_settings 

232 

233 def proxy_url_for(self, url): 

234 """Retrieves the corresponding proxy url for a given url.""" 

235 parsed_url = urlparse(url) 

236 proxy = self._proxies.get(parsed_url.scheme) 

237 if proxy: 

238 proxy = self._fix_proxy_url(proxy) 

239 return proxy 

240 

241 def proxy_headers_for(self, proxy_url): 

242 """Retrieves the corresponding proxy headers for a given proxy url.""" 

243 headers = {} 

244 username, password = self._get_auth_from_url(proxy_url) 

245 if username and password: 

246 basic_auth = self._construct_basic_auth(username, password) 

247 headers['Proxy-Authorization'] = basic_auth 

248 return headers 

249 

250 @property 

251 def settings(self): 

252 return self._proxies_settings 

253 

254 def _fix_proxy_url(self, proxy_url): 

255 if proxy_url.startswith('http:') or proxy_url.startswith('https:'): 

256 return proxy_url 

257 elif proxy_url.startswith('//'): 

258 return 'http:' + proxy_url 

259 else: 

260 return 'http://' + proxy_url 

261 

262 def _construct_basic_auth(self, username, password): 

263 auth_str = f'{username}:{password}' 

264 encoded_str = b64encode(auth_str.encode('ascii')).strip().decode() 

265 return f'Basic {encoded_str}' 

266 

267 def _get_auth_from_url(self, url): 

268 parsed_url = urlparse(url) 

269 try: 

270 return unquote(parsed_url.username), unquote(parsed_url.password) 

271 except (AttributeError, TypeError): 

272 return None, None 

273 

274 

275class URLLib3Session: 

276 """A basic HTTP client that supports connection pooling and proxies. 

277 

278 This class is inspired by requests.adapters.HTTPAdapter, but has been 

279 boiled down to meet the use cases needed by botocore. For the most part 

280 this classes matches the functionality of HTTPAdapter in requests v2.7.0 

281 (the same as our vendored version). The only major difference of note is 

282 that we currently do not support sending chunked requests. While requests 

283 v2.7.0 implemented this themselves, later version urllib3 support this 

284 directly via a flag to urlopen so enabling it if needed should be trivial. 

285 """ 

286 

287 def __init__( 

288 self, 

289 verify=True, 

290 proxies=None, 

291 timeout=None, 

292 max_pool_connections=MAX_POOL_CONNECTIONS, 

293 socket_options=None, 

294 client_cert=None, 

295 proxies_config=None, 

296 ): 

297 self._verify = verify 

298 self._proxy_config = ProxyConfiguration( 

299 proxies=proxies, proxies_settings=proxies_config 

300 ) 

301 self._pool_classes_by_scheme = { 

302 'http': botocore.awsrequest.AWSHTTPConnectionPool, 

303 'https': botocore.awsrequest.AWSHTTPSConnectionPool, 

304 } 

305 if timeout is None: 

306 timeout = DEFAULT_TIMEOUT 

307 if not isinstance(timeout, (int, float)): 

308 timeout = Timeout(connect=timeout[0], read=timeout[1]) 

309 

310 self._cert_file = None 

311 self._key_file = None 

312 if isinstance(client_cert, str): 

313 self._cert_file = client_cert 

314 elif isinstance(client_cert, tuple): 

315 self._cert_file, self._key_file = client_cert 

316 

317 self._timeout = timeout 

318 self._max_pool_connections = max_pool_connections 

319 self._socket_options = socket_options 

320 if socket_options is None: 

321 self._socket_options = [] 

322 self._proxy_managers = {} 

323 self._manager = PoolManager(**self._get_pool_manager_kwargs()) 

324 self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

325 

326 def _proxies_kwargs(self, **kwargs): 

327 proxies_settings = self._proxy_config.settings 

328 proxies_kwargs = { 

329 'use_forwarding_for_https': proxies_settings.get( 

330 'proxy_use_forwarding_for_https' 

331 ), 

332 **kwargs, 

333 } 

334 return {k: v for k, v in proxies_kwargs.items() if v is not None} 

335 

336 def _get_pool_manager_kwargs(self, **extra_kwargs): 

337 pool_manager_kwargs = { 

338 'timeout': self._timeout, 

339 'maxsize': self._max_pool_connections, 

340 'ssl_context': self._get_ssl_context(), 

341 'socket_options': self._socket_options, 

342 'cert_file': self._cert_file, 

343 'key_file': self._key_file, 

344 } 

345 pool_manager_kwargs.update(**extra_kwargs) 

346 return pool_manager_kwargs 

347 

348 def _get_ssl_context(self): 

349 return create_urllib3_context() 

350 

351 def _get_proxy_manager(self, proxy_url): 

352 if proxy_url not in self._proxy_managers: 

353 proxy_headers = self._proxy_config.proxy_headers_for(proxy_url) 

354 proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url) 

355 proxy_manager_kwargs = self._get_pool_manager_kwargs( 

356 proxy_headers=proxy_headers 

357 ) 

358 proxy_manager_kwargs.update( 

359 self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context) 

360 ) 

361 proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs) 

362 proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

363 self._proxy_managers[proxy_url] = proxy_manager 

364 

365 return self._proxy_managers[proxy_url] 

366 

367 def _path_url(self, url): 

368 parsed_url = urlparse(url) 

369 path = parsed_url.path 

370 if not path: 

371 path = '/' 

372 if parsed_url.query: 

373 path = path + '?' + parsed_url.query 

374 return path 

375 

376 def _setup_ssl_cert(self, conn, url, verify): 

377 if url.lower().startswith('https') and verify: 

378 conn.cert_reqs = 'CERT_REQUIRED' 

379 conn.ca_certs = get_cert_path(verify) 

380 else: 

381 conn.cert_reqs = 'CERT_NONE' 

382 conn.ca_certs = None 

383 

384 def _setup_proxy_ssl_context(self, proxy_url): 

385 proxies_settings = self._proxy_config.settings 

386 proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') 

387 proxy_cert = proxies_settings.get('proxy_client_cert') 

388 if proxy_ca_bundle is None and proxy_cert is None: 

389 return None 

390 

391 context = self._get_ssl_context() 

392 try: 

393 url = parse_url(proxy_url) 

394 # urllib3 disables this by default but we need it for proper 

395 # proxy tls negotiation when proxy_url is not an IP Address 

396 if not _is_ipaddress(url.host): 

397 context.check_hostname = True 

398 if proxy_ca_bundle is not None: 

399 context.load_verify_locations(cafile=proxy_ca_bundle) 

400 

401 if isinstance(proxy_cert, tuple): 

402 context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1]) 

403 elif isinstance(proxy_cert, str): 

404 context.load_cert_chain(proxy_cert) 

405 

406 return context 

407 except (OSError, URLLib3SSLError, LocationParseError) as e: 

408 raise InvalidProxiesConfigError(error=e) 

409 

410 def _get_connection_manager(self, url, proxy_url=None): 

411 if proxy_url: 

412 manager = self._get_proxy_manager(proxy_url) 

413 else: 

414 manager = self._manager 

415 return manager 

416 

417 def _get_request_target(self, url, proxy_url): 

418 has_proxy = proxy_url is not None 

419 

420 if not has_proxy: 

421 return self._path_url(url) 

422 

423 # HTTP proxies expect the request_target to be the absolute url to know 

424 # which host to establish a connection to. urllib3 also supports 

425 # forwarding for HTTPS through the 'use_forwarding_for_https' parameter. 

426 proxy_scheme = urlparse(proxy_url).scheme 

427 using_https_forwarding_proxy = ( 

428 proxy_scheme == 'https' 

429 and self._proxies_kwargs().get('use_forwarding_for_https', False) 

430 ) 

431 

432 if using_https_forwarding_proxy or url.startswith('http:'): 

433 return url 

434 else: 

435 return self._path_url(url) 

436 

437 def _chunked(self, headers): 

438 transfer_encoding = headers.get('Transfer-Encoding', b'') 

439 transfer_encoding = ensure_bytes(transfer_encoding) 

440 return transfer_encoding.lower() == b'chunked' 

441 

442 def close(self): 

443 self._manager.clear() 

444 for manager in self._proxy_managers.values(): 

445 manager.clear() 

446 

447 def send(self, request): 

448 try: 

449 proxy_url = self._proxy_config.proxy_url_for(request.url) 

450 manager = self._get_connection_manager(request.url, proxy_url) 

451 conn = manager.connection_from_url(request.url) 

452 self._setup_ssl_cert(conn, request.url, self._verify) 

453 if ensure_boolean( 

454 os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '') 

455 ): 

456 # This is currently an "experimental" feature which provides 

457 # no guarantees of backwards compatibility. It may be subject 

458 # to change or removal in any patch version. Anyone opting in 

459 # to this feature should strictly pin botocore. 

460 host = urlparse(request.url).hostname 

461 conn.proxy_headers['host'] = host 

462 

463 request_target = self._get_request_target(request.url, proxy_url) 

464 urllib_response = conn.urlopen( 

465 method=request.method, 

466 url=request_target, 

467 body=request.body, 

468 headers=request.headers, 

469 retries=Retry(False), 

470 assert_same_host=False, 

471 preload_content=False, 

472 decode_content=False, 

473 chunked=self._chunked(request.headers), 

474 ) 

475 

476 http_response = botocore.awsrequest.AWSResponse( 

477 request.url, 

478 urllib_response.status, 

479 urllib_response.headers, 

480 urllib_response, 

481 ) 

482 

483 if not request.stream_output: 

484 # Cause the raw stream to be exhausted immediately. We do it 

485 # this way instead of using preload_content because 

486 # preload_content will never buffer chunked responses 

487 http_response.content 

488 

489 return http_response 

490 except URLLib3SSLError as e: 

491 raise SSLError(endpoint_url=request.url, error=e) 

492 except (NewConnectionError, socket.gaierror) as e: 

493 raise EndpointConnectionError(endpoint_url=request.url, error=e) 

494 except ProxyError as e: 

495 raise ProxyConnectionError( 

496 proxy_url=mask_proxy_url(proxy_url), error=e 

497 ) 

498 except URLLib3ConnectTimeoutError as e: 

499 raise ConnectTimeoutError(endpoint_url=request.url, error=e) 

500 except URLLib3ReadTimeoutError as e: 

501 raise ReadTimeoutError(endpoint_url=request.url, error=e) 

502 except ProtocolError as e: 

503 raise ConnectionClosedError( 

504 error=e, request=request, endpoint_url=request.url 

505 ) 

506 except Exception as e: 

507 message = 'Exception received when sending urllib3 HTTP request' 

508 logger.debug(message, exc_info=True) 

509 raise HTTPClientError(error=e)