Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/requests/adapters.py: 23%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

237 statements  

1""" 

2requests.adapters 

3~~~~~~~~~~~~~~~~~ 

4 

5This module contains the transport adapters that Requests uses to define 

6and maintain connections. 

7""" 

8 

9import os.path 

10import socket # noqa: F401 

11import typing 

12 

13from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError 

14from urllib3.exceptions import HTTPError as _HTTPError 

15from urllib3.exceptions import InvalidHeader as _InvalidHeader 

16from urllib3.exceptions import ( 

17 LocationValueError, 

18 MaxRetryError, 

19 NewConnectionError, 

20 ProtocolError, 

21) 

22from urllib3.exceptions import ProxyError as _ProxyError 

23from urllib3.exceptions import ReadTimeoutError, ResponseError 

24from urllib3.exceptions import SSLError as _SSLError 

25from urllib3.poolmanager import PoolManager, proxy_from_url 

26from urllib3.util import Timeout as TimeoutSauce 

27from urllib3.util import parse_url 

28from urllib3.util.retry import Retry 

29from urllib3.util.ssl_ import create_urllib3_context 

30 

31from .auth import _basic_auth_str 

32from .compat import basestring, urlparse 

33from .cookies import extract_cookies_to_jar 

34from .exceptions import ( 

35 ConnectionError, 

36 ConnectTimeout, 

37 InvalidHeader, 

38 InvalidProxyURL, 

39 InvalidSchema, 

40 InvalidURL, 

41 ProxyError, 

42 ReadTimeout, 

43 RetryError, 

44 SSLError, 

45) 

46from .models import Response 

47from .structures import CaseInsensitiveDict 

48from .utils import ( 

49 DEFAULT_CA_BUNDLE_PATH, 

50 extract_zipped_paths, 

51 get_auth_from_url, 

52 get_encoding_from_headers, 

53 prepend_scheme_if_needed, 

54 select_proxy, 

55 urldefragauth, 

56) 

57 

58try: 

59 from urllib3.contrib.socks import SOCKSProxyManager 

60except ImportError: 

61 

62 def SOCKSProxyManager(*args, **kwargs): 

63 raise InvalidSchema("Missing dependencies for SOCKS support.") 

64 

65 

66if typing.TYPE_CHECKING: 

67 from .models import PreparedRequest 

68 

69 

70DEFAULT_POOLBLOCK = False 

71DEFAULT_POOLSIZE = 10 

72DEFAULT_RETRIES = 0 

73DEFAULT_POOL_TIMEOUT = None 

74 

75_preloaded_ssl_context = create_urllib3_context() 

76_preloaded_ssl_context.load_verify_locations( 

77 extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) 

78) 

79 

80 

81def _urllib3_request_context( 

82 request: "PreparedRequest", 

83 verify: "bool | str | None", 

84 client_cert: "typing.Tuple[str, str] | str | None", 

85) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": 

86 host_params = {} 

87 pool_kwargs = {} 

88 parsed_request_url = urlparse(request.url) 

89 scheme = parsed_request_url.scheme.lower() 

90 port = parsed_request_url.port 

91 cert_reqs = "CERT_REQUIRED" 

92 if verify is False: 

93 cert_reqs = "CERT_NONE" 

94 elif verify is True: 

95 pool_kwargs["ssl_context"] = _preloaded_ssl_context 

96 elif isinstance(verify, str): 

97 if not os.path.isdir(verify): 

98 pool_kwargs["ca_certs"] = verify 

99 else: 

100 pool_kwargs["ca_cert_dir"] = verify 

101 pool_kwargs["cert_reqs"] = cert_reqs 

102 if client_cert is not None: 

103 if isinstance(client_cert, tuple) and len(client_cert) == 2: 

104 pool_kwargs["cert_file"] = client_cert[0] 

105 pool_kwargs["key_file"] = client_cert[1] 

106 else: 

107 # According to our docs, we allow users to specify just the client 

108 # cert path 

109 pool_kwargs["cert_file"] = client_cert 

110 host_params = { 

111 "scheme": scheme, 

112 "host": parsed_request_url.hostname, 

113 "port": port, 

114 } 

115 return host_params, pool_kwargs 

116 

117 

118class BaseAdapter: 

119 """The Base Transport Adapter""" 

120 

121 def __init__(self): 

122 super().__init__() 

123 

124 def send( 

125 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None 

126 ): 

127 """Sends PreparedRequest object. Returns Response object. 

128 

129 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. 

130 :param stream: (optional) Whether to stream the request content. 

131 :param timeout: (optional) How long to wait for the server to send 

132 data before giving up, as a float, or a :ref:`(connect timeout, 

133 read timeout) <timeouts>` tuple. 

134 :type timeout: float or tuple 

135 :param verify: (optional) Either a boolean, in which case it controls whether we verify 

136 the server's TLS certificate, or a string, in which case it must be a path 

137 to a CA bundle to use 

138 :param cert: (optional) Any user-provided SSL certificate to be trusted. 

139 :param proxies: (optional) The proxies dictionary to apply to the request. 

140 """ 

141 raise NotImplementedError 

142 

143 def close(self): 

144 """Cleans up adapter specific items.""" 

145 raise NotImplementedError 

146 

147 

148class HTTPAdapter(BaseAdapter): 

149 """The built-in HTTP Adapter for urllib3. 

150 

151 Provides a general-case interface for Requests sessions to contact HTTP and 

152 HTTPS urls by implementing the Transport Adapter interface. This class will 

153 usually be created by the :class:`Session <Session>` class under the 

154 covers. 

155 

156 :param pool_connections: The number of urllib3 connection pools to cache. 

157 :param pool_maxsize: The maximum number of connections to save in the pool. 

158 :param max_retries: The maximum number of retries each connection 

159 should attempt. Note, this applies only to failed DNS lookups, socket 

160 connections and connection timeouts, never to requests where data has 

161 made it to the server. By default, Requests does not retry failed 

162 connections. If you need granular control over the conditions under 

163 which we retry a request, import urllib3's ``Retry`` class and pass 

164 that instead. 

165 :param pool_block: Whether the connection pool should block for connections. 

166 

167 Usage:: 

168 

169 >>> import requests 

170 >>> s = requests.Session() 

171 >>> a = requests.adapters.HTTPAdapter(max_retries=3) 

172 >>> s.mount('http://', a) 

173 """ 

174 

175 __attrs__ = [ 

176 "max_retries", 

177 "config", 

178 "_pool_connections", 

179 "_pool_maxsize", 

180 "_pool_block", 

181 ] 

182 

183 def __init__( 

184 self, 

185 pool_connections=DEFAULT_POOLSIZE, 

186 pool_maxsize=DEFAULT_POOLSIZE, 

187 max_retries=DEFAULT_RETRIES, 

188 pool_block=DEFAULT_POOLBLOCK, 

189 ): 

190 if max_retries == DEFAULT_RETRIES: 

191 self.max_retries = Retry(0, read=False) 

192 else: 

193 self.max_retries = Retry.from_int(max_retries) 

194 self.config = {} 

195 self.proxy_manager = {} 

196 

197 super().__init__() 

198 

199 self._pool_connections = pool_connections 

200 self._pool_maxsize = pool_maxsize 

201 self._pool_block = pool_block 

202 

203 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) 

204 

205 def __getstate__(self): 

206 return {attr: getattr(self, attr, None) for attr in self.__attrs__} 

207 

208 def __setstate__(self, state): 

209 # Can't handle by adding 'proxy_manager' to self.__attrs__ because 

210 # self.poolmanager uses a lambda function, which isn't pickleable. 

211 self.proxy_manager = {} 

212 self.config = {} 

213 

214 for attr, value in state.items(): 

215 setattr(self, attr, value) 

216 

217 self.init_poolmanager( 

218 self._pool_connections, self._pool_maxsize, block=self._pool_block 

219 ) 

220 

221 def init_poolmanager( 

222 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs 

223 ): 

224 """Initializes a urllib3 PoolManager. 

225 

226 This method should not be called from user code, and is only 

227 exposed for use when subclassing the 

228 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

229 

230 :param connections: The number of urllib3 connection pools to cache. 

231 :param maxsize: The maximum number of connections to save in the pool. 

232 :param block: Block when no free connections are available. 

233 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. 

234 """ 

235 # save these values for pickling 

236 self._pool_connections = connections 

237 self._pool_maxsize = maxsize 

238 self._pool_block = block 

239 

240 self.poolmanager = PoolManager( 

241 num_pools=connections, 

242 maxsize=maxsize, 

243 block=block, 

244 **pool_kwargs, 

245 ) 

246 

247 def proxy_manager_for(self, proxy, **proxy_kwargs): 

248 """Return urllib3 ProxyManager for the given proxy. 

249 

250 This method should not be called from user code, and is only 

251 exposed for use when subclassing the 

252 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

253 

254 :param proxy: The proxy to return a urllib3 ProxyManager for. 

255 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. 

256 :returns: ProxyManager 

257 :rtype: urllib3.ProxyManager 

258 """ 

259 if proxy in self.proxy_manager: 

260 manager = self.proxy_manager[proxy] 

261 elif proxy.lower().startswith("socks"): 

262 username, password = get_auth_from_url(proxy) 

263 manager = self.proxy_manager[proxy] = SOCKSProxyManager( 

264 proxy, 

265 username=username, 

266 password=password, 

267 num_pools=self._pool_connections, 

268 maxsize=self._pool_maxsize, 

269 block=self._pool_block, 

270 **proxy_kwargs, 

271 ) 

272 else: 

273 proxy_headers = self.proxy_headers(proxy) 

274 manager = self.proxy_manager[proxy] = proxy_from_url( 

275 proxy, 

276 proxy_headers=proxy_headers, 

277 num_pools=self._pool_connections, 

278 maxsize=self._pool_maxsize, 

279 block=self._pool_block, 

280 **proxy_kwargs, 

281 ) 

282 

283 return manager 

284 

285 def cert_verify(self, conn, url, verify, cert): 

286 """Verify a SSL certificate. This method should not be called from user 

287 code, and is only exposed for use when subclassing the 

288 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

289 

290 :param conn: The urllib3 connection object associated with the cert. 

291 :param url: The requested URL. 

292 :param verify: Either a boolean, in which case it controls whether we verify 

293 the server's TLS certificate, or a string, in which case it must be a path 

294 to a CA bundle to use 

295 :param cert: The SSL certificate to verify. 

296 """ 

297 if url.lower().startswith("https") and verify: 

298 conn.cert_reqs = "CERT_REQUIRED" 

299 

300 # Only load the CA certificates if 'verify' is a string indicating the CA bundle to use. 

301 # Otherwise, if verify is a boolean, we don't load anything since 

302 # the connection will be using a context with the default certificates already loaded, 

303 # and this avoids a call to the slow load_verify_locations() 

304 if verify is not True: 

305 # `verify` must be a str with a path then 

306 cert_loc = verify 

307 

308 if not os.path.exists(cert_loc): 

309 raise OSError( 

310 f"Could not find a suitable TLS CA certificate bundle, " 

311 f"invalid path: {cert_loc}" 

312 ) 

313 

314 if not os.path.isdir(cert_loc): 

315 conn.ca_certs = cert_loc 

316 else: 

317 conn.ca_cert_dir = cert_loc 

318 else: 

319 conn.cert_reqs = "CERT_NONE" 

320 conn.ca_certs = None 

321 conn.ca_cert_dir = None 

322 

323 if cert: 

324 if not isinstance(cert, basestring): 

325 conn.cert_file = cert[0] 

326 conn.key_file = cert[1] 

327 else: 

328 conn.cert_file = cert 

329 conn.key_file = None 

330 if conn.cert_file and not os.path.exists(conn.cert_file): 

331 raise OSError( 

332 f"Could not find the TLS certificate file, " 

333 f"invalid path: {conn.cert_file}" 

334 ) 

335 if conn.key_file and not os.path.exists(conn.key_file): 

336 raise OSError( 

337 f"Could not find the TLS key file, invalid path: {conn.key_file}" 

338 ) 

339 

340 def build_response(self, req, resp): 

341 """Builds a :class:`Response <requests.Response>` object from a urllib3 

342 response. This should not be called from user code, and is only exposed 

343 for use when subclassing the 

344 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>` 

345 

346 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response. 

347 :param resp: The urllib3 response object. 

348 :rtype: requests.Response 

349 """ 

350 response = Response() 

351 

352 # Fallback to None if there's no status_code, for whatever reason. 

353 response.status_code = getattr(resp, "status", None) 

354 

355 # Make headers case-insensitive. 

356 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {})) 

357 

358 # Set encoding. 

359 response.encoding = get_encoding_from_headers(response.headers) 

360 response.raw = resp 

361 response.reason = response.raw.reason 

362 

363 if isinstance(req.url, bytes): 

364 response.url = req.url.decode("utf-8") 

365 else: 

366 response.url = req.url 

367 

368 # Add new cookies from the server. 

369 extract_cookies_to_jar(response.cookies, req, resp) 

370 

371 # Give the Response some context. 

372 response.request = req 

373 response.connection = self 

374 

375 return response 

376 

377 def _get_connection(self, request, verify, proxies=None, cert=None): 

378 # Replace the existing get_connection without breaking things and 

379 # ensure that TLS settings are considered when we interact with 

380 # urllib3 HTTP Pools 

381 proxy = select_proxy(request.url, proxies) 

382 try: 

383 host_params, pool_kwargs = _urllib3_request_context(request, verify, cert) 

384 except ValueError as e: 

385 raise InvalidURL(e, request=request) 

386 if proxy: 

387 proxy = prepend_scheme_if_needed(proxy, "http") 

388 proxy_url = parse_url(proxy) 

389 if not proxy_url.host: 

390 raise InvalidProxyURL( 

391 "Please check proxy URL. It is malformed " 

392 "and could be missing the host." 

393 ) 

394 proxy_manager = self.proxy_manager_for(proxy) 

395 conn = proxy_manager.connection_from_host( 

396 **host_params, pool_kwargs=pool_kwargs 

397 ) 

398 else: 

399 # Only scheme should be lower case 

400 conn = self.poolmanager.connection_from_host( 

401 **host_params, pool_kwargs=pool_kwargs 

402 ) 

403 

404 return conn 

405 

406 def get_connection(self, url, proxies=None): 

407 """Returns a urllib3 connection for the given URL. This should not be 

408 called from user code, and is only exposed for use when subclassing the 

409 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

410 

411 :param url: The URL to connect to. 

412 :param proxies: (optional) A Requests-style dictionary of proxies used on this request. 

413 :rtype: urllib3.ConnectionPool 

414 """ 

415 proxy = select_proxy(url, proxies) 

416 

417 if proxy: 

418 proxy = prepend_scheme_if_needed(proxy, "http") 

419 proxy_url = parse_url(proxy) 

420 if not proxy_url.host: 

421 raise InvalidProxyURL( 

422 "Please check proxy URL. It is malformed " 

423 "and could be missing the host." 

424 ) 

425 proxy_manager = self.proxy_manager_for(proxy) 

426 conn = proxy_manager.connection_from_url(url) 

427 else: 

428 # Only scheme should be lower case 

429 parsed = urlparse(url) 

430 url = parsed.geturl() 

431 conn = self.poolmanager.connection_from_url(url) 

432 

433 return conn 

434 

435 def close(self): 

436 """Disposes of any internal state. 

437 

438 Currently, this closes the PoolManager and any active ProxyManager, 

439 which closes any pooled connections. 

440 """ 

441 self.poolmanager.clear() 

442 for proxy in self.proxy_manager.values(): 

443 proxy.clear() 

444 

445 def request_url(self, request, proxies): 

446 """Obtain the url to use when making the final request. 

447 

448 If the message is being sent through a HTTP proxy, the full URL has to 

449 be used. Otherwise, we should only use the path portion of the URL. 

450 

451 This should not be called from user code, and is only exposed for use 

452 when subclassing the 

453 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

454 

455 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. 

456 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. 

457 :rtype: str 

458 """ 

459 proxy = select_proxy(request.url, proxies) 

460 scheme = urlparse(request.url).scheme 

461 

462 is_proxied_http_request = proxy and scheme != "https" 

463 using_socks_proxy = False 

464 if proxy: 

465 proxy_scheme = urlparse(proxy).scheme.lower() 

466 using_socks_proxy = proxy_scheme.startswith("socks") 

467 

468 url = request.path_url 

469 if url.startswith("//"): # Don't confuse urllib3 

470 url = f"/{url.lstrip('/')}" 

471 

472 if is_proxied_http_request and not using_socks_proxy: 

473 url = urldefragauth(request.url) 

474 

475 return url 

476 

477 def add_headers(self, request, **kwargs): 

478 """Add any headers needed by the connection. As of v2.0 this does 

479 nothing by default, but is left for overriding by users that subclass 

480 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

481 

482 This should not be called from user code, and is only exposed for use 

483 when subclassing the 

484 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

485 

486 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to. 

487 :param kwargs: The keyword arguments from the call to send(). 

488 """ 

489 pass 

490 

491 def proxy_headers(self, proxy): 

492 """Returns a dictionary of the headers to add to any request sent 

493 through a proxy. This works with urllib3 magic to ensure that they are 

494 correctly sent to the proxy, rather than in a tunnelled request if 

495 CONNECT is being used. 

496 

497 This should not be called from user code, and is only exposed for use 

498 when subclassing the 

499 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. 

500 

501 :param proxy: The url of the proxy being used for this request. 

502 :rtype: dict 

503 """ 

504 headers = {} 

505 username, password = get_auth_from_url(proxy) 

506 

507 if username: 

508 headers["Proxy-Authorization"] = _basic_auth_str(username, password) 

509 

510 return headers 

511 

512 def send( 

513 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None 

514 ): 

515 """Sends PreparedRequest object. Returns Response object. 

516 

517 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. 

518 :param stream: (optional) Whether to stream the request content. 

519 :param timeout: (optional) How long to wait for the server to send 

520 data before giving up, as a float, or a :ref:`(connect timeout, 

521 read timeout) <timeouts>` tuple. 

522 :type timeout: float or tuple or urllib3 Timeout object 

523 :param verify: (optional) Either a boolean, in which case it controls whether 

524 we verify the server's TLS certificate, or a string, in which case it 

525 must be a path to a CA bundle to use 

526 :param cert: (optional) Any user-provided SSL certificate to be trusted. 

527 :param proxies: (optional) The proxies dictionary to apply to the request. 

528 :rtype: requests.Response 

529 """ 

530 

531 try: 

532 conn = self._get_connection(request, verify, proxies=proxies, cert=cert) 

533 except LocationValueError as e: 

534 raise InvalidURL(e, request=request) 

535 

536 self.cert_verify(conn, request.url, verify, cert) 

537 url = self.request_url(request, proxies) 

538 self.add_headers( 

539 request, 

540 stream=stream, 

541 timeout=timeout, 

542 verify=verify, 

543 cert=cert, 

544 proxies=proxies, 

545 ) 

546 

547 chunked = not (request.body is None or "Content-Length" in request.headers) 

548 

549 if isinstance(timeout, tuple): 

550 try: 

551 connect, read = timeout 

552 timeout = TimeoutSauce(connect=connect, read=read) 

553 except ValueError: 

554 raise ValueError( 

555 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " 

556 f"or a single float to set both timeouts to the same value." 

557 ) 

558 elif isinstance(timeout, TimeoutSauce): 

559 pass 

560 else: 

561 timeout = TimeoutSauce(connect=timeout, read=timeout) 

562 

563 try: 

564 resp = conn.urlopen( 

565 method=request.method, 

566 url=url, 

567 body=request.body, 

568 headers=request.headers, 

569 redirect=False, 

570 assert_same_host=False, 

571 preload_content=False, 

572 decode_content=False, 

573 retries=self.max_retries, 

574 timeout=timeout, 

575 chunked=chunked, 

576 ) 

577 

578 except (ProtocolError, OSError) as err: 

579 raise ConnectionError(err, request=request) 

580 

581 except MaxRetryError as e: 

582 if isinstance(e.reason, ConnectTimeoutError): 

583 # TODO: Remove this in 3.0.0: see #2811 

584 if not isinstance(e.reason, NewConnectionError): 

585 raise ConnectTimeout(e, request=request) 

586 

587 if isinstance(e.reason, ResponseError): 

588 raise RetryError(e, request=request) 

589 

590 if isinstance(e.reason, _ProxyError): 

591 raise ProxyError(e, request=request) 

592 

593 if isinstance(e.reason, _SSLError): 

594 # This branch is for urllib3 v1.22 and later. 

595 raise SSLError(e, request=request) 

596 

597 raise ConnectionError(e, request=request) 

598 

599 except ClosedPoolError as e: 

600 raise ConnectionError(e, request=request) 

601 

602 except _ProxyError as e: 

603 raise ProxyError(e) 

604 

605 except (_SSLError, _HTTPError) as e: 

606 if isinstance(e, _SSLError): 

607 # This branch is for urllib3 versions earlier than v1.22 

608 raise SSLError(e, request=request) 

609 elif isinstance(e, ReadTimeoutError): 

610 raise ReadTimeout(e, request=request) 

611 elif isinstance(e, _InvalidHeader): 

612 raise InvalidHeader(e, request=request) 

613 else: 

614 raise 

615 

616 return self.build_response(request, resp)