Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py: 20%

1from __future__ import absolute_import 

2 

3import errno 

4import logging 

5import re 

6import socket 

7import sys 

8import warnings 

9from socket import error as SocketError 

10from socket import timeout as SocketTimeout 

11 

12from ._collections import HTTPHeaderDict 

13from .connection import ( 

14 BaseSSLError, 

15 BrokenPipeError, 

16 DummyConnection, 

17 HTTPConnection, 

18 HTTPException, 

19 HTTPSConnection, 

20 VerifiedHTTPSConnection, 

21 port_by_scheme, 

22) 

23from .exceptions import ( 

24 ClosedPoolError, 

25 EmptyPoolError, 

26 HeaderParsingError, 

27 HostChangedError, 

28 InsecureRequestWarning, 

29 LocationValueError, 

30 MaxRetryError, 

31 NewConnectionError, 

32 ProtocolError, 

33 ProxyError, 

34 ReadTimeoutError, 

35 SSLError, 

36 TimeoutError, 

37) 

38from .packages import six 

39from .packages.six.moves import queue 

40from .request import RequestMethods 

41from .response import HTTPResponse 

42from .util.connection import is_connection_dropped 

43from .util.proxy import connection_requires_http_tunnel 

44from .util.queue import LifoQueue 

45from .util.request import set_file_position 

46from .util.response import assert_header_parsing 

47from .util.retry import Retry 

48from .util.ssl_match_hostname import CertificateError 

49from .util.timeout import Timeout 

50from .util.url import Url, _encode_target 

51from .util.url import _normalize_host as normalize_host 

52from .util.url import get_host, parse_url 

53 

54try: # Platform-specific: Python 3 

55 import weakref 

56 

57 weakref_finalize = weakref.finalize 

58except AttributeError: # Platform-specific: Python 2 

59 from .packages.backports.weakref_finalize import weakref_finalize 

60 

61xrange = six.moves.xrange 

62 

63log = logging.getLogger(__name__) 

64 

65_Default = object() 

66 

67 

68# Pool objects 

69class ConnectionPool(object): 

70 """ 

71 Base class for all connection pools, such as 

72 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. 

73 

74 .. note:: 

75 ConnectionPool.urlopen() does not normalize or percent-encode target URIs 

76 which is useful if your target server doesn't support percent-encoded 

77 target URIs. 

78 """ 

79 

80 scheme = None 

81 QueueCls = LifoQueue 

82 

83 def __init__(self, host, port=None): 

84 if not host: 

85 raise LocationValueError("No host specified.") 

86 

87 self.host = _normalize_host(host, scheme=self.scheme) 

88 self._proxy_host = host.lower() 

89 self.port = port 

90 

91 def __str__(self): 

92 return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) 

93 

94 def __enter__(self): 

95 return self 

96 

97 def __exit__(self, exc_type, exc_val, exc_tb): 

98 self.close() 

99 # Return False to re-raise any potential exceptions 

100 return False 

101 

102 def close(self): 

103 """ 

104 Close all pooled connections and disable the pool. 

105 """ 

106 pass 

107 

108 

109# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 

110_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} 

111 

112 

113class HTTPConnectionPool(ConnectionPool, RequestMethods): 

114 """ 

115 Thread-safe connection pool for one host. 

116 

117 :param host: 

118 Host used for this HTTP Connection (e.g. "localhost"), passed into 

119 :class:`http.client.HTTPConnection`. 

120 

121 :param port: 

122 Port used for this HTTP Connection (None is equivalent to 80), passed 

123 into :class:`http.client.HTTPConnection`. 

124 

125 :param strict: 

126 Causes BadStatusLine to be raised if the status line can't be parsed 

127 as a valid HTTP/1.0 or 1.1 status line, passed into 

128 :class:`http.client.HTTPConnection`. 

129 

130 .. note:: 

131 Only works in Python 2. This parameter is ignored in Python 3. 

132 

133 :param timeout: 

134 Socket timeout in seconds for each individual connection. This can 

135 be a float or integer, which sets the timeout for the HTTP request, 

136 or an instance of :class:`urllib3.util.Timeout` which gives you more 

137 fine-grained control over request timeouts. After the constructor has 

138 been parsed, this is always a `urllib3.util.Timeout` object. 

139 

140 :param maxsize: 

141 Number of connections to save that can be reused. More than 1 is useful 

142 in multithreaded situations. If ``block`` is set to False, more 

143 connections will be created but they will not be saved once they've 

144 been used. 

145 

146 :param block: 

147 If set to True, no more than ``maxsize`` connections will be used at 

148 a time. When no free connections are available, the call will block 

149 until a connection has been released. This is a useful side effect for 

150 particular multithreaded situations where one does not want to use more 

151 than maxsize connections per host to prevent flooding. 

152 

153 :param headers: 

154 Headers to include with all requests, unless other headers are given 

155 explicitly. 

156 

157 :param retries: 

158 Retry configuration to use by default with requests in this pool. 

159 

160 :param _proxy: 

161 Parsed proxy URL, should not be used directly, instead, see 

162 :class:`urllib3.ProxyManager` 

163 

164 :param _proxy_headers: 

165 A dictionary with proxy headers, should not be used directly, 

166 instead, see :class:`urllib3.ProxyManager` 

167 

168 :param \\**conn_kw: 

169 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, 

170 :class:`urllib3.connection.HTTPSConnection` instances. 

171 """ 

172 

173 scheme = "http" 

174 ConnectionCls = HTTPConnection 

175 ResponseCls = HTTPResponse 

176 

177 def __init__( 

178 self, 

179 host, 

180 port=None, 

181 strict=False, 

182 timeout=Timeout.DEFAULT_TIMEOUT, 

183 maxsize=1, 

184 block=False, 

185 headers=None, 

186 retries=None, 

187 _proxy=None, 

188 _proxy_headers=None, 

189 _proxy_config=None, 

190 **conn_kw 

191 ): 

192 ConnectionPool.__init__(self, host, port) 

193 RequestMethods.__init__(self, headers) 

194 

195 self.strict = strict 

196 

197 if not isinstance(timeout, Timeout): 

198 timeout = Timeout.from_float(timeout) 

199 

200 if retries is None: 

201 retries = Retry.DEFAULT 

202 

203 self.timeout = timeout 

204 self.retries = retries 

205 

206 self.pool = self.QueueCls(maxsize) 

207 self.block = block 

208 

209 self.proxy = _proxy 

210 self.proxy_headers = _proxy_headers or {} 

211 self.proxy_config = _proxy_config 

212 

213 # Fill the queue up so that doing get() on it will block properly 

214 for _ in xrange(maxsize): 

215 self.pool.put(None) 

216 

217 # These are mostly for testing and debugging purposes. 

218 self.num_connections = 0 

219 self.num_requests = 0 

220 self.conn_kw = conn_kw 

221 

222 if self.proxy: 

223 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. 

224 # We cannot know if the user has added default socket options, so we cannot replace the 

225 # list. 

226 self.conn_kw.setdefault("socket_options", []) 

227 

228 self.conn_kw["proxy"] = self.proxy 

229 self.conn_kw["proxy_config"] = self.proxy_config 

230 

231 # Do not pass 'self' as callback to 'finalize'. 

232 # Then the 'finalize' would keep an endless living (leak) to self. 

233 # By just passing a reference to the pool allows the garbage collector 

234 # to free self if nobody else has a reference to it. 

235 pool = self.pool 

236 

237 # Close all the HTTPConnections in the pool before the 

238 # HTTPConnectionPool object is garbage collected. 

239 weakref_finalize(self, _close_pool_connections, pool) 

240 

241 def _new_conn(self): 

242 """ 

243 Return a fresh :class:`HTTPConnection`. 

244 """ 

245 self.num_connections += 1 

246 log.debug( 

247 "Starting new HTTP connection (%d): %s:%s", 

248 self.num_connections, 

249 self.host, 

250 self.port or "80", 

251 ) 

252 

253 conn = self.ConnectionCls( 

254 host=self.host, 

255 port=self.port, 

256 timeout=self.timeout.connect_timeout, 

257 strict=self.strict, 

258 **self.conn_kw 

259 ) 

260 return conn 

261 

262 def _get_conn(self, timeout=None): 

263 """ 

264 Get a connection. Will return a pooled connection if one is available. 

265 

266 If no connections are available and :prop:`.block` is ``False``, then a 

267 fresh connection is returned. 

268 

269 :param timeout: 

270 Seconds to wait before giving up and raising 

271 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and 

272 :prop:`.block` is ``True``. 

273 """ 

274 conn = None 

275 try: 

276 conn = self.pool.get(block=self.block, timeout=timeout) 

277 

278 except AttributeError: # self.pool is None 

279 raise ClosedPoolError(self, "Pool is closed.") 

280 

281 except queue.Empty: 

282 if self.block: 

283 raise EmptyPoolError( 

284 self, 

285 "Pool reached maximum size and no more connections are allowed.", 

286 ) 

287 pass # Oh well, we'll create a new connection then 

288 

289 # If this is a persistent connection, check if it got disconnected 

290 if conn and is_connection_dropped(conn): 

291 log.debug("Resetting dropped connection: %s", self.host) 

292 conn.close() 

293 if getattr(conn, "auto_open", 1) == 0: 

294 # This is a proxied connection that has been mutated by 

295 # http.client._tunnel() and cannot be reused (since it would 

296 # attempt to bypass the proxy) 

297 conn = None 

298 

299 return conn or self._new_conn() 

300 

301 def _put_conn(self, conn): 

302 """ 

303 Put a connection back into the pool. 

304 

305 :param conn: 

306 Connection object for the current host and port as returned by 

307 :meth:`._new_conn` or :meth:`._get_conn`. 

308 

309 If the pool is already full, the connection is closed and discarded 

310 because we exceeded maxsize. If connections are discarded frequently, 

311 then maxsize should be increased. 

312 

313 If the pool is closed, then the connection will be closed and discarded. 

314 """ 

315 try: 

316 self.pool.put(conn, block=False) 

317 return # Everything is dandy, done. 

318 except AttributeError: 

319 # self.pool is None. 

320 pass 

321 except queue.Full: 

322 # This should never happen if self.block == True 

323 log.warning( 

324 "Connection pool is full, discarding connection: %s. Connection pool size: %s", 

325 self.host, 

326 self.pool.qsize(), 

327 ) 

328 # Connection never got put back into the pool, close it. 

329 if conn: 

330 conn.close() 

331 

332 def _validate_conn(self, conn): 

333 """ 

334 Called right before a request is made, after the socket is created. 

335 """ 

336 pass 

337 

338 def _prepare_proxy(self, conn): 

339 # Nothing to do for HTTP connections. 

340 pass 

341 

342 def _get_timeout(self, timeout): 

343 """Helper that always returns a :class:`urllib3.util.Timeout`""" 

344 if timeout is _Default: 

345 return self.timeout.clone() 

346 

347 if isinstance(timeout, Timeout): 

348 return timeout.clone() 

349 else: 

350 # User passed us an int/float. This is for backwards compatibility, 

351 # can be removed later 

352 return Timeout.from_float(timeout) 

353 

354 def _raise_timeout(self, err, url, timeout_value): 

355 """Is the error actually a timeout? Will raise a ReadTimeout or pass""" 

356 

357 if isinstance(err, SocketTimeout): 

358 raise ReadTimeoutError( 

359 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

360 ) 

361 

362 # See the above comment about EAGAIN in Python 3. In Python 2 we have 

363 # to specifically catch it and throw the timeout error 

364 if hasattr(err, "errno") and err.errno in _blocking_errnos: 

365 raise ReadTimeoutError( 

366 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

367 ) 

368 

369 # Catch possible read timeouts thrown as SSL errors. If not the 

370 # case, rethrow the original. We need to do this because of: 

371 # http://bugs.python.org/issue10272 

372 if "timed out" in str(err) or "did not complete (read)" in str( 

373 err 

374 ): # Python < 2.7.4 

375 raise ReadTimeoutError( 

376 self, url, "Read timed out. (read timeout=%s)" % timeout_value 

377 ) 

378 

379 def _make_request( 

380 self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw 

381 ): 

382 """ 

383 Perform a request on a given urllib connection object taken from our 

384 pool. 

385 

386 :param conn: 

387 a connection from one of our connection pools 

388 

389 :param timeout: 

390 Socket timeout in seconds for the request. This can be a 

391 float or integer, which will set the same timeout value for 

392 the socket connect and the socket read, or an instance of 

393 :class:`urllib3.util.Timeout`, which gives you more fine-grained 

394 control over your timeouts. 

395 """ 

396 self.num_requests += 1 

397 

398 timeout_obj = self._get_timeout(timeout) 

399 timeout_obj.start_connect() 

400 conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) 

401 

402 # Trigger any extra validation we need to do. 

403 try: 

404 self._validate_conn(conn) 

405 except (SocketTimeout, BaseSSLError) as e: 

406 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. 

407 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) 

408 raise 

409 

410 # conn.request() calls http.client.*.request, not the method in 

411 # urllib3.request. It also calls makefile (recv) on the socket. 

412 try: 

413 if chunked: 

414 conn.request_chunked(method, url, **httplib_request_kw) 

415 else: 

416 conn.request(method, url, **httplib_request_kw) 

417 

418 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is 

419 # legitimately able to close the connection after sending a valid response. 

420 # With this behaviour, the received response is still readable. 

421 except BrokenPipeError: 

422 # Python 3 

423 pass 

424 except IOError as e: 

425 # Python 2 and macOS/Linux 

426 # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS 

427 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ 

428 if e.errno not in { 

429 errno.EPIPE, 

430 errno.ESHUTDOWN, 

431 errno.EPROTOTYPE, 

432 }: 

433 raise 

434 

435 # Reset the timeout for the recv() on the socket 

436 read_timeout = timeout_obj.read_timeout 

437 

438 # App Engine doesn't have a sock attr 

439 if getattr(conn, "sock", None): 

440 # In Python 3 socket.py will catch EAGAIN and return None when you 

441 # try and read into the file pointer created by http.client, which 

442 # instead raises a BadStatusLine exception. Instead of catching 

443 # the exception and assuming all BadStatusLine exceptions are read 

444 # timeouts, check for a zero timeout before making the request. 

445 if read_timeout == 0: 

446 raise ReadTimeoutError( 

447 self, url, "Read timed out. (read timeout=%s)" % read_timeout 

448 ) 

449 if read_timeout is Timeout.DEFAULT_TIMEOUT: 

450 conn.sock.settimeout(socket.getdefaulttimeout()) 

451 else: # None or a value 

452 conn.sock.settimeout(read_timeout) 

453 

454 # Receive the response from the server 

455 try: 

456 try: 

457 # Python 2.7, use buffering of HTTP responses 

458 httplib_response = conn.getresponse(buffering=True) 

459 except TypeError: 

460 # Python 3 

461 try: 

462 httplib_response = conn.getresponse() 

463 except BaseException as e: 

464 # Remove the TypeError from the exception chain in 

465 # Python 3 (including for exceptions like SystemExit). 

466 # Otherwise it looks like a bug in the code. 

467 six.raise_from(e, None) 

468 except (SocketTimeout, BaseSSLError, SocketError) as e: 

469 self._raise_timeout(err=e, url=url, timeout_value=read_timeout) 

470 raise 

471 

472 # AppEngine doesn't have a version attr. 

473 http_version = getattr(conn, "_http_vsn_str", "HTTP/?") 

474 log.debug( 

475 '%s://%s:%s "%s %s %s" %s %s', 

476 self.scheme, 

477 self.host, 

478 self.port, 

479 method, 

480 url, 

481 http_version, 

482 httplib_response.status, 

483 httplib_response.length, 

484 ) 

485 

486 try: 

487 assert_header_parsing(httplib_response.msg) 

488 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 

489 log.warning( 

490 "Failed to parse headers (url=%s): %s", 

491 self._absolute_url(url), 

492 hpe, 

493 exc_info=True, 

494 ) 

495 

496 return httplib_response 

497 

498 def _absolute_url(self, path): 

499 return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url 

500 

501 def close(self): 

502 """ 

503 Close all pooled connections and disable the pool. 

504 """ 

505 if self.pool is None: 

506 return 

507 # Disable access to the pool 

508 old_pool, self.pool = self.pool, None 

509 

510 # Close all the HTTPConnections in the pool. 

511 _close_pool_connections(old_pool) 

512 

513 def is_same_host(self, url): 

514 """ 

515 Check if the given ``url`` is a member of the same host as this 

516 connection pool. 

517 """ 

518 if url.startswith("/"): 

519 return True 

520 

521 # TODO: Add optional support for socket.gethostbyname checking. 

522 scheme, host, port = get_host(url) 

523 if host is not None: 

524 host = _normalize_host(host, scheme=scheme) 

525 

526 # Use explicit default port for comparison when none is given 

527 if self.port and not port: 

528 port = port_by_scheme.get(scheme) 

529 elif not self.port and port == port_by_scheme.get(scheme): 

530 port = None 

531 

532 return (scheme, host, port) == (self.scheme, self.host, self.port) 

533 

534 def urlopen( 

535 self, 

536 method, 

537 url, 

538 body=None, 

539 headers=None, 

540 retries=None, 

541 redirect=True, 

542 assert_same_host=True, 

543 timeout=_Default, 

544 pool_timeout=None, 

545 release_conn=None, 

546 chunked=False, 

547 body_pos=None, 

548 **response_kw 

549 ): 

550 """ 

551 Get a connection from the pool and perform an HTTP request. This is the 

552 lowest level call for making a request, so you'll need to specify all 

553 the raw details. 

554 

555 .. note:: 

556 

557 More commonly, it's appropriate to use a convenience method provided 

558 by :class:`.RequestMethods`, such as :meth:`request`. 

559 

560 .. note:: 

561 

562 `release_conn` will only behave as expected if 

563 `preload_content=False` because we want to make 

564 `preload_content=False` the default behaviour someday soon without 

565 breaking backwards compatibility. 

566 

567 :param method: 

568 HTTP request method (such as GET, POST, PUT, etc.) 

569 

570 :param url: 

571 The URL to perform the request on. 

572 

573 :param body: 

574 Data to send in the request body, either :class:`str`, :class:`bytes`, 

575 an iterable of :class:`str`/:class:`bytes`, or a file-like object. 

576 

577 :param headers: 

578 Dictionary of custom headers to send, such as User-Agent, 

579 If-None-Match, etc. If None, pool headers are used. If provided, 

580 these headers completely replace any pool-specific headers. 

581 

582 :param retries: 

583 Configure the number of retries to allow before raising a 

584 :class:`~urllib3.exceptions.MaxRetryError` exception. 

585 

586 Pass ``None`` to retry until you receive a response. Pass a 

587 :class:`~urllib3.util.retry.Retry` object for fine-grained control 

588 over different types of retries. 

589 Pass an integer number to retry connection errors that many times, 

590 but no other types of errors. Pass zero to never retry. 

591 

592 If ``False``, then retries are disabled and any exception is raised 

593 immediately. Also, instead of raising a MaxRetryError on redirects, 

594 the redirect response will be returned. 

595 

596 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. 

597 

598 :param redirect: 

599 If True, automatically handle redirects (status codes 301, 302, 

600 303, 307, 308). Each redirect counts as a retry. Disabling retries 

601 will disable redirect, too. 

602 

603 :param assert_same_host: 

604 If ``True``, will make sure that the host of the pool requests is 

605 consistent else will raise HostChangedError. When ``False``, you can 

606 use the pool on an HTTP proxy and request foreign hosts. 

607 

608 :param timeout: 

609 If specified, overrides the default timeout for this one 

610 request. It may be a float (in seconds) or an instance of 

611 :class:`urllib3.util.Timeout`. 

612 

613 :param pool_timeout: 

614 If set and the pool is set to block=True, then this method will 

615 block for ``pool_timeout`` seconds and raise EmptyPoolError if no 

616 connection is available within the time period. 

617 

618 :param release_conn: 

619 If False, then the urlopen call will not release the connection 

620 back into the pool once a response is received (but will release if 

621 you read the entire contents of the response such as when 

622 `preload_content=True`). This is useful if you're not preloading 

623 the response's content immediately. You will need to call 

624 ``r.release_conn()`` on the response ``r`` to return the connection 

625 back into the pool. If None, it takes the value of 

626 ``response_kw.get('preload_content', True)``. 

627 

628 :param chunked: 

629 If True, urllib3 will send the body using chunked transfer 

630 encoding. Otherwise, urllib3 will send the body using the standard 

631 content-length form. Defaults to False. 

632 

633 :param int body_pos: 

634 Position to seek to in file-like body in the event of a retry or 

635 redirect. Typically this won't need to be set because urllib3 will 

636 auto-populate the value when needed. 

637 

638 :param \\**response_kw: 

639 Additional parameters are passed to 

640 :meth:`urllib3.response.HTTPResponse.from_httplib` 

641 """ 

642 

643 parsed_url = parse_url(url) 

644 destination_scheme = parsed_url.scheme 

645 

646 if headers is None: 

647 headers = self.headers 

648 

649 if not isinstance(retries, Retry): 

650 retries = Retry.from_int(retries, redirect=redirect, default=self.retries) 

651 

652 if release_conn is None: 

653 release_conn = response_kw.get("preload_content", True) 

654 

655 # Check host 

656 if assert_same_host and not self.is_same_host(url): 

657 raise HostChangedError(self, url, retries) 

658 

659 # Ensure that the URL we're connecting to is properly encoded 

660 if url.startswith("/"): 

661 url = six.ensure_str(_encode_target(url)) 

662 else: 

663 url = six.ensure_str(parsed_url.url) 

664 

665 conn = None 

666 

667 # Track whether `conn` needs to be released before 

668 # returning/raising/recursing. Update this variable if necessary, and 

669 # leave `release_conn` constant throughout the function. That way, if 

670 # the function recurses, the original value of `release_conn` will be 

671 # passed down into the recursive call, and its value will be respected. 

672 # 

673 # See issue #651 [1] for details. 

674 # 

675 # [1] <https://github.com/urllib3/urllib3/issues/651> 

676 release_this_conn = release_conn 

677 

678 http_tunnel_required = connection_requires_http_tunnel( 

679 self.proxy, self.proxy_config, destination_scheme 

680 ) 

681 

682 # Merge the proxy headers. Only done when not using HTTP CONNECT. We 

683 # have to copy the headers dict so we can safely change it without those 

684 # changes being reflected in anyone else's copy. 

685 if not http_tunnel_required: 

686 headers = headers.copy() 

687 headers.update(self.proxy_headers) 

688 

689 # Must keep the exception bound to a separate variable or else Python 3 

690 # complains about UnboundLocalError. 

691 err = None 

692 

693 # Keep track of whether we cleanly exited the except block. This 

694 # ensures we do proper cleanup in finally. 

695 clean_exit = False 

696 

697 # Rewind body position, if needed. Record current position 

698 # for future rewinds in the event of a redirect/retry. 

699 body_pos = set_file_position(body, body_pos) 

700 

701 try: 

702 # Request a connection from the queue. 

703 timeout_obj = self._get_timeout(timeout) 

704 conn = self._get_conn(timeout=pool_timeout) 

705 

706 conn.timeout = timeout_obj.connect_timeout 

707 

708 is_new_proxy_conn = self.proxy is not None and not getattr( 

709 conn, "sock", None 

710 ) 

711 if is_new_proxy_conn and http_tunnel_required: 

712 self._prepare_proxy(conn) 

713 

714 # Make the request on the httplib connection object. 

715 httplib_response = self._make_request( 

716 conn, 

717 method, 

718 url, 

719 timeout=timeout_obj, 

720 body=body, 

721 headers=headers, 

722 chunked=chunked, 

723 ) 

724 

725 # If we're going to release the connection in ``finally:``, then 

726 # the response doesn't need to know about the connection. Otherwise 

727 # it will also try to release it and we'll have a double-release 

728 # mess. 

729 response_conn = conn if not release_conn else None 

730 

731 # Pass method to Response for length checking 

732 response_kw["request_method"] = method 

733 

734 # Import httplib's response into our own wrapper object 

735 response = self.ResponseCls.from_httplib( 

736 httplib_response, 

737 pool=self, 

738 connection=response_conn, 

739 retries=retries, 

740 **response_kw 

741 ) 

742 

743 # Everything went great! 

744 clean_exit = True 

745 

746 except EmptyPoolError: 

747 # Didn't get a connection from the pool, no need to clean up 

748 clean_exit = True 

749 release_this_conn = False 

750 raise 

751 

752 except ( 

753 TimeoutError, 

754 HTTPException, 

755 SocketError, 

756 ProtocolError, 

757 BaseSSLError, 

758 SSLError, 

759 CertificateError, 

760 ) as e: 

761 # Discard the connection for these exceptions. It will be 

762 # replaced during the next _get_conn() call. 

763 clean_exit = False 

764 

765 def _is_ssl_error_message_from_http_proxy(ssl_error): 

766 # We're trying to detect the message 'WRONG_VERSION_NUMBER' but 

767 # SSLErrors are kinda all over the place when it comes to the message, 

768 # so we try to cover our bases here! 

769 message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) 

770 return ( 

771 "wrong version number" in message or "unknown protocol" in message 

772 ) 

773 

774 # Try to detect a common user error with proxies which is to 

775 # set an HTTP proxy to be HTTPS when it should be 'http://' 

776 # (ie {'http': 'http://proxy', 'https': 'https://proxy'}) 

777 # Instead we add a nice error message and point to a URL. 

778 if ( 

779 isinstance(e, BaseSSLError) 

780 and self.proxy 

781 and _is_ssl_error_message_from_http_proxy(e) 

782 and conn.proxy 

783 and conn.proxy.scheme == "https" 

784 ): 

785 e = ProxyError( 

786 "Your proxy appears to only use HTTP and not HTTPS, " 

787 "try changing your proxy URL to be HTTP. See: " 

788 "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" 

789 "#https-proxy-error-http-proxy", 

790 SSLError(e), 

791 ) 

792 elif isinstance(e, (BaseSSLError, CertificateError)): 

793 e = SSLError(e) 

794 elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: 

795 e = ProxyError("Cannot connect to proxy.", e) 

796 elif isinstance(e, (SocketError, HTTPException)): 

797 e = ProtocolError("Connection aborted.", e) 

798 

799 retries = retries.increment( 

800 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] 

801 ) 

802 retries.sleep() 

803 

804 # Keep track of the error for the retry warning. 

805 err = e 

806 

807 finally: 

808 if not clean_exit: 

809 # We hit some kind of exception, handled or otherwise. We need 

810 # to throw the connection away unless explicitly told not to. 

811 # Close the connection, set the variable to None, and make sure 

812 # we put the None back in the pool to avoid leaking it. 

813 conn = conn and conn.close() 

814 release_this_conn = True 

815 

816 if release_this_conn: 

817 # Put the connection back to be reused. If the connection is 

818 # expired then it will be None, which will get replaced with a 

819 # fresh connection during _get_conn. 

820 self._put_conn(conn) 

821 

822 if not conn: 

823 # Try again 

824 log.warning( 

825 "Retrying (%r) after connection broken by '%r': %s", retries, err, url 

826 ) 

827 return self.urlopen( 

828 method, 

829 url, 

830 body, 

831 headers, 

832 retries, 

833 redirect, 

834 assert_same_host, 

835 timeout=timeout, 

836 pool_timeout=pool_timeout, 

837 release_conn=release_conn, 

838 chunked=chunked, 

839 body_pos=body_pos, 

840 **response_kw 

841 ) 

842 

843 # Handle redirect? 

844 redirect_location = redirect and response.get_redirect_location() 

845 if redirect_location: 

846 if response.status == 303: 

847 # Change the method according to RFC 9110, Section 15.4.4. 

848 method = "GET" 

849 # And lose the body not to transfer anything sensitive. 

850 body = None 

851 headers = HTTPHeaderDict(headers)._prepare_for_method_change() 

852 

853 try: 

854 retries = retries.increment(method, url, response=response, _pool=self) 

855 except MaxRetryError: 

856 if retries.raise_on_redirect: 

857 response.drain_conn() 

858 raise 

859 return response 

860 

861 response.drain_conn() 

862 retries.sleep_for_retry(response) 

863 log.debug("Redirecting %s -> %s", url, redirect_location) 

864 return self.urlopen( 

865 method, 

866 redirect_location, 

867 body, 

868 headers, 

869 retries=retries, 

870 redirect=redirect, 

871 assert_same_host=assert_same_host, 

872 timeout=timeout, 

873 pool_timeout=pool_timeout, 

874 release_conn=release_conn, 

875 chunked=chunked, 

876 body_pos=body_pos, 

877 **response_kw 

878 ) 

879 

880 # Check if we should retry the HTTP response. 

881 has_retry_after = bool(response.headers.get("Retry-After")) 

882 if retries.is_retry(method, response.status, has_retry_after): 

883 try: 

884 retries = retries.increment(method, url, response=response, _pool=self) 

885 except MaxRetryError: 

886 if retries.raise_on_status: 

887 response.drain_conn() 

888 raise 

889 return response 

890 

891 response.drain_conn() 

892 retries.sleep(response) 

893 log.debug("Retry: %s", url) 

894 return self.urlopen( 

895 method, 

896 url, 

897 body, 

898 headers, 

899 retries=retries, 

900 redirect=redirect, 

901 assert_same_host=assert_same_host, 

902 timeout=timeout, 

903 pool_timeout=pool_timeout, 

904 release_conn=release_conn, 

905 chunked=chunked, 

906 body_pos=body_pos, 

907 **response_kw 

908 ) 

909 

910 return response 

911 

912 

913class HTTPSConnectionPool(HTTPConnectionPool): 

914 """ 

915 Same as :class:`.HTTPConnectionPool`, but HTTPS. 

916 

917 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``, 

918 ``assert_hostname`` and ``host`` in this order to verify connections. 

919 If ``assert_hostname`` is False, no verification is done. 

920 

921 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, 

922 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` 

923 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade 

924 the connection socket into an SSL socket. 

925 """ 

926 

927 scheme = "https" 

928 ConnectionCls = HTTPSConnection 

929 

930 def __init__( 

931 self, 

932 host, 

933 port=None, 

934 strict=False, 

935 timeout=Timeout.DEFAULT_TIMEOUT, 

936 maxsize=1, 

937 block=False, 

938 headers=None, 

939 retries=None, 

940 _proxy=None, 

941 _proxy_headers=None, 

942 key_file=None, 

943 cert_file=None, 

944 cert_reqs=None, 

945 key_password=None, 

946 ca_certs=None, 

947 ssl_version=None, 

948 assert_hostname=None, 

949 assert_fingerprint=None, 

950 ca_cert_dir=None, 

951 **conn_kw 

952 ): 

953 

954 HTTPConnectionPool.__init__( 

955 self, 

956 host, 

957 port, 

958 strict, 

959 timeout, 

960 maxsize, 

961 block, 

962 headers, 

963 retries, 

964 _proxy, 

965 _proxy_headers, 

966 **conn_kw 

967 ) 

968 

969 self.key_file = key_file 

970 self.cert_file = cert_file 

971 self.cert_reqs = cert_reqs 

972 self.key_password = key_password 

973 self.ca_certs = ca_certs 

974 self.ca_cert_dir = ca_cert_dir 

975 self.ssl_version = ssl_version 

976 self.assert_hostname = assert_hostname 

977 self.assert_fingerprint = assert_fingerprint 

978 

979 def _prepare_conn(self, conn): 

980 """ 

981 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` 

982 and establish the tunnel if proxy is used. 

983 """ 

984 

985 if isinstance(conn, VerifiedHTTPSConnection): 

986 conn.set_cert( 

987 key_file=self.key_file, 

988 key_password=self.key_password, 

989 cert_file=self.cert_file, 

990 cert_reqs=self.cert_reqs, 

991 ca_certs=self.ca_certs, 

992 ca_cert_dir=self.ca_cert_dir, 

993 assert_hostname=self.assert_hostname, 

994 assert_fingerprint=self.assert_fingerprint, 

995 ) 

996 conn.ssl_version = self.ssl_version 

997 return conn 

998 

999 def _prepare_proxy(self, conn): 

1000 """ 

1001 Establishes a tunnel connection through HTTP CONNECT.