Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py: 35%

1from __future__ import annotations 

2 

3import email 

4import logging 

5import math 

6import random 

7import re 

8import time 

9import typing 

10import warnings 

11from collections.abc import Collection 

12from itertools import takewhile 

13from types import TracebackType 

14 

15from ..exceptions import ( 

16 ConnectTimeoutError, 

17 InvalidHeader, 

18 MaxRetryError, 

19 ProtocolError, 

20 ProxyError, 

21 ReadTimeoutError, 

22 ResponseError, 

23) 

24from .util import reraise 

25 

26if typing.TYPE_CHECKING: 

27 from typing_extensions import Self 

28 

29 from ..connectionpool import ConnectionPool 

30 from ..response import BaseHTTPResponse 

31 

32log = logging.getLogger(__name__) 

33 

34 

35# Data structure for representing the metadata of requests that result in a retry. 

36class RequestHistory(typing.NamedTuple): 

37 method: str | None 

38 url: str | None 

39 error: Exception | None 

40 status: int | None 

41 redirect_location: str | None 

42 

43 

44class Retry: 

45 """Retry configuration. 

46 

47 Each retry attempt will create a new Retry object with updated values, so 

48 they can be safely reused. 

49 

50 Retries can be defined as a default for a pool: 

51 

52 .. code-block:: python 

53 

54 retries = Retry(connect=5, read=2, redirect=5) 

55 http = PoolManager(retries=retries) 

56 response = http.request("GET", "https://example.com/") 

57 

58 Or per-request (which overrides the default for the pool): 

59 

60 .. code-block:: python 

61 

62 response = http.request("GET", "https://example.com/", retries=Retry(10)) 

63 

64 Retries can be disabled by passing ``False``: 

65 

66 .. code-block:: python 

67 

68 response = http.request("GET", "https://example.com/", retries=False) 

69 

70 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless 

71 retries are disabled, in which case the causing exception will be raised. 

72 

73 :param int total: 

74 Total number of retries to allow. Takes precedence over other counts. 

75 

76 Set to ``None`` to remove this constraint and fall back on other 

77 counts. 

78 

79 Set to ``0`` to fail on the first retry. 

80 

81 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

82 

83 :param int connect: 

84 How many connection-related errors to retry on. 

85 

86 These are errors raised before the request is sent to the remote server, 

87 which we assume has not triggered the server to process the request. 

88 

89 Set to ``0`` to fail on the first retry of this type. 

90 

91 :param int read: 

92 How many times to retry on read errors. 

93 

94 These errors are raised after the request was sent to the server, so the 

95 request may have side-effects. 

96 

97 Set to ``0`` to fail on the first retry of this type. 

98 

99 :param int redirect: 

100 How many redirects to perform. Limit this to avoid infinite redirect 

101 loops. 

102 

103 A redirect is a HTTP response with a status code 301, 302, 303, 307 or 

104 308. 

105 

106 Set to ``0`` to fail on the first retry of this type. 

107 

108 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

109 

110 :param int status: 

111 How many times to retry on bad status codes. 

112 

113 These are retries made on responses, where status code matches 

114 ``status_forcelist``. 

115 

116 Set to ``0`` to fail on the first retry of this type. 

117 

118 :param int other: 

119 How many times to retry on other errors. 

120 

121 Other errors are errors that are not connect, read, redirect or status errors. 

122 These errors might be raised after the request was sent to the server, so the 

123 request might have side-effects. 

124 

125 Set to ``0`` to fail on the first retry of this type. 

126 

127 If ``total`` is not set, it's a good idea to set this to 0 to account 

128 for unexpected edge cases and avoid infinite retry loops. 

129 

130 :param Collection allowed_methods: 

131 Set of uppercased HTTP method verbs that we should retry on. 

132 

133 By default, we only retry on methods which are considered to be 

134 idempotent (multiple requests with the same parameters end with the 

135 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. 

136 

137 Set to a ``None`` value to retry on any verb. 

138 

139 :param Collection status_forcelist: 

140 A set of integer HTTP status codes that we should force a retry on. 

141 A retry is initiated if the request method is in ``allowed_methods`` 

142 and the response status code is in ``status_forcelist``. 

143 

144 By default, this is disabled with ``None``. 

145 

146 :param float backoff_factor: 

147 A backoff factor to apply between attempts after the second try 

148 (most errors are resolved immediately by a second try without a 

149 delay). urllib3 will sleep for:: 

150 

151 {backoff factor} * (2 ** ({number of previous retries})) 

152 

153 seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: 

154 

155 random.uniform(0, {backoff jitter}) 

156 

157 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will 

158 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever 

159 be longer than `backoff_max`. 

160 

161 By default, backoff is disabled (factor set to 0). In production services, 

162 setting a non-zero factor (e.g. 0.5 to 1.0) along with `backoff_jitter` is 

163 recommended to prevent overloading servers with immediate retries. 

164 

165 :param float backoff_max: 

166 The maximum backoff time (in seconds) between retry attempts. 

167 This value caps the computed backoff from `backoff_factor`. 

168 

169 :param float backoff_jitter: 

170 Random jitter amount (in seconds) added to the computed backoff. 

171 Jitter is sampled uniformly from `0` to `backoff_jitter`. 

172 

173 :param bool raise_on_redirect: Whether, if the number of redirects is 

174 exhausted, to raise a MaxRetryError, or to return a response with a 

175 response code in the 3xx range. 

176 

177 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: 

178 whether we should raise an exception, or return a response, 

179 if status falls in ``status_forcelist`` range and retries have 

180 been exhausted. 

181 

182 :param tuple history: The history of the request encountered during 

183 each call to :meth:`~Retry.increment`. The list is in the order 

184 the requests occurred. Each list item is of class :class:`RequestHistory`. 

185 

186 :param bool respect_retry_after_header: 

187 Whether to respect Retry-After header on status codes defined as 

188 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. 

189 

190 :param Collection remove_headers_on_redirect: 

191 Sequence of headers to remove from the request when a response 

192 indicating a redirect is returned before firing off the redirected 

193 request. 

194 

195 :param int retry_after_max: Number of seconds to allow as the maximum for 

196 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`. 

197 Any Retry-After headers larger than this value will be limited to this 

198 value. 

199 """ 

200 

201 #: Default methods to be used for ``allowed_methods`` 

202 DEFAULT_ALLOWED_METHODS = frozenset( 

203 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] 

204 ) 

205 

206 #: Default status codes that trigger a retry when a Retry-After header is 

207 #: present and ``respect_retry_after_header`` is enabled. 

208 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) 

209 

210 #: Default headers to be used for ``remove_headers_on_redirect`` 

211 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( 

212 ["Cookie", "Authorization", "Proxy-Authorization"] 

213 ) 

214 

215 #: Default maximum backoff time. 

216 DEFAULT_BACKOFF_MAX = 120 

217 

218 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries. 

219 #: Default maximum allowed value for Retry-After headers in seconds 

220 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600 

221 

222 # Backward compatibility; assigned outside of the class. 

223 DEFAULT: typing.ClassVar[Retry] 

224 

225 def __init__( 

226 self, 

227 total: bool | int | None = 10, 

228 connect: int | None = None, 

229 read: int | None = None, 

230 redirect: bool | int | None = None, 

231 status: int | None = None, 

232 other: int | None = None, 

233 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, 

234 status_forcelist: typing.Collection[int] | None = None, 

235 backoff_factor: float = 0, 

236 backoff_max: float = DEFAULT_BACKOFF_MAX, 

237 raise_on_redirect: bool = True, 

238 raise_on_status: bool = True, 

239 history: tuple[RequestHistory, ...] | None = None, 

240 respect_retry_after_header: bool = True, 

241 remove_headers_on_redirect: typing.Collection[ 

242 str 

243 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, 

244 backoff_jitter: float = 0.0, 

245 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX, 

246 ) -> None: 

247 self.total = total 

248 self.connect = connect 

249 self.read = read 

250 self.status = status 

251 self.other = other 

252 

253 if redirect is False or total is False: 

254 redirect = 0 

255 raise_on_redirect = False 

256 

257 self.redirect = redirect 

258 self.status_forcelist = status_forcelist or set() 

259 if not allowed_methods and isinstance(allowed_methods, Collection): 

260 warnings.warn( 

261 "Using an empty collection for 'allowed_methods' option to " 

262 "retry on any verb is deprecated and will skip retries for " 

263 "all verbs in urllib3 v3.0. Instead use " 

264 "Retry(..., allowed_methods=None).", 

265 FutureWarning, 

266 stacklevel=2, 

267 ) 

268 self.allowed_methods = allowed_methods 

269 self.backoff_factor = backoff_factor 

270 self.backoff_max = backoff_max 

271 self.retry_after_max = retry_after_max 

272 self.raise_on_redirect = raise_on_redirect 

273 self.raise_on_status = raise_on_status 

274 self.history = history or () 

275 self.respect_retry_after_header = respect_retry_after_header 

276 self.remove_headers_on_redirect = frozenset( 

277 h.lower() for h in remove_headers_on_redirect 

278 ) 

279 self.backoff_jitter = backoff_jitter 

280 

281 def new(self, **kw: typing.Any) -> Self: 

282 params = dict( 

283 total=self.total, 

284 connect=self.connect, 

285 read=self.read, 

286 redirect=self.redirect, 

287 status=self.status, 

288 other=self.other, 

289 allowed_methods=self.allowed_methods, 

290 status_forcelist=self.status_forcelist, 

291 backoff_factor=self.backoff_factor, 

292 backoff_max=self.backoff_max, 

293 retry_after_max=self.retry_after_max, 

294 raise_on_redirect=self.raise_on_redirect, 

295 raise_on_status=self.raise_on_status, 

296 history=self.history, 

297 remove_headers_on_redirect=self.remove_headers_on_redirect, 

298 respect_retry_after_header=self.respect_retry_after_header, 

299 backoff_jitter=self.backoff_jitter, 

300 ) 

301 

302 params.update(kw) 

303 return type(self)(**params) # type: ignore[arg-type] 

304 

305 @classmethod 

306 def from_int( 

307 cls, 

308 retries: Retry | bool | int | None, 

309 redirect: bool | int | None = True, 

310 default: Retry | bool | int | None = None, 

311 ) -> Retry: 

312 """Backwards-compatibility for the old retries format.""" 

313 if retries is None: 

314 retries = default if default is not None else cls.DEFAULT 

315 

316 if isinstance(retries, Retry): 

317 return retries 

318 

319 redirect = bool(redirect) and None 

320 new_retries = cls(retries, redirect=redirect) 

321 log.debug("Converted retries value: %r -> %r", retries, new_retries) 

322 return new_retries 

323 

324 def get_backoff_time(self) -> float: 

325 """Formula for computing the current backoff 

326 

327 :rtype: float 

328 """ 

329 # We want to consider only the last consecutive errors sequence (Ignore redirects). 

330 consecutive_errors_len = len( 

331 list( 

332 takewhile(lambda x: x.redirect_location is None, reversed(self.history)) 

333 ) 

334 ) 

335 if consecutive_errors_len <= 1: 

336 return 0 

337 

338 try: 

339 backoff_value = math.ldexp(self.backoff_factor, consecutive_errors_len - 1) 

340 except OverflowError: 

341 backoff_value = math.inf if self.backoff_factor > 0 else -math.inf 

342 if self.backoff_jitter != 0.0: 

343 backoff_value += random.random() * self.backoff_jitter 

344 return float(max(0, min(self.backoff_max, backoff_value))) 

345 

346 def parse_retry_after(self, retry_after: str) -> float: 

347 seconds: float 

348 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 

349 if re.match(r"^\s*[0-9]+\s*$", retry_after): 

350 seconds = int(retry_after) 

351 else: 

352 retry_date_tuple = email.utils.parsedate_tz(retry_after) 

353 if retry_date_tuple is None: 

354 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") 

355 

356 try: 

357 retry_date = email.utils.mktime_tz(retry_date_tuple) 

358 except (ValueError, OverflowError): 

359 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") 

360 seconds = retry_date - time.time() 

361 

362 seconds = max(seconds, 0) 

363 

364 # Check the seconds do not exceed the specified maximum 

365 if seconds > self.retry_after_max: 

366 seconds = self.retry_after_max 

367 

368 return seconds 

369 

370 def get_retry_after(self, response: BaseHTTPResponse) -> float | None: 

371 """Get the value of Retry-After in seconds.""" 

372 

373 retry_after = response.headers.get("Retry-After") 

374 

375 if retry_after is None: 

376 return None 

377 

378 return self.parse_retry_after(retry_after) 

379 

380 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: 

381 retry_after = self.get_retry_after(response) 

382 if retry_after: 

383 time.sleep(retry_after) 

384 return True 

385 

386 return False 

387 

388 def _sleep_backoff(self) -> None: 

389 backoff = self.get_backoff_time() 

390 if backoff <= 0: 

391 return 

392 time.sleep(backoff) 

393 

394 def sleep(self, response: BaseHTTPResponse | None = None) -> None: 

395 """Sleep between retry attempts. 

396 

397 This method will respect a server's ``Retry-After`` response header 

398 and sleep the duration of the time requested. If that is not present, it 

399 will use an exponential backoff. By default, the backoff factor is 0 and 

400 this method will return immediately. 

401 """ 

402 

403 if self.respect_retry_after_header and response: 

404 slept = self.sleep_for_retry(response) 

405 if slept: 

406 return 

407 

408 self._sleep_backoff() 

409 

410 def _is_connection_error(self, err: Exception) -> bool: 

411 """Errors when we're fairly sure that the server did not receive the 

412 request, so it should be safe to retry. 

413 """ 

414 if isinstance(err, ProxyError): 

415 err = err.original_error 

416 return isinstance(err, ConnectTimeoutError) 

417 

418 def _is_read_error(self, err: Exception) -> bool: 

419 """Errors that occur after the request has been started, so we should 

420 assume that the server began processing it. 

421 """ 

422 return isinstance(err, (ReadTimeoutError, ProtocolError)) 

423 

424 def _is_method_retryable(self, method: str) -> bool: 

425 """Checks if a given HTTP method should be retried upon, depending if 

426 it is included in the allowed_methods 

427 """ 

428 if self.allowed_methods and method.upper() not in self.allowed_methods: 

429 return False 

430 return True 

431 

432 def is_retry( 

433 self, method: str, status_code: int, has_retry_after: bool = False 

434 ) -> bool: 

435 """Is this method/status code retryable? (Based on allowlists and control 

436 variables such as the number of total retries to allow, whether to 

437 respect the Retry-After header, whether this header is present, and 

438 whether the returned status code is on the list of status codes to 

439 be retried upon on the presence of the aforementioned header) 

440 """ 

441 if not self._is_method_retryable(method): 

442 return False 

443 

444 if self.status_forcelist and status_code in self.status_forcelist: 

445 return True 

446 

447 return bool( 

448 self.total 

449 and self.respect_retry_after_header 

450 and has_retry_after 

451 and (status_code in self.RETRY_AFTER_STATUS_CODES) 

452 ) 

453 

454 def is_exhausted(self) -> bool: 

455 """Are we out of retries?""" 

456 retry_counts = [ 

457 x 

458 for x in ( 

459 self.total, 

460 self.connect, 

461 self.read, 

462 self.redirect, 

463 self.status, 

464 self.other, 

465 ) 

466 if x 

467 ] 

468 if not retry_counts: 

469 return False 

470 

471 return min(retry_counts) < 0 

472 

473 def increment( 

474 self, 

475 method: str | None = None, 

476 url: str | None = None, 

477 response: BaseHTTPResponse | None = None, 

478 error: Exception | None = None, 

479 _pool: ConnectionPool | None = None, 

480 _stacktrace: TracebackType | None = None, 

481 ) -> Self: 

482 """Return a new Retry object with incremented retry counters. 

483 

484 :param response: A response object, or None, if the server did not 

485 return a response. 

486 :type response: :class:`~urllib3.response.BaseHTTPResponse` 

487 :param Exception error: An error encountered during the request, or 

488 None if the response was received successfully. 

489 

490 :return: A new ``Retry`` object. 

491 """ 

492 if self.total is False and error: 

493 # Disabled, indicate to re-raise the error. 

494 raise reraise(type(error), error, _stacktrace) 

495 

496 total = self.total 

497 if total is not None: 

498 total -= 1 

499 

500 connect = self.connect 

501 read = self.read 

502 redirect = self.redirect 

503 status_count = self.status 

504 other = self.other 

505 cause = "unknown" 

506 status = None 

507 redirect_location = None 

508 

509 if error and self._is_connection_error(error): 

510 # Connect retry? 

511 if connect is False: 

512 raise reraise(type(error), error, _stacktrace) 

513 elif connect is not None: 

514 connect -= 1 

515 

516 elif error and self._is_read_error(error): 

517 # Read retry? 

518 if read is False or method is None or not self._is_method_retryable(method): 

519 raise reraise(type(error), error, _stacktrace) 

520 elif read is not None: 

521 read -= 1 

522 

523 elif error: 

524 # Other retry? 

525 if other is not None: 

526 other -= 1 

527 

528 elif response and response.get_redirect_location(): 

529 # Redirect retry? 

530 if redirect is not None: 

531 redirect -= 1 

532 cause = "too many redirects" 

533 response_redirect_location = response.get_redirect_location() 

534 if response_redirect_location: 

535 redirect_location = response_redirect_location 

536 status = response.status 

537 

538 else: 

539 # Incrementing because of a server error like a 500 in 

540 # status_forcelist and the given method is in the allowed_methods 

541 cause = ResponseError.GENERIC_ERROR 

542 if response and response.status: 

543 if status_count is not None: 

544 status_count -= 1 

545 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) 

546 status = response.status 

547 

548 history = self.history + ( 

549 RequestHistory(method, url, error, status, redirect_location), 

550 ) 

551 

552 new_retry = self.new( 

553 total=total, 

554 connect=connect, 

555 read=read, 

556 redirect=redirect, 

557 status=status_count, 

558 other=other, 

559 history=history, 

560 ) 

561 

562 if new_retry.is_exhausted(): 

563 reason = error or ResponseError(cause) 

564 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] 

565 

566 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) 

567 

568 return new_retry 

569 

570 def __repr__(self) -> str: 

571 return ( 

572 f"{type(self).__name__}(total={self.total}, connect={self.connect}, " 

573 f"read={self.read}, redirect={self.redirect}, status={self.status})" 

574 ) 

575 

576 

577# For backwards compatibility (equivalent to pre-v1.9): 

578Retry.DEFAULT = Retry(3)