Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py: 36%

1from __future__ import annotations 

2 

3import email 

4import logging 

5import random 

6import re 

7import time 

8import typing 

9import warnings 

10from collections.abc import Collection 

11from itertools import takewhile 

12from types import TracebackType 

13 

14from ..exceptions import ( 

15 ConnectTimeoutError, 

16 InvalidHeader, 

17 MaxRetryError, 

18 ProtocolError, 

19 ProxyError, 

20 ReadTimeoutError, 

21 ResponseError, 

22) 

23from .util import reraise 

24 

25if typing.TYPE_CHECKING: 

26 from typing_extensions import Self 

27 

28 from ..connectionpool import ConnectionPool 

29 from ..response import BaseHTTPResponse 

30 

31log = logging.getLogger(__name__) 

32 

33 

34# Data structure for representing the metadata of requests that result in a retry. 

35class RequestHistory(typing.NamedTuple): 

36 method: str | None 

37 url: str | None 

38 error: Exception | None 

39 status: int | None 

40 redirect_location: str | None 

41 

42 

43class Retry: 

44 """Retry configuration. 

45 

46 Each retry attempt will create a new Retry object with updated values, so 

47 they can be safely reused. 

48 

49 Retries can be defined as a default for a pool: 

50 

51 .. code-block:: python 

52 

53 retries = Retry(connect=5, read=2, redirect=5) 

54 http = PoolManager(retries=retries) 

55 response = http.request("GET", "https://example.com/") 

56 

57 Or per-request (which overrides the default for the pool): 

58 

59 .. code-block:: python 

60 

61 response = http.request("GET", "https://example.com/", retries=Retry(10)) 

62 

63 Retries can be disabled by passing ``False``: 

64 

65 .. code-block:: python 

66 

67 response = http.request("GET", "https://example.com/", retries=False) 

68 

69 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless 

70 retries are disabled, in which case the causing exception will be raised. 

71 

72 :param int total: 

73 Total number of retries to allow. Takes precedence over other counts. 

74 

75 Set to ``None`` to remove this constraint and fall back on other 

76 counts. 

77 

78 Set to ``0`` to fail on the first retry. 

79 

80 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

81 

82 :param int connect: 

83 How many connection-related errors to retry on. 

84 

85 These are errors raised before the request is sent to the remote server, 

86 which we assume has not triggered the server to process the request. 

87 

88 Set to ``0`` to fail on the first retry of this type. 

89 

90 :param int read: 

91 How many times to retry on read errors. 

92 

93 These errors are raised after the request was sent to the server, so the 

94 request may have side-effects. 

95 

96 Set to ``0`` to fail on the first retry of this type. 

97 

98 :param int redirect: 

99 How many redirects to perform. Limit this to avoid infinite redirect 

100 loops. 

101 

102 A redirect is a HTTP response with a status code 301, 302, 303, 307 or 

103 308. 

104 

105 Set to ``0`` to fail on the first retry of this type. 

106 

107 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

108 

109 :param int status: 

110 How many times to retry on bad status codes. 

111 

112 These are retries made on responses, where status code matches 

113 ``status_forcelist``. 

114 

115 Set to ``0`` to fail on the first retry of this type. 

116 

117 :param int other: 

118 How many times to retry on other errors. 

119 

120 Other errors are errors that are not connect, read, redirect or status errors. 

121 These errors might be raised after the request was sent to the server, so the 

122 request might have side-effects. 

123 

124 Set to ``0`` to fail on the first retry of this type. 

125 

126 If ``total`` is not set, it's a good idea to set this to 0 to account 

127 for unexpected edge cases and avoid infinite retry loops. 

128 

129 :param Collection allowed_methods: 

130 Set of uppercased HTTP method verbs that we should retry on. 

131 

132 By default, we only retry on methods which are considered to be 

133 idempotent (multiple requests with the same parameters end with the 

134 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. 

135 

136 Set to a ``None`` value to retry on any verb. 

137 

138 :param Collection status_forcelist: 

139 A set of integer HTTP status codes that we should force a retry on. 

140 A retry is initiated if the request method is in ``allowed_methods`` 

141 and the response status code is in ``status_forcelist``. 

142 

143 By default, this is disabled with ``None``. 

144 

145 :param float backoff_factor: 

146 A backoff factor to apply between attempts after the second try 

147 (most errors are resolved immediately by a second try without a 

148 delay). urllib3 will sleep for:: 

149 

150 {backoff factor} * (2 ** ({number of previous retries})) 

151 

152 seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: 

153 

154 random.uniform(0, {backoff jitter}) 

155 

156 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will 

157 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever 

158 be longer than `backoff_max`. 

159 

160 By default, backoff is disabled (factor set to 0). In production services, 

161 setting a non-zero factor (e.g. 0.5 to 1.0) along with `backoff_jitter` is 

162 recommended to prevent overloading servers with immediate retries. 

163 

164 :param float backoff_max: 

165 The maximum backoff time (in seconds) between retry attempts. 

166 This value caps the computed backoff from `backoff_factor`. 

167 

168 :param float backoff_jitter: 

169 Random jitter amount (in seconds) added to the computed backoff. 

170 Jitter is sampled uniformly from `0` to `backoff_jitter`. 

171 

172 :param bool raise_on_redirect: Whether, if the number of redirects is 

173 exhausted, to raise a MaxRetryError, or to return a response with a 

174 response code in the 3xx range. 

175 

176 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: 

177 whether we should raise an exception, or return a response, 

178 if status falls in ``status_forcelist`` range and retries have 

179 been exhausted. 

180 

181 :param tuple history: The history of the request encountered during 

182 each call to :meth:`~Retry.increment`. The list is in the order 

183 the requests occurred. Each list item is of class :class:`RequestHistory`. 

184 

185 :param bool respect_retry_after_header: 

186 Whether to respect Retry-After header on status codes defined as 

187 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. 

188 

189 :param Collection remove_headers_on_redirect: 

190 Sequence of headers to remove from the request when a response 

191 indicating a redirect is returned before firing off the redirected 

192 request. 

193 

194 :param int retry_after_max: Number of seconds to allow as the maximum for 

195 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`. 

196 Any Retry-After headers larger than this value will be limited to this 

197 value. 

198 """ 

199 

200 #: Default methods to be used for ``allowed_methods`` 

201 DEFAULT_ALLOWED_METHODS = frozenset( 

202 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] 

203 ) 

204 

205 #: Default status codes that trigger a retry when a Retry-After header is 

206 #: present and ``respect_retry_after_header`` is enabled. 

207 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) 

208 

209 #: Default headers to be used for ``remove_headers_on_redirect`` 

210 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( 

211 ["Cookie", "Authorization", "Proxy-Authorization"] 

212 ) 

213 

214 #: Default maximum backoff time. 

215 DEFAULT_BACKOFF_MAX = 120 

216 

217 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries. 

218 #: Default maximum allowed value for Retry-After headers in seconds 

219 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600 

220 

221 # Backward compatibility; assigned outside of the class. 

222 DEFAULT: typing.ClassVar[Retry] 

223 

224 def __init__( 

225 self, 

226 total: bool | int | None = 10, 

227 connect: int | None = None, 

228 read: int | None = None, 

229 redirect: bool | int | None = None, 

230 status: int | None = None, 

231 other: int | None = None, 

232 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, 

233 status_forcelist: typing.Collection[int] | None = None, 

234 backoff_factor: float = 0, 

235 backoff_max: float = DEFAULT_BACKOFF_MAX, 

236 raise_on_redirect: bool = True, 

237 raise_on_status: bool = True, 

238 history: tuple[RequestHistory, ...] | None = None, 

239 respect_retry_after_header: bool = True, 

240 remove_headers_on_redirect: typing.Collection[ 

241 str 

242 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, 

243 backoff_jitter: float = 0.0, 

244 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX, 

245 ) -> None: 

246 self.total = total 

247 self.connect = connect 

248 self.read = read 

249 self.status = status 

250 self.other = other 

251 

252 if redirect is False or total is False: 

253 redirect = 0 

254 raise_on_redirect = False 

255 

256 self.redirect = redirect 

257 self.status_forcelist = status_forcelist or set() 

258 if not allowed_methods and isinstance(allowed_methods, Collection): 

259 warnings.warn( 

260 "Using an empty collection for 'allowed_methods' option to " 

261 "retry on any verb is deprecated and will skip retries for " 

262 "all verbs in urllib3 v3.0. Instead use " 

263 "Retry(..., allowed_methods=None).", 

264 FutureWarning, 

265 stacklevel=2, 

266 ) 

267 self.allowed_methods = allowed_methods 

268 self.backoff_factor = backoff_factor 

269 self.backoff_max = backoff_max 

270 self.retry_after_max = retry_after_max 

271 self.raise_on_redirect = raise_on_redirect 

272 self.raise_on_status = raise_on_status 

273 self.history = history or () 

274 self.respect_retry_after_header = respect_retry_after_header 

275 self.remove_headers_on_redirect = frozenset( 

276 h.lower() for h in remove_headers_on_redirect 

277 ) 

278 self.backoff_jitter = backoff_jitter 

279 

280 def new(self, **kw: typing.Any) -> Self: 

281 params = dict( 

282 total=self.total, 

283 connect=self.connect, 

284 read=self.read, 

285 redirect=self.redirect, 

286 status=self.status, 

287 other=self.other, 

288 allowed_methods=self.allowed_methods, 

289 status_forcelist=self.status_forcelist, 

290 backoff_factor=self.backoff_factor, 

291 backoff_max=self.backoff_max, 

292 retry_after_max=self.retry_after_max, 

293 raise_on_redirect=self.raise_on_redirect, 

294 raise_on_status=self.raise_on_status, 

295 history=self.history, 

296 remove_headers_on_redirect=self.remove_headers_on_redirect, 

297 respect_retry_after_header=self.respect_retry_after_header, 

298 backoff_jitter=self.backoff_jitter, 

299 ) 

300 

301 params.update(kw) 

302 return type(self)(**params) # type: ignore[arg-type] 

303 

304 @classmethod 

305 def from_int( 

306 cls, 

307 retries: Retry | bool | int | None, 

308 redirect: bool | int | None = True, 

309 default: Retry | bool | int | None = None, 

310 ) -> Retry: 

311 """Backwards-compatibility for the old retries format.""" 

312 if retries is None: 

313 retries = default if default is not None else cls.DEFAULT 

314 

315 if isinstance(retries, Retry): 

316 return retries 

317 

318 redirect = bool(redirect) and None 

319 new_retries = cls(retries, redirect=redirect) 

320 log.debug("Converted retries value: %r -> %r", retries, new_retries) 

321 return new_retries 

322 

323 def get_backoff_time(self) -> float: 

324 """Formula for computing the current backoff 

325 

326 :rtype: float 

327 """ 

328 # We want to consider only the last consecutive errors sequence (Ignore redirects). 

329 consecutive_errors_len = len( 

330 list( 

331 takewhile(lambda x: x.redirect_location is None, reversed(self.history)) 

332 ) 

333 ) 

334 if consecutive_errors_len <= 1: 

335 return 0 

336 

337 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) 

338 if self.backoff_jitter != 0.0: 

339 backoff_value += random.random() * self.backoff_jitter 

340 return float(max(0, min(self.backoff_max, backoff_value))) 

341 

342 def parse_retry_after(self, retry_after: str) -> float: 

343 seconds: float 

344 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 

345 if re.match(r"^\s*[0-9]+\s*$", retry_after): 

346 seconds = int(retry_after) 

347 else: 

348 retry_date_tuple = email.utils.parsedate_tz(retry_after) 

349 if retry_date_tuple is None: 

350 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") 

351 

352 retry_date = email.utils.mktime_tz(retry_date_tuple) 

353 seconds = retry_date - time.time() 

354 

355 seconds = max(seconds, 0) 

356 

357 # Check the seconds do not exceed the specified maximum 

358 if seconds > self.retry_after_max: 

359 seconds = self.retry_after_max 

360 

361 return seconds 

362 

363 def get_retry_after(self, response: BaseHTTPResponse) -> float | None: 

364 """Get the value of Retry-After in seconds.""" 

365 

366 retry_after = response.headers.get("Retry-After") 

367 

368 if retry_after is None: 

369 return None 

370 

371 return self.parse_retry_after(retry_after) 

372 

373 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: 

374 retry_after = self.get_retry_after(response) 

375 if retry_after: 

376 time.sleep(retry_after) 

377 return True 

378 

379 return False 

380 

381 def _sleep_backoff(self) -> None: 

382 backoff = self.get_backoff_time() 

383 if backoff <= 0: 

384 return 

385 time.sleep(backoff) 

386 

387 def sleep(self, response: BaseHTTPResponse | None = None) -> None: 

388 """Sleep between retry attempts. 

389 

390 This method will respect a server's ``Retry-After`` response header 

391 and sleep the duration of the time requested. If that is not present, it 

392 will use an exponential backoff. By default, the backoff factor is 0 and 

393 this method will return immediately. 

394 """ 

395 

396 if self.respect_retry_after_header and response: 

397 slept = self.sleep_for_retry(response) 

398 if slept: 

399 return 

400 

401 self._sleep_backoff() 

402 

403 def _is_connection_error(self, err: Exception) -> bool: 

404 """Errors when we're fairly sure that the server did not receive the 

405 request, so it should be safe to retry. 

406 """ 

407 if isinstance(err, ProxyError): 

408 err = err.original_error 

409 return isinstance(err, ConnectTimeoutError) 

410 

411 def _is_read_error(self, err: Exception) -> bool: 

412 """Errors that occur after the request has been started, so we should 

413 assume that the server began processing it. 

414 """ 

415 return isinstance(err, (ReadTimeoutError, ProtocolError)) 

416 

417 def _is_method_retryable(self, method: str) -> bool: 

418 """Checks if a given HTTP method should be retried upon, depending if 

419 it is included in the allowed_methods 

420 """ 

421 if self.allowed_methods and method.upper() not in self.allowed_methods: 

422 return False 

423 return True 

424 

425 def is_retry( 

426 self, method: str, status_code: int, has_retry_after: bool = False 

427 ) -> bool: 

428 """Is this method/status code retryable? (Based on allowlists and control 

429 variables such as the number of total retries to allow, whether to 

430 respect the Retry-After header, whether this header is present, and 

431 whether the returned status code is on the list of status codes to 

432 be retried upon on the presence of the aforementioned header) 

433 """ 

434 if not self._is_method_retryable(method): 

435 return False 

436 

437 if self.status_forcelist and status_code in self.status_forcelist: 

438 return True 

439 

440 return bool( 

441 self.total 

442 and self.respect_retry_after_header 

443 and has_retry_after 

444 and (status_code in self.RETRY_AFTER_STATUS_CODES) 

445 ) 

446 

447 def is_exhausted(self) -> bool: 

448 """Are we out of retries?""" 

449 retry_counts = [ 

450 x 

451 for x in ( 

452 self.total, 

453 self.connect, 

454 self.read, 

455 self.redirect, 

456 self.status, 

457 self.other, 

458 ) 

459 if x 

460 ] 

461 if not retry_counts: 

462 return False 

463 

464 return min(retry_counts) < 0 

465 

466 def increment( 

467 self, 

468 method: str | None = None, 

469 url: str | None = None, 

470 response: BaseHTTPResponse | None = None, 

471 error: Exception | None = None, 

472 _pool: ConnectionPool | None = None, 

473 _stacktrace: TracebackType | None = None, 

474 ) -> Self: 

475 """Return a new Retry object with incremented retry counters. 

476 

477 :param response: A response object, or None, if the server did not 

478 return a response. 

479 :type response: :class:`~urllib3.response.BaseHTTPResponse` 

480 :param Exception error: An error encountered during the request, or 

481 None if the response was received successfully. 

482 

483 :return: A new ``Retry`` object. 

484 """ 

485 if self.total is False and error: 

486 # Disabled, indicate to re-raise the error. 

487 raise reraise(type(error), error, _stacktrace) 

488 

489 total = self.total 

490 if total is not None: 

491 total -= 1 

492 

493 connect = self.connect 

494 read = self.read 

495 redirect = self.redirect 

496 status_count = self.status 

497 other = self.other 

498 cause = "unknown" 

499 status = None 

500 redirect_location = None 

501 

502 if error and self._is_connection_error(error): 

503 # Connect retry? 

504 if connect is False: 

505 raise reraise(type(error), error, _stacktrace) 

506 elif connect is not None: 

507 connect -= 1 

508 

509 elif error and self._is_read_error(error): 

510 # Read retry? 

511 if read is False or method is None or not self._is_method_retryable(method): 

512 raise reraise(type(error), error, _stacktrace) 

513 elif read is not None: 

514 read -= 1 

515 

516 elif error: 

517 # Other retry? 

518 if other is not None: 

519 other -= 1 

520 

521 elif response and response.get_redirect_location(): 

522 # Redirect retry? 

523 if redirect is not None: 

524 redirect -= 1 

525 cause = "too many redirects" 

526 response_redirect_location = response.get_redirect_location() 

527 if response_redirect_location: 

528 redirect_location = response_redirect_location 

529 status = response.status 

530 

531 else: 

532 # Incrementing because of a server error like a 500 in 

533 # status_forcelist and the given method is in the allowed_methods 

534 cause = ResponseError.GENERIC_ERROR 

535 if response and response.status: 

536 if status_count is not None: 

537 status_count -= 1 

538 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) 

539 status = response.status 

540 

541 history = self.history + ( 

542 RequestHistory(method, url, error, status, redirect_location), 

543 ) 

544 

545 new_retry = self.new( 

546 total=total, 

547 connect=connect, 

548 read=read, 

549 redirect=redirect, 

550 status=status_count, 

551 other=other, 

552 history=history, 

553 ) 

554 

555 if new_retry.is_exhausted(): 

556 reason = error or ResponseError(cause) 

557 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] 

558 

559 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) 

560 

561 return new_retry 

562 

563 def __repr__(self) -> str: 

564 return ( 

565 f"{type(self).__name__}(total={self.total}, connect={self.connect}, " 

566 f"read={self.read}, redirect={self.redirect}, status={self.status})" 

567 ) 

568 

569 

570# For backwards compatibility (equivalent to pre-v1.9): 

571Retry.DEFAULT = Retry(3)