Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py: 36%

1from __future__ import annotations 

2 

3import email 

4import logging 

5import random 

6import re 

7import time 

8import typing 

9import warnings 

10from collections.abc import Collection 

11from itertools import takewhile 

12from types import TracebackType 

13 

14from ..exceptions import ( 

15 ConnectTimeoutError, 

16 InvalidHeader, 

17 MaxRetryError, 

18 ProtocolError, 

19 ProxyError, 

20 ReadTimeoutError, 

21 ResponseError, 

22) 

23from .util import reraise 

24 

25if typing.TYPE_CHECKING: 

26 from typing_extensions import Self 

27 

28 from ..connectionpool import ConnectionPool 

29 from ..response import BaseHTTPResponse 

30 

31log = logging.getLogger(__name__) 

32 

33 

34# Data structure for representing the metadata of requests that result in a retry. 

35class RequestHistory(typing.NamedTuple): 

36 method: str | None 

37 url: str | None 

38 error: Exception | None 

39 status: int | None 

40 redirect_location: str | None 

41 

42 

43class Retry: 

44 """Retry configuration. 

45 

46 Each retry attempt will create a new Retry object with updated values, so 

47 they can be safely reused. 

48 

49 Retries can be defined as a default for a pool: 

50 

51 .. code-block:: python 

52 

53 retries = Retry(connect=5, read=2, redirect=5) 

54 http = PoolManager(retries=retries) 

55 response = http.request("GET", "https://example.com/") 

56 

57 Or per-request (which overrides the default for the pool): 

58 

59 .. code-block:: python 

60 

61 response = http.request("GET", "https://example.com/", retries=Retry(10)) 

62 

63 Retries can be disabled by passing ``False``: 

64 

65 .. code-block:: python 

66 

67 response = http.request("GET", "https://example.com/", retries=False) 

68 

69 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless 

70 retries are disabled, in which case the causing exception will be raised. 

71 

72 :param int total: 

73 Total number of retries to allow. Takes precedence over other counts. 

74 

75 Set to ``None`` to remove this constraint and fall back on other 

76 counts. 

77 

78 Set to ``0`` to fail on the first retry. 

79 

80 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

81 

82 :param int connect: 

83 How many connection-related errors to retry on. 

84 

85 These are errors raised before the request is sent to the remote server, 

86 which we assume has not triggered the server to process the request. 

87 

88 Set to ``0`` to fail on the first retry of this type. 

89 

90 :param int read: 

91 How many times to retry on read errors. 

92 

93 These errors are raised after the request was sent to the server, so the 

94 request may have side-effects. 

95 

96 Set to ``0`` to fail on the first retry of this type. 

97 

98 :param int redirect: 

99 How many redirects to perform. Limit this to avoid infinite redirect 

100 loops. 

101 

102 A redirect is a HTTP response with a status code 301, 302, 303, 307 or 

103 308. 

104 

105 Set to ``0`` to fail on the first retry of this type. 

106 

107 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

108 

109 :param int status: 

110 How many times to retry on bad status codes. 

111 

112 These are retries made on responses, where status code matches 

113 ``status_forcelist``. 

114 

115 Set to ``0`` to fail on the first retry of this type. 

116 

117 :param int other: 

118 How many times to retry on other errors. 

119 

120 Other errors are errors that are not connect, read, redirect or status errors. 

121 These errors might be raised after the request was sent to the server, so the 

122 request might have side-effects. 

123 

124 Set to ``0`` to fail on the first retry of this type. 

125 

126 If ``total`` is not set, it's a good idea to set this to 0 to account 

127 for unexpected edge cases and avoid infinite retry loops. 

128 

129 :param Collection allowed_methods: 

130 Set of uppercased HTTP method verbs that we should retry on. 

131 

132 By default, we only retry on methods which are considered to be 

133 idempotent (multiple requests with the same parameters end with the 

134 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. 

135 

136 Set to a ``None`` value to retry on any verb. 

137 

138 :param Collection status_forcelist: 

139 A set of integer HTTP status codes that we should force a retry on. 

140 A retry is initiated if the request method is in ``allowed_methods`` 

141 and the response status code is in ``status_forcelist``. 

142 

143 By default, this is disabled with ``None``. 

144 

145 :param float backoff_factor: 

146 A backoff factor to apply between attempts after the second try 

147 (most errors are resolved immediately by a second try without a 

148 delay). urllib3 will sleep for:: 

149 

150 {backoff factor} * (2 ** ({number of previous retries})) 

151 

152 seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: 

153 

154 random.uniform(0, {backoff jitter}) 

155 

156 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will 

157 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever 

158 be longer than `backoff_max`. 

159 

160 By default, backoff is disabled (factor set to 0). 

161 

162 :param float backoff_max: 

163 The maximum backoff time (in seconds) between retry attempts. 

164 This value caps the computed backoff from `backoff_factor`. 

165 

166 :param float backoff_jitter: 

167 Random jitter amount (in seconds) added to the computed backoff. 

168 Jitter is sampled uniformly from `0` to `backoff_jitter`. 

169 

170 :param bool raise_on_redirect: Whether, if the number of redirects is 

171 exhausted, to raise a MaxRetryError, or to return a response with a 

172 response code in the 3xx range. 

173 

174 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: 

175 whether we should raise an exception, or return a response, 

176 if status falls in ``status_forcelist`` range and retries have 

177 been exhausted. 

178 

179 :param tuple history: The history of the request encountered during 

180 each call to :meth:`~Retry.increment`. The list is in the order 

181 the requests occurred. Each list item is of class :class:`RequestHistory`. 

182 

183 :param bool respect_retry_after_header: 

184 Whether to respect Retry-After header on status codes defined as 

185 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. 

186 

187 :param Collection remove_headers_on_redirect: 

188 Sequence of headers to remove from the request when a response 

189 indicating a redirect is returned before firing off the redirected 

190 request. 

191 

192 :param int retry_after_max: Number of seconds to allow as the maximum for 

193 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`. 

194 Any Retry-After headers larger than this value will be limited to this 

195 value. 

196 """ 

197 

198 #: Default methods to be used for ``allowed_methods`` 

199 DEFAULT_ALLOWED_METHODS = frozenset( 

200 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] 

201 ) 

202 

203 #: Default status codes that trigger a retry when a Retry-After header is 

204 #: present and :attr:`Retry.respect_retry_after_header` is enabled. 

205 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) 

206 

207 #: Default headers to be used for ``remove_headers_on_redirect`` 

208 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( 

209 ["Cookie", "Authorization", "Proxy-Authorization"] 

210 ) 

211 

212 #: Default maximum backoff time. 

213 DEFAULT_BACKOFF_MAX = 120 

214 

215 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries. 

216 #: Default maximum allowed value for Retry-After headers in seconds 

217 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600 

218 

219 # Backward compatibility; assigned outside of the class. 

220 DEFAULT: typing.ClassVar[Retry] 

221 

222 def __init__( 

223 self, 

224 total: bool | int | None = 10, 

225 connect: int | None = None, 

226 read: int | None = None, 

227 redirect: bool | int | None = None, 

228 status: int | None = None, 

229 other: int | None = None, 

230 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, 

231 status_forcelist: typing.Collection[int] | None = None, 

232 backoff_factor: float = 0, 

233 backoff_max: float = DEFAULT_BACKOFF_MAX, 

234 raise_on_redirect: bool = True, 

235 raise_on_status: bool = True, 

236 history: tuple[RequestHistory, ...] | None = None, 

237 respect_retry_after_header: bool = True, 

238 remove_headers_on_redirect: typing.Collection[ 

239 str 

240 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, 

241 backoff_jitter: float = 0.0, 

242 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX, 

243 ) -> None: 

244 self.total = total 

245 self.connect = connect 

246 self.read = read 

247 self.status = status 

248 self.other = other 

249 

250 if redirect is False or total is False: 

251 redirect = 0 

252 raise_on_redirect = False 

253 

254 self.redirect = redirect 

255 self.status_forcelist = status_forcelist or set() 

256 if not allowed_methods and isinstance(allowed_methods, Collection): 

257 warnings.warn( 

258 "Using an empty collection for 'allowed_methods' option to " 

259 "retry on any verb is deprecated and will skip retries for " 

260 "all verbs in urllib3 v3.0. Instead use " 

261 "Retry(..., allowed_methods=None).", 

262 FutureWarning, 

263 stacklevel=2, 

264 ) 

265 self.allowed_methods = allowed_methods 

266 self.backoff_factor = backoff_factor 

267 self.backoff_max = backoff_max 

268 self.retry_after_max = retry_after_max 

269 self.raise_on_redirect = raise_on_redirect 

270 self.raise_on_status = raise_on_status 

271 self.history = history or () 

272 self.respect_retry_after_header = respect_retry_after_header 

273 self.remove_headers_on_redirect = frozenset( 

274 h.lower() for h in remove_headers_on_redirect 

275 ) 

276 self.backoff_jitter = backoff_jitter 

277 

278 def new(self, **kw: typing.Any) -> Self: 

279 params = dict( 

280 total=self.total, 

281 connect=self.connect, 

282 read=self.read, 

283 redirect=self.redirect, 

284 status=self.status, 

285 other=self.other, 

286 allowed_methods=self.allowed_methods, 

287 status_forcelist=self.status_forcelist, 

288 backoff_factor=self.backoff_factor, 

289 backoff_max=self.backoff_max, 

290 retry_after_max=self.retry_after_max, 

291 raise_on_redirect=self.raise_on_redirect, 

292 raise_on_status=self.raise_on_status, 

293 history=self.history, 

294 remove_headers_on_redirect=self.remove_headers_on_redirect, 

295 respect_retry_after_header=self.respect_retry_after_header, 

296 backoff_jitter=self.backoff_jitter, 

297 ) 

298 

299 params.update(kw) 

300 return type(self)(**params) # type: ignore[arg-type] 

301 

302 @classmethod 

303 def from_int( 

304 cls, 

305 retries: Retry | bool | int | None, 

306 redirect: bool | int | None = True, 

307 default: Retry | bool | int | None = None, 

308 ) -> Retry: 

309 """Backwards-compatibility for the old retries format.""" 

310 if retries is None: 

311 retries = default if default is not None else cls.DEFAULT 

312 

313 if isinstance(retries, Retry): 

314 return retries 

315 

316 redirect = bool(redirect) and None 

317 new_retries = cls(retries, redirect=redirect) 

318 log.debug("Converted retries value: %r -> %r", retries, new_retries) 

319 return new_retries 

320 

321 def get_backoff_time(self) -> float: 

322 """Formula for computing the current backoff 

323 

324 :rtype: float 

325 """ 

326 # We want to consider only the last consecutive errors sequence (Ignore redirects). 

327 consecutive_errors_len = len( 

328 list( 

329 takewhile(lambda x: x.redirect_location is None, reversed(self.history)) 

330 ) 

331 ) 

332 if consecutive_errors_len <= 1: 

333 return 0 

334 

335 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) 

336 if self.backoff_jitter != 0.0: 

337 backoff_value += random.random() * self.backoff_jitter 

338 return float(max(0, min(self.backoff_max, backoff_value))) 

339 

340 def parse_retry_after(self, retry_after: str) -> float: 

341 seconds: float 

342 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 

343 if re.match(r"^\s*[0-9]+\s*$", retry_after): 

344 seconds = int(retry_after) 

345 else: 

346 retry_date_tuple = email.utils.parsedate_tz(retry_after) 

347 if retry_date_tuple is None: 

348 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") 

349 

350 retry_date = email.utils.mktime_tz(retry_date_tuple) 

351 seconds = retry_date - time.time() 

352 

353 seconds = max(seconds, 0) 

354 

355 # Check the seconds do not exceed the specified maximum 

356 if seconds > self.retry_after_max: 

357 seconds = self.retry_after_max 

358 

359 return seconds 

360 

361 def get_retry_after(self, response: BaseHTTPResponse) -> float | None: 

362 """Get the value of Retry-After in seconds.""" 

363 

364 retry_after = response.headers.get("Retry-After") 

365 

366 if retry_after is None: 

367 return None 

368 

369 return self.parse_retry_after(retry_after) 

370 

371 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: 

372 retry_after = self.get_retry_after(response) 

373 if retry_after: 

374 time.sleep(retry_after) 

375 return True 

376 

377 return False 

378 

379 def _sleep_backoff(self) -> None: 

380 backoff = self.get_backoff_time() 

381 if backoff <= 0: 

382 return 

383 time.sleep(backoff) 

384 

385 def sleep(self, response: BaseHTTPResponse | None = None) -> None: 

386 """Sleep between retry attempts. 

387 

388 This method will respect a server's ``Retry-After`` response header 

389 and sleep the duration of the time requested. If that is not present, it 

390 will use an exponential backoff. By default, the backoff factor is 0 and 

391 this method will return immediately. 

392 """ 

393 

394 if self.respect_retry_after_header and response: 

395 slept = self.sleep_for_retry(response) 

396 if slept: 

397 return 

398 

399 self._sleep_backoff() 

400 

401 def _is_connection_error(self, err: Exception) -> bool: 

402 """Errors when we're fairly sure that the server did not receive the 

403 request, so it should be safe to retry. 

404 """ 

405 if isinstance(err, ProxyError): 

406 err = err.original_error 

407 return isinstance(err, ConnectTimeoutError) 

408 

409 def _is_read_error(self, err: Exception) -> bool: 

410 """Errors that occur after the request has been started, so we should 

411 assume that the server began processing it. 

412 """ 

413 return isinstance(err, (ReadTimeoutError, ProtocolError)) 

414 

415 def _is_method_retryable(self, method: str) -> bool: 

416 """Checks if a given HTTP method should be retried upon, depending if 

417 it is included in the allowed_methods 

418 """ 

419 if self.allowed_methods and method.upper() not in self.allowed_methods: 

420 return False 

421 return True 

422 

423 def is_retry( 

424 self, method: str, status_code: int, has_retry_after: bool = False 

425 ) -> bool: 

426 """Is this method/status code retryable? (Based on allowlists and control 

427 variables such as the number of total retries to allow, whether to 

428 respect the Retry-After header, whether this header is present, and 

429 whether the returned status code is on the list of status codes to 

430 be retried upon on the presence of the aforementioned header) 

431 """ 

432 if not self._is_method_retryable(method): 

433 return False 

434 

435 if self.status_forcelist and status_code in self.status_forcelist: 

436 return True 

437 

438 return bool( 

439 self.total 

440 and self.respect_retry_after_header 

441 and has_retry_after 

442 and (status_code in self.RETRY_AFTER_STATUS_CODES) 

443 ) 

444 

445 def is_exhausted(self) -> bool: 

446 """Are we out of retries?""" 

447 retry_counts = [ 

448 x 

449 for x in ( 

450 self.total, 

451 self.connect, 

452 self.read, 

453 self.redirect, 

454 self.status, 

455 self.other, 

456 ) 

457 if x 

458 ] 

459 if not retry_counts: 

460 return False 

461 

462 return min(retry_counts) < 0 

463 

464 def increment( 

465 self, 

466 method: str | None = None, 

467 url: str | None = None, 

468 response: BaseHTTPResponse | None = None, 

469 error: Exception | None = None, 

470 _pool: ConnectionPool | None = None, 

471 _stacktrace: TracebackType | None = None, 

472 ) -> Self: 

473 """Return a new Retry object with incremented retry counters. 

474 

475 :param response: A response object, or None, if the server did not 

476 return a response. 

477 :type response: :class:`~urllib3.response.BaseHTTPResponse` 

478 :param Exception error: An error encountered during the request, or 

479 None if the response was received successfully. 

480 

481 :return: A new ``Retry`` object. 

482 """ 

483 if self.total is False and error: 

484 # Disabled, indicate to re-raise the error. 

485 raise reraise(type(error), error, _stacktrace) 

486 

487 total = self.total 

488 if total is not None: 

489 total -= 1 

490 

491 connect = self.connect 

492 read = self.read 

493 redirect = self.redirect 

494 status_count = self.status 

495 other = self.other 

496 cause = "unknown" 

497 status = None 

498 redirect_location = None 

499 

500 if error and self._is_connection_error(error): 

501 # Connect retry? 

502 if connect is False: 

503 raise reraise(type(error), error, _stacktrace) 

504 elif connect is not None: 

505 connect -= 1 

506 

507 elif error and self._is_read_error(error): 

508 # Read retry? 

509 if read is False or method is None or not self._is_method_retryable(method): 

510 raise reraise(type(error), error, _stacktrace) 

511 elif read is not None: 

512 read -= 1 

513 

514 elif error: 

515 # Other retry? 

516 if other is not None: 

517 other -= 1 

518 

519 elif response and response.get_redirect_location(): 

520 # Redirect retry? 

521 if redirect is not None: 

522 redirect -= 1 

523 cause = "too many redirects" 

524 response_redirect_location = response.get_redirect_location() 

525 if response_redirect_location: 

526 redirect_location = response_redirect_location 

527 status = response.status 

528 

529 else: 

530 # Incrementing because of a server error like a 500 in 

531 # status_forcelist and the given method is in the allowed_methods 

532 cause = ResponseError.GENERIC_ERROR 

533 if response and response.status: 

534 if status_count is not None: 

535 status_count -= 1 

536 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) 

537 status = response.status 

538 

539 history = self.history + ( 

540 RequestHistory(method, url, error, status, redirect_location), 

541 ) 

542 

543 new_retry = self.new( 

544 total=total, 

545 connect=connect, 

546 read=read, 

547 redirect=redirect, 

548 status=status_count, 

549 other=other, 

550 history=history, 

551 ) 

552 

553 if new_retry.is_exhausted(): 

554 reason = error or ResponseError(cause) 

555 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] 

556 

557 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) 

558 

559 return new_retry 

560 

561 def __repr__(self) -> str: 

562 return ( 

563 f"{type(self).__name__}(total={self.total}, connect={self.connect}, " 

564 f"read={self.read}, redirect={self.redirect}, status={self.status})" 

565 ) 

566 

567 

568# For backwards compatibility (equivalent to pre-v1.9): 

569Retry.DEFAULT = Retry(3)