Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py: 35%

1from __future__ import annotations 

2 

3import email 

4import logging 

5import random 

6import re 

7import time 

8import typing 

9from itertools import takewhile 

10from types import TracebackType 

11 

12from ..exceptions import ( 

13 ConnectTimeoutError, 

14 InvalidHeader, 

15 MaxRetryError, 

16 ProtocolError, 

17 ProxyError, 

18 ReadTimeoutError, 

19 ResponseError, 

20) 

21from .util import reraise 

22 

23if typing.TYPE_CHECKING: 

24 from typing_extensions import Self 

25 

26 from ..connectionpool import ConnectionPool 

27 from ..response import BaseHTTPResponse 

28 

29log = logging.getLogger(__name__) 

30 

31 

32# Data structure for representing the metadata of requests that result in a retry. 

33class RequestHistory(typing.NamedTuple): 

34 method: str | None 

35 url: str | None 

36 error: Exception | None 

37 status: int | None 

38 redirect_location: str | None 

39 

40 

41class Retry: 

42 """Retry configuration. 

43 

44 Each retry attempt will create a new Retry object with updated values, so 

45 they can be safely reused. 

46 

47 Retries can be defined as a default for a pool: 

48 

49 .. code-block:: python 

50 

51 retries = Retry(connect=5, read=2, redirect=5) 

52 http = PoolManager(retries=retries) 

53 response = http.request("GET", "https://example.com/") 

54 

55 Or per-request (which overrides the default for the pool): 

56 

57 .. code-block:: python 

58 

59 response = http.request("GET", "https://example.com/", retries=Retry(10)) 

60 

61 Retries can be disabled by passing ``False``: 

62 

63 .. code-block:: python 

64 

65 response = http.request("GET", "https://example.com/", retries=False) 

66 

67 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless 

68 retries are disabled, in which case the causing exception will be raised. 

69 

70 :param int total: 

71 Total number of retries to allow. Takes precedence over other counts. 

72 

73 Set to ``None`` to remove this constraint and fall back on other 

74 counts. 

75 

76 Set to ``0`` to fail on the first retry. 

77 

78 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

79 

80 :param int connect: 

81 How many connection-related errors to retry on. 

82 

83 These are errors raised before the request is sent to the remote server, 

84 which we assume has not triggered the server to process the request. 

85 

86 Set to ``0`` to fail on the first retry of this type. 

87 

88 :param int read: 

89 How many times to retry on read errors. 

90 

91 These errors are raised after the request was sent to the server, so the 

92 request may have side-effects. 

93 

94 Set to ``0`` to fail on the first retry of this type. 

95 

96 :param int redirect: 

97 How many redirects to perform. Limit this to avoid infinite redirect 

98 loops. 

99 

100 A redirect is a HTTP response with a status code 301, 302, 303, 307 or 

101 308. 

102 

103 Set to ``0`` to fail on the first retry of this type. 

104 

105 Set to ``False`` to disable and imply ``raise_on_redirect=False``. 

106 

107 :param int status: 

108 How many times to retry on bad status codes. 

109 

110 These are retries made on responses, where status code matches 

111 ``status_forcelist``. 

112 

113 Set to ``0`` to fail on the first retry of this type. 

114 

115 :param int other: 

116 How many times to retry on other errors. 

117 

118 Other errors are errors that are not connect, read, redirect or status errors. 

119 These errors might be raised after the request was sent to the server, so the 

120 request might have side-effects. 

121 

122 Set to ``0`` to fail on the first retry of this type. 

123 

124 If ``total`` is not set, it's a good idea to set this to 0 to account 

125 for unexpected edge cases and avoid infinite retry loops. 

126 

127 :param Collection allowed_methods: 

128 Set of uppercased HTTP method verbs that we should retry on. 

129 

130 By default, we only retry on methods which are considered to be 

131 idempotent (multiple requests with the same parameters end with the 

132 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. 

133 

134 Set to a ``None`` value to retry on any verb. 

135 

136 :param Collection status_forcelist: 

137 A set of integer HTTP status codes that we should force a retry on. 

138 A retry is initiated if the request method is in ``allowed_methods`` 

139 and the response status code is in ``status_forcelist``. 

140 

141 By default, this is disabled with ``None``. 

142 

143 :param float backoff_factor: 

144 A backoff factor to apply between attempts after the second try 

145 (most errors are resolved immediately by a second try without a 

146 delay). urllib3 will sleep for:: 

147 

148 {backoff factor} * (2 ** ({number of previous retries})) 

149 

150 seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: 

151 

152 random.uniform(0, {backoff jitter}) 

153 

154 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will 

155 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever 

156 be longer than `backoff_max`. 

157 

158 By default, backoff is disabled (factor set to 0). 

159 

160 :param bool raise_on_redirect: Whether, if the number of redirects is 

161 exhausted, to raise a MaxRetryError, or to return a response with a 

162 response code in the 3xx range. 

163 

164 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: 

165 whether we should raise an exception, or return a response, 

166 if status falls in ``status_forcelist`` range and retries have 

167 been exhausted. 

168 

169 :param tuple history: The history of the request encountered during 

170 each call to :meth:`~Retry.increment`. The list is in the order 

171 the requests occurred. Each list item is of class :class:`RequestHistory`. 

172 

173 :param bool respect_retry_after_header: 

174 Whether to respect Retry-After header on status codes defined as 

175 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. 

176 

177 :param Collection remove_headers_on_redirect: 

178 Sequence of headers to remove from the request when a response 

179 indicating a redirect is returned before firing off the redirected 

180 request. 

181 

182 :param int retry_after_max: Number of seconds to allow as the maximum for 

183 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`. 

184 Any Retry-After headers larger than this value will be limited to this 

185 value. 

186 """ 

187 

188 #: Default methods to be used for ``allowed_methods`` 

189 DEFAULT_ALLOWED_METHODS = frozenset( 

190 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] 

191 ) 

192 

193 #: Default status codes to be used for ``status_forcelist`` 

194 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) 

195 

196 #: Default headers to be used for ``remove_headers_on_redirect`` 

197 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( 

198 ["Cookie", "Authorization", "Proxy-Authorization"] 

199 ) 

200 

201 #: Default maximum backoff time. 

202 DEFAULT_BACKOFF_MAX = 120 

203 

204 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries. 

205 #: Default maximum allowed value for Retry-After headers in seconds 

206 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600 

207 

208 # Backward compatibility; assigned outside of the class. 

209 DEFAULT: typing.ClassVar[Retry] 

210 

211 def __init__( 

212 self, 

213 total: bool | int | None = 10, 

214 connect: int | None = None, 

215 read: int | None = None, 

216 redirect: bool | int | None = None, 

217 status: int | None = None, 

218 other: int | None = None, 

219 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, 

220 status_forcelist: typing.Collection[int] | None = None, 

221 backoff_factor: float = 0, 

222 backoff_max: float = DEFAULT_BACKOFF_MAX, 

223 raise_on_redirect: bool = True, 

224 raise_on_status: bool = True, 

225 history: tuple[RequestHistory, ...] | None = None, 

226 respect_retry_after_header: bool = True, 

227 remove_headers_on_redirect: typing.Collection[ 

228 str 

229 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, 

230 backoff_jitter: float = 0.0, 

231 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX, 

232 ) -> None: 

233 self.total = total 

234 self.connect = connect 

235 self.read = read 

236 self.status = status 

237 self.other = other 

238 

239 if redirect is False or total is False: 

240 redirect = 0 

241 raise_on_redirect = False 

242 

243 self.redirect = redirect 

244 self.status_forcelist = status_forcelist or set() 

245 self.allowed_methods = allowed_methods 

246 self.backoff_factor = backoff_factor 

247 self.backoff_max = backoff_max 

248 self.retry_after_max = retry_after_max 

249 self.raise_on_redirect = raise_on_redirect 

250 self.raise_on_status = raise_on_status 

251 self.history = history or () 

252 self.respect_retry_after_header = respect_retry_after_header 

253 self.remove_headers_on_redirect = frozenset( 

254 h.lower() for h in remove_headers_on_redirect 

255 ) 

256 self.backoff_jitter = backoff_jitter 

257 

258 def new(self, **kw: typing.Any) -> Self: 

259 params = dict( 

260 total=self.total, 

261 connect=self.connect, 

262 read=self.read, 

263 redirect=self.redirect, 

264 status=self.status, 

265 other=self.other, 

266 allowed_methods=self.allowed_methods, 

267 status_forcelist=self.status_forcelist, 

268 backoff_factor=self.backoff_factor, 

269 backoff_max=self.backoff_max, 

270 retry_after_max=self.retry_after_max, 

271 raise_on_redirect=self.raise_on_redirect, 

272 raise_on_status=self.raise_on_status, 

273 history=self.history, 

274 remove_headers_on_redirect=self.remove_headers_on_redirect, 

275 respect_retry_after_header=self.respect_retry_after_header, 

276 backoff_jitter=self.backoff_jitter, 

277 ) 

278 

279 params.update(kw) 

280 return type(self)(**params) # type: ignore[arg-type] 

281 

282 @classmethod 

283 def from_int( 

284 cls, 

285 retries: Retry | bool | int | None, 

286 redirect: bool | int | None = True, 

287 default: Retry | bool | int | None = None, 

288 ) -> Retry: 

289 """Backwards-compatibility for the old retries format.""" 

290 if retries is None: 

291 retries = default if default is not None else cls.DEFAULT 

292 

293 if isinstance(retries, Retry): 

294 return retries 

295 

296 redirect = bool(redirect) and None 

297 new_retries = cls(retries, redirect=redirect) 

298 log.debug("Converted retries value: %r -> %r", retries, new_retries) 

299 return new_retries 

300 

301 def get_backoff_time(self) -> float: 

302 """Formula for computing the current backoff 

303 

304 :rtype: float 

305 """ 

306 # We want to consider only the last consecutive errors sequence (Ignore redirects). 

307 consecutive_errors_len = len( 

308 list( 

309 takewhile(lambda x: x.redirect_location is None, reversed(self.history)) 

310 ) 

311 ) 

312 if consecutive_errors_len <= 1: 

313 return 0 

314 

315 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) 

316 if self.backoff_jitter != 0.0: 

317 backoff_value += random.random() * self.backoff_jitter 

318 return float(max(0, min(self.backoff_max, backoff_value))) 

319 

320 def parse_retry_after(self, retry_after: str) -> float: 

321 seconds: float 

322 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 

323 if re.match(r"^\s*[0-9]+\s*$", retry_after): 

324 seconds = int(retry_after) 

325 else: 

326 retry_date_tuple = email.utils.parsedate_tz(retry_after) 

327 if retry_date_tuple is None: 

328 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") 

329 

330 retry_date = email.utils.mktime_tz(retry_date_tuple) 

331 seconds = retry_date - time.time() 

332 

333 seconds = max(seconds, 0) 

334 

335 # Check the seconds do not exceed the specified maximum 

336 if seconds > self.retry_after_max: 

337 seconds = self.retry_after_max 

338 

339 return seconds 

340 

341 def get_retry_after(self, response: BaseHTTPResponse) -> float | None: 

342 """Get the value of Retry-After in seconds.""" 

343 

344 retry_after = response.headers.get("Retry-After") 

345 

346 if retry_after is None: 

347 return None 

348 

349 return self.parse_retry_after(retry_after) 

350 

351 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: 

352 retry_after = self.get_retry_after(response) 

353 if retry_after: 

354 time.sleep(retry_after) 

355 return True 

356 

357 return False 

358 

359 def _sleep_backoff(self) -> None: 

360 backoff = self.get_backoff_time() 

361 if backoff <= 0: 

362 return 

363 time.sleep(backoff) 

364 

365 def sleep(self, response: BaseHTTPResponse | None = None) -> None: 

366 """Sleep between retry attempts. 

367 

368 This method will respect a server's ``Retry-After`` response header 

369 and sleep the duration of the time requested. If that is not present, it 

370 will use an exponential backoff. By default, the backoff factor is 0 and 

371 this method will return immediately. 

372 """ 

373 

374 if self.respect_retry_after_header and response: 

375 slept = self.sleep_for_retry(response) 

376 if slept: 

377 return 

378 

379 self._sleep_backoff() 

380 

381 def _is_connection_error(self, err: Exception) -> bool: 

382 """Errors when we're fairly sure that the server did not receive the 

383 request, so it should be safe to retry. 

384 """ 

385 if isinstance(err, ProxyError): 

386 err = err.original_error 

387 return isinstance(err, ConnectTimeoutError) 

388 

389 def _is_read_error(self, err: Exception) -> bool: 

390 """Errors that occur after the request has been started, so we should 

391 assume that the server began processing it. 

392 """ 

393 return isinstance(err, (ReadTimeoutError, ProtocolError)) 

394 

395 def _is_method_retryable(self, method: str) -> bool: 

396 """Checks if a given HTTP method should be retried upon, depending if 

397 it is included in the allowed_methods 

398 """ 

399 if self.allowed_methods and method.upper() not in self.allowed_methods: 

400 return False 

401 return True 

402 

403 def is_retry( 

404 self, method: str, status_code: int, has_retry_after: bool = False 

405 ) -> bool: 

406 """Is this method/status code retryable? (Based on allowlists and control 

407 variables such as the number of total retries to allow, whether to 

408 respect the Retry-After header, whether this header is present, and 

409 whether the returned status code is on the list of status codes to 

410 be retried upon on the presence of the aforementioned header) 

411 """ 

412 if not self._is_method_retryable(method): 

413 return False 

414 

415 if self.status_forcelist and status_code in self.status_forcelist: 

416 return True 

417 

418 return bool( 

419 self.total 

420 and self.respect_retry_after_header 

421 and has_retry_after 

422 and (status_code in self.RETRY_AFTER_STATUS_CODES) 

423 ) 

424 

425 def is_exhausted(self) -> bool: 

426 """Are we out of retries?""" 

427 retry_counts = [ 

428 x 

429 for x in ( 

430 self.total, 

431 self.connect, 

432 self.read, 

433 self.redirect, 

434 self.status, 

435 self.other, 

436 ) 

437 if x 

438 ] 

439 if not retry_counts: 

440 return False 

441 

442 return min(retry_counts) < 0 

443 

444 def increment( 

445 self, 

446 method: str | None = None, 

447 url: str | None = None, 

448 response: BaseHTTPResponse | None = None, 

449 error: Exception | None = None, 

450 _pool: ConnectionPool | None = None, 

451 _stacktrace: TracebackType | None = None, 

452 ) -> Self: 

453 """Return a new Retry object with incremented retry counters. 

454 

455 :param response: A response object, or None, if the server did not 

456 return a response. 

457 :type response: :class:`~urllib3.response.BaseHTTPResponse` 

458 :param Exception error: An error encountered during the request, or 

459 None if the response was received successfully. 

460 

461 :return: A new ``Retry`` object. 

462 """ 

463 if self.total is False and error: 

464 # Disabled, indicate to re-raise the error. 

465 raise reraise(type(error), error, _stacktrace) 

466 

467 total = self.total 

468 if total is not None: 

469 total -= 1 

470 

471 connect = self.connect 

472 read = self.read 

473 redirect = self.redirect 

474 status_count = self.status 

475 other = self.other 

476 cause = "unknown" 

477 status = None 

478 redirect_location = None 

479 

480 if error and self._is_connection_error(error): 

481 # Connect retry? 

482 if connect is False: 

483 raise reraise(type(error), error, _stacktrace) 

484 elif connect is not None: 

485 connect -= 1 

486 

487 elif error and self._is_read_error(error): 

488 # Read retry? 

489 if read is False or method is None or not self._is_method_retryable(method): 

490 raise reraise(type(error), error, _stacktrace) 

491 elif read is not None: 

492 read -= 1 

493 

494 elif error: 

495 # Other retry? 

496 if other is not None: 

497 other -= 1 

498 

499 elif response and response.get_redirect_location(): 

500 # Redirect retry? 

501 if redirect is not None: 

502 redirect -= 1 

503 cause = "too many redirects" 

504 response_redirect_location = response.get_redirect_location() 

505 if response_redirect_location: 

506 redirect_location = response_redirect_location 

507 status = response.status 

508 

509 else: 

510 # Incrementing because of a server error like a 500 in 

511 # status_forcelist and the given method is in the allowed_methods 

512 cause = ResponseError.GENERIC_ERROR 

513 if response and response.status: 

514 if status_count is not None: 

515 status_count -= 1 

516 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) 

517 status = response.status 

518 

519 history = self.history + ( 

520 RequestHistory(method, url, error, status, redirect_location), 

521 ) 

522 

523 new_retry = self.new( 

524 total=total, 

525 connect=connect, 

526 read=read, 

527 redirect=redirect, 

528 status=status_count, 

529 other=other, 

530 history=history, 

531 ) 

532 

533 if new_retry.is_exhausted(): 

534 reason = error or ResponseError(cause) 

535 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] 

536 

537 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) 

538 

539 return new_retry 

540 

541 def __repr__(self) -> str: 

542 return ( 

543 f"{type(self).__name__}(total={self.total}, connect={self.connect}, " 

544 f"read={self.read}, redirect={self.redirect}, status={self.status})" 

545 ) 

546 

547 

548# For backwards compatibility (equivalent to pre-v1.9): 

549Retry.DEFAULT = Retry(3)