Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py: 35%

1from __future__ import annotations

3import email

4import logging

5import random

6import re

7import time

8import typing

9from itertools import takewhile

10from types import TracebackType

12from ..exceptions import (

13 ConnectTimeoutError,

14 InvalidHeader,

15 MaxRetryError,

16 ProtocolError,

17 ProxyError,

18 ReadTimeoutError,

19 ResponseError,

20)

21from .util import reraise

23if typing.TYPE_CHECKING:

24 from typing_extensions import Self

26 from ..connectionpool import ConnectionPool

27 from ..response import BaseHTTPResponse

29log = logging.getLogger(__name__)

32# Data structure for representing the metadata of requests that result in a retry.

33class RequestHistory(typing.NamedTuple):

34 method: str | None

35 url: str | None

36 error: Exception | None

37 status: int | None

38 redirect_location: str | None

41class Retry:

42 """Retry configuration.

44 Each retry attempt will create a new Retry object with updated values, so

45 they can be safely reused.

47 Retries can be defined as a default for a pool:

49 .. code-block:: python

51 retries = Retry(connect=5, read=2, redirect=5)

52 http = PoolManager(retries=retries)

53 response = http.request("GET", "https://example.com/")

55 Or per-request (which overrides the default for the pool):

57 .. code-block:: python

59 response = http.request("GET", "https://example.com/", retries=Retry(10))

61 Retries can be disabled by passing ``False``:

63 .. code-block:: python

65 response = http.request("GET", "https://example.com/", retries=False)

67 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless

68 retries are disabled, in which case the causing exception will be raised.

70 :param int total:

71 Total number of retries to allow. Takes precedence over other counts.

73 Set to ``None`` to remove this constraint and fall back on other

74 counts.

76 Set to ``0`` to fail on the first retry.

78 Set to ``False`` to disable and imply ``raise_on_redirect=False``.

80 :param int connect:

81 How many connection-related errors to retry on.

83 These are errors raised before the request is sent to the remote server,

84 which we assume has not triggered the server to process the request.

86 Set to ``0`` to fail on the first retry of this type.

88 :param int read:

89 How many times to retry on read errors.

91 These errors are raised after the request was sent to the server, so the

92 request may have side-effects.

94 Set to ``0`` to fail on the first retry of this type.

96 :param int redirect:

97 How many redirects to perform. Limit this to avoid infinite redirect

98 loops.

100 A redirect is a HTTP response with a status code 301, 302, 303, 307 or

101 308.

102

103 Set to ``0`` to fail on the first retry of this type.

104

105 Set to ``False`` to disable and imply ``raise_on_redirect=False``.

106

107 :param int status:

108 How many times to retry on bad status codes.

109

110 These are retries made on responses, where status code matches

111 ``status_forcelist``.

112

113 Set to ``0`` to fail on the first retry of this type.

114

115 :param int other:

116 How many times to retry on other errors.

117

118 Other errors are errors that are not connect, read, redirect or status errors.

119 These errors might be raised after the request was sent to the server, so the

120 request might have side-effects.

121

122 Set to ``0`` to fail on the first retry of this type.

123

124 If ``total`` is not set, it's a good idea to set this to 0 to account

125 for unexpected edge cases and avoid infinite retry loops.

126

127 :param Collection allowed_methods:

128 Set of uppercased HTTP method verbs that we should retry on.

129

130 By default, we only retry on methods which are considered to be

131 idempotent (multiple requests with the same parameters end with the

132 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.

133

134 Set to a ``None`` value to retry on any verb.

135

136 :param Collection status_forcelist:

137 A set of integer HTTP status codes that we should force a retry on.

138 A retry is initiated if the request method is in ``allowed_methods``

139 and the response status code is in ``status_forcelist``.

140

141 By default, this is disabled with ``None``.

142

143 :param float backoff_factor:

144 A backoff factor to apply between attempts after the second try

145 (most errors are resolved immediately by a second try without a

146 delay). urllib3 will sleep for::

147

148 {backoff factor} * (2 ** ({number of previous retries}))

149

150 seconds. If `backoff_jitter` is non-zero, this sleep is extended by::

151

152 random.uniform(0, {backoff jitter})

153

154 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will

155 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever

156 be longer than `backoff_max`.

157

158 By default, backoff is disabled (factor set to 0).

159

160 :param float backoff_max:

161 The maximum backoff time (in seconds) between retry attempts.

162 This value caps the computed backoff from `backoff_factor`.

163

164 :param float backoff_jitter:

165 Random jitter amount (in seconds) added to the computed backoff.

166 Jitter is sampled uniformly from `0` to `backoff_jitter`.

167

168 :param bool raise_on_redirect: Whether, if the number of redirects is

169 exhausted, to raise a MaxRetryError, or to return a response with a

170 response code in the 3xx range.

171

172 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:

173 whether we should raise an exception, or return a response,

174 if status falls in ``status_forcelist`` range and retries have

175 been exhausted.

176

177 :param tuple history: The history of the request encountered during

178 each call to :meth:`~Retry.increment`. The list is in the order

179 the requests occurred. Each list item is of class :class:`RequestHistory`.

180

181 :param bool respect_retry_after_header:

182 Whether to respect Retry-After header on status codes defined as

183 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.

184

185 :param Collection remove_headers_on_redirect:

186 Sequence of headers to remove from the request when a response

187 indicating a redirect is returned before firing off the redirected

188 request.

189

190 :param int retry_after_max: Number of seconds to allow as the maximum for

191 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`.

192 Any Retry-After headers larger than this value will be limited to this

193 value.

194 """

195

196 #: Default methods to be used for ``allowed_methods``

197 DEFAULT_ALLOWED_METHODS = frozenset(

198 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]

199 )

200

201 #: Default status codes to be used for ``status_forcelist``

202 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])

203

204 #: Default headers to be used for ``remove_headers_on_redirect``

205 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(

206 ["Cookie", "Authorization", "Proxy-Authorization"]

207 )

208

209 #: Default maximum backoff time.

210 DEFAULT_BACKOFF_MAX = 120

211

212 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries.

213 #: Default maximum allowed value for Retry-After headers in seconds

214 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600

215

216 # Backward compatibility; assigned outside of the class.

217 DEFAULT: typing.ClassVar[Retry]

218

219 def __init__(

220 self,

221 total: bool | int | None = 10,

222 connect: int | None = None,

223 read: int | None = None,

224 redirect: bool | int | None = None,

225 status: int | None = None,

226 other: int | None = None,

227 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,

228 status_forcelist: typing.Collection[int] | None = None,

229 backoff_factor: float = 0,

230 backoff_max: float = DEFAULT_BACKOFF_MAX,

231 raise_on_redirect: bool = True,

232 raise_on_status: bool = True,

233 history: tuple[RequestHistory, ...] | None = None,

234 respect_retry_after_header: bool = True,

235 remove_headers_on_redirect: typing.Collection[

236 str

237 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,

238 backoff_jitter: float = 0.0,

239 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX,

240 ) -> None:

241 self.total = total

242 self.connect = connect

243 self.read = read

244 self.status = status

245 self.other = other

246

247 if redirect is False or total is False:

248 redirect = 0

249 raise_on_redirect = False

250

251 self.redirect = redirect

252 self.status_forcelist = status_forcelist or set()

253 self.allowed_methods = allowed_methods

254 self.backoff_factor = backoff_factor

255 self.backoff_max = backoff_max

256 self.retry_after_max = retry_after_max

257 self.raise_on_redirect = raise_on_redirect

258 self.raise_on_status = raise_on_status

259 self.history = history or ()

260 self.respect_retry_after_header = respect_retry_after_header

261 self.remove_headers_on_redirect = frozenset(

262 h.lower() for h in remove_headers_on_redirect

263 )

264 self.backoff_jitter = backoff_jitter

265

266 def new(self, **kw: typing.Any) -> Self:

267 params = dict(

268 total=self.total,

269 connect=self.connect,

270 read=self.read,

271 redirect=self.redirect,

272 status=self.status,

273 other=self.other,

274 allowed_methods=self.allowed_methods,

275 status_forcelist=self.status_forcelist,

276 backoff_factor=self.backoff_factor,

277 backoff_max=self.backoff_max,

278 retry_after_max=self.retry_after_max,

279 raise_on_redirect=self.raise_on_redirect,

280 raise_on_status=self.raise_on_status,

281 history=self.history,

282 remove_headers_on_redirect=self.remove_headers_on_redirect,

283 respect_retry_after_header=self.respect_retry_after_header,

284 backoff_jitter=self.backoff_jitter,

285 )

286

287 params.update(kw)

288 return type(self)(**params) # type: ignore[arg-type]

289

290 @classmethod

291 def from_int(

292 cls,

293 retries: Retry | bool | int | None,

294 redirect: bool | int | None = True,

295 default: Retry | bool | int | None = None,

296 ) -> Retry:

297 """Backwards-compatibility for the old retries format."""

298 if retries is None:

299 retries = default if default is not None else cls.DEFAULT

300

301 if isinstance(retries, Retry):

302 return retries

303

304 redirect = bool(redirect) and None

305 new_retries = cls(retries, redirect=redirect)

306 log.debug("Converted retries value: %r -> %r", retries, new_retries)

307 return new_retries

308

309 def get_backoff_time(self) -> float:

310 """Formula for computing the current backoff

311

312 :rtype: float

313 """

314 # We want to consider only the last consecutive errors sequence (Ignore redirects).

315 consecutive_errors_len = len(

316 list(

317 takewhile(lambda x: x.redirect_location is None, reversed(self.history))

318 )

319 )

320 if consecutive_errors_len <= 1:

321 return 0

322

323 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))

324 if self.backoff_jitter != 0.0:

325 backoff_value += random.random() * self.backoff_jitter

326 return float(max(0, min(self.backoff_max, backoff_value)))

327

328 def parse_retry_after(self, retry_after: str) -> float:

329 seconds: float

330 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4

331 if re.match(r"^\s*[0-9]+\s*$", retry_after):

332 seconds = int(retry_after)

333 else:

334 retry_date_tuple = email.utils.parsedate_tz(retry_after)

335 if retry_date_tuple is None:

336 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")

337

338 retry_date = email.utils.mktime_tz(retry_date_tuple)

339 seconds = retry_date - time.time()

340

341 seconds = max(seconds, 0)

342

343 # Check the seconds do not exceed the specified maximum

344 if seconds > self.retry_after_max:

345 seconds = self.retry_after_max

346

347 return seconds

348

349 def get_retry_after(self, response: BaseHTTPResponse) -> float | None:

350 """Get the value of Retry-After in seconds."""

351

352 retry_after = response.headers.get("Retry-After")

353

354 if retry_after is None:

355 return None

356

357 return self.parse_retry_after(retry_after)

358

359 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:

360 retry_after = self.get_retry_after(response)

361 if retry_after:

362 time.sleep(retry_after)

363 return True

364

365 return False

366

367 def _sleep_backoff(self) -> None:

368 backoff = self.get_backoff_time()

369 if backoff <= 0:

370 return

371 time.sleep(backoff)

372

373 def sleep(self, response: BaseHTTPResponse | None = None) -> None:

374 """Sleep between retry attempts.

375

376 This method will respect a server's ``Retry-After`` response header

377 and sleep the duration of the time requested. If that is not present, it

378 will use an exponential backoff. By default, the backoff factor is 0 and

379 this method will return immediately.

380 """

381

382 if self.respect_retry_after_header and response:

383 slept = self.sleep_for_retry(response)

384 if slept:

385 return

386

387 self._sleep_backoff()

388

389 def _is_connection_error(self, err: Exception) -> bool:

390 """Errors when we're fairly sure that the server did not receive the

391 request, so it should be safe to retry.

392 """

393 if isinstance(err, ProxyError):

394 err = err.original_error

395 return isinstance(err, ConnectTimeoutError)

396

397 def _is_read_error(self, err: Exception) -> bool:

398 """Errors that occur after the request has been started, so we should

399 assume that the server began processing it.

400 """

401 return isinstance(err, (ReadTimeoutError, ProtocolError))

402

403 def _is_method_retryable(self, method: str) -> bool:

404 """Checks if a given HTTP method should be retried upon, depending if

405 it is included in the allowed_methods

406 """

407 if self.allowed_methods and method.upper() not in self.allowed_methods:

408 return False

409 return True

410

411 def is_retry(

412 self, method: str, status_code: int, has_retry_after: bool = False

413 ) -> bool:

414 """Is this method/status code retryable? (Based on allowlists and control

415 variables such as the number of total retries to allow, whether to

416 respect the Retry-After header, whether this header is present, and

417 whether the returned status code is on the list of status codes to

418 be retried upon on the presence of the aforementioned header)

419 """

420 if not self._is_method_retryable(method):

421 return False

422

423 if self.status_forcelist and status_code in self.status_forcelist:

424 return True

425

426 return bool(

427 self.total

428 and self.respect_retry_after_header

429 and has_retry_after

430 and (status_code in self.RETRY_AFTER_STATUS_CODES)

431 )

432

433 def is_exhausted(self) -> bool:

434 """Are we out of retries?"""

435 retry_counts = [

436 x

437 for x in (

438 self.total,

439 self.connect,

440 self.read,

441 self.redirect,

442 self.status,

443 self.other,

444 )

445 if x

446 ]

447 if not retry_counts:

448 return False

449

450 return min(retry_counts) < 0

451

452 def increment(

453 self,

454 method: str | None = None,

455 url: str | None = None,

456 response: BaseHTTPResponse | None = None,

457 error: Exception | None = None,

458 _pool: ConnectionPool | None = None,

459 _stacktrace: TracebackType | None = None,

460 ) -> Self:

461 """Return a new Retry object with incremented retry counters.

462

463 :param response: A response object, or None, if the server did not

464 return a response.

465 :type response: :class:`~urllib3.response.BaseHTTPResponse`

466 :param Exception error: An error encountered during the request, or

467 None if the response was received successfully.

468

469 :return: A new ``Retry`` object.

470 """

471 if self.total is False and error:

472 # Disabled, indicate to re-raise the error.

473 raise reraise(type(error), error, _stacktrace)

474

475 total = self.total

476 if total is not None:

477 total -= 1

478

479 connect = self.connect

480 read = self.read

481 redirect = self.redirect

482 status_count = self.status

483 other = self.other

484 cause = "unknown"

485 status = None

486 redirect_location = None

487

488 if error and self._is_connection_error(error):

489 # Connect retry?

490 if connect is False:

491 raise reraise(type(error), error, _stacktrace)

492 elif connect is not None:

493 connect -= 1

494

495 elif error and self._is_read_error(error):

496 # Read retry?

497 if read is False or method is None or not self._is_method_retryable(method):

498 raise reraise(type(error), error, _stacktrace)

499 elif read is not None:

500 read -= 1

501

502 elif error:

503 # Other retry?

504 if other is not None:

505 other -= 1

506

507 elif response and response.get_redirect_location():

508 # Redirect retry?

509 if redirect is not None:

510 redirect -= 1

511 cause = "too many redirects"

512 response_redirect_location = response.get_redirect_location()

513 if response_redirect_location:

514 redirect_location = response_redirect_location

515 status = response.status

516

517 else:

518 # Incrementing because of a server error like a 500 in

519 # status_forcelist and the given method is in the allowed_methods

520 cause = ResponseError.GENERIC_ERROR

521 if response and response.status:

522 if status_count is not None:

523 status_count -= 1

524 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)

525 status = response.status

526

527 history = self.history + (

528 RequestHistory(method, url, error, status, redirect_location),

529 )

530

531 new_retry = self.new(

532 total=total,

533 connect=connect,

534 read=read,

535 redirect=redirect,

536 status=status_count,

537 other=other,

538 history=history,

539 )

540

541 if new_retry.is_exhausted():

542 reason = error or ResponseError(cause)

543 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]

544

545 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)

546

547 return new_retry

548

549 def __repr__(self) -> str:

550 return (

551 f"{type(self).__name__}(total={self.total}, connect={self.connect}, "

552 f"read={self.read}, redirect={self.redirect}, status={self.status})"

553 )

554

555

556# For backwards compatibility (equivalent to pre-v1.9):

557Retry.DEFAULT = Retry(3)