Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import email
4import logging
5import random
6import re
7import time
8import typing
9from itertools import takewhile
10from types import TracebackType
12from ..exceptions import (
13 ConnectTimeoutError,
14 InvalidHeader,
15 MaxRetryError,
16 ProtocolError,
17 ProxyError,
18 ReadTimeoutError,
19 ResponseError,
20)
21from .util import reraise
23if typing.TYPE_CHECKING:
24 from typing_extensions import Self
26 from ..connectionpool import ConnectionPool
27 from ..response import BaseHTTPResponse
29log = logging.getLogger(__name__)
32# Data structure for representing the metadata of requests that result in a retry.
33class RequestHistory(typing.NamedTuple):
34 method: str | None
35 url: str | None
36 error: Exception | None
37 status: int | None
38 redirect_location: str | None
41class Retry:
42 """Retry configuration.
44 Each retry attempt will create a new Retry object with updated values, so
45 they can be safely reused.
47 Retries can be defined as a default for a pool:
49 .. code-block:: python
51 retries = Retry(connect=5, read=2, redirect=5)
52 http = PoolManager(retries=retries)
53 response = http.request("GET", "https://example.com/")
55 Or per-request (which overrides the default for the pool):
57 .. code-block:: python
59 response = http.request("GET", "https://example.com/", retries=Retry(10))
61 Retries can be disabled by passing ``False``:
63 .. code-block:: python
65 response = http.request("GET", "https://example.com/", retries=False)
67 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
68 retries are disabled, in which case the causing exception will be raised.
70 :param int total:
71 Total number of retries to allow. Takes precedence over other counts.
73 Set to ``None`` to remove this constraint and fall back on other
74 counts.
76 Set to ``0`` to fail on the first retry.
78 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
80 :param int connect:
81 How many connection-related errors to retry on.
83 These are errors raised before the request is sent to the remote server,
84 which we assume has not triggered the server to process the request.
86 Set to ``0`` to fail on the first retry of this type.
88 :param int read:
89 How many times to retry on read errors.
91 These errors are raised after the request was sent to the server, so the
92 request may have side-effects.
94 Set to ``0`` to fail on the first retry of this type.
96 :param int redirect:
97 How many redirects to perform. Limit this to avoid infinite redirect
98 loops.
100 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
101 308.
103 Set to ``0`` to fail on the first retry of this type.
105 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
107 :param int status:
108 How many times to retry on bad status codes.
110 These are retries made on responses, where status code matches
111 ``status_forcelist``.
113 Set to ``0`` to fail on the first retry of this type.
115 :param int other:
116 How many times to retry on other errors.
118 Other errors are errors that are not connect, read, redirect or status errors.
119 These errors might be raised after the request was sent to the server, so the
120 request might have side-effects.
122 Set to ``0`` to fail on the first retry of this type.
124 If ``total`` is not set, it's a good idea to set this to 0 to account
125 for unexpected edge cases and avoid infinite retry loops.
127 :param Collection allowed_methods:
128 Set of uppercased HTTP method verbs that we should retry on.
130 By default, we only retry on methods which are considered to be
131 idempotent (multiple requests with the same parameters end with the
132 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
134 Set to a ``None`` value to retry on any verb.
136 :param Collection status_forcelist:
137 A set of integer HTTP status codes that we should force a retry on.
138 A retry is initiated if the request method is in ``allowed_methods``
139 and the response status code is in ``status_forcelist``.
141 By default, this is disabled with ``None``.
143 :param float backoff_factor:
144 A backoff factor to apply between attempts after the second try
145 (most errors are resolved immediately by a second try without a
146 delay). urllib3 will sleep for::
148 {backoff factor} * (2 ** ({number of previous retries}))
150 seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
152 random.uniform(0, {backoff jitter})
154 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
155 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
156 be longer than `backoff_max`.
158 By default, backoff is disabled (factor set to 0).
160 :param bool raise_on_redirect: Whether, if the number of redirects is
161 exhausted, to raise a MaxRetryError, or to return a response with a
162 response code in the 3xx range.
164 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
165 whether we should raise an exception, or return a response,
166 if status falls in ``status_forcelist`` range and retries have
167 been exhausted.
169 :param tuple history: The history of the request encountered during
170 each call to :meth:`~Retry.increment`. The list is in the order
171 the requests occurred. Each list item is of class :class:`RequestHistory`.
173 :param bool respect_retry_after_header:
174 Whether to respect Retry-After header on status codes defined as
175 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
177 :param Collection remove_headers_on_redirect:
178 Sequence of headers to remove from the request when a response
179 indicating a redirect is returned before firing off the redirected
180 request.
181 """
183 #: Default methods to be used for ``allowed_methods``
184 DEFAULT_ALLOWED_METHODS = frozenset(
185 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
186 )
188 #: Default status codes to be used for ``status_forcelist``
189 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
191 #: Default headers to be used for ``remove_headers_on_redirect``
192 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
193 ["Cookie", "Authorization", "Proxy-Authorization"]
194 )
196 #: Default maximum backoff time.
197 DEFAULT_BACKOFF_MAX = 120
199 # Backward compatibility; assigned outside of the class.
200 DEFAULT: typing.ClassVar[Retry]
202 def __init__(
203 self,
204 total: bool | int | None = 10,
205 connect: int | None = None,
206 read: int | None = None,
207 redirect: bool | int | None = None,
208 status: int | None = None,
209 other: int | None = None,
210 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
211 status_forcelist: typing.Collection[int] | None = None,
212 backoff_factor: float = 0,
213 backoff_max: float = DEFAULT_BACKOFF_MAX,
214 raise_on_redirect: bool = True,
215 raise_on_status: bool = True,
216 history: tuple[RequestHistory, ...] | None = None,
217 respect_retry_after_header: bool = True,
218 remove_headers_on_redirect: typing.Collection[
219 str
220 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
221 backoff_jitter: float = 0.0,
222 ) -> None:
223 self.total = total
224 self.connect = connect
225 self.read = read
226 self.status = status
227 self.other = other
229 if redirect is False or total is False:
230 redirect = 0
231 raise_on_redirect = False
233 self.redirect = redirect
234 self.status_forcelist = status_forcelist or set()
235 self.allowed_methods = allowed_methods
236 self.backoff_factor = backoff_factor
237 self.backoff_max = backoff_max
238 self.raise_on_redirect = raise_on_redirect
239 self.raise_on_status = raise_on_status
240 self.history = history or ()
241 self.respect_retry_after_header = respect_retry_after_header
242 self.remove_headers_on_redirect = frozenset(
243 h.lower() for h in remove_headers_on_redirect
244 )
245 self.backoff_jitter = backoff_jitter
247 def new(self, **kw: typing.Any) -> Self:
248 params = dict(
249 total=self.total,
250 connect=self.connect,
251 read=self.read,
252 redirect=self.redirect,
253 status=self.status,
254 other=self.other,
255 allowed_methods=self.allowed_methods,
256 status_forcelist=self.status_forcelist,
257 backoff_factor=self.backoff_factor,
258 backoff_max=self.backoff_max,
259 raise_on_redirect=self.raise_on_redirect,
260 raise_on_status=self.raise_on_status,
261 history=self.history,
262 remove_headers_on_redirect=self.remove_headers_on_redirect,
263 respect_retry_after_header=self.respect_retry_after_header,
264 backoff_jitter=self.backoff_jitter,
265 )
267 params.update(kw)
268 return type(self)(**params) # type: ignore[arg-type]
270 @classmethod
271 def from_int(
272 cls,
273 retries: Retry | bool | int | None,
274 redirect: bool | int | None = True,
275 default: Retry | bool | int | None = None,
276 ) -> Retry:
277 """Backwards-compatibility for the old retries format."""
278 if retries is None:
279 retries = default if default is not None else cls.DEFAULT
281 if isinstance(retries, Retry):
282 return retries
284 redirect = bool(redirect) and None
285 new_retries = cls(retries, redirect=redirect)
286 log.debug("Converted retries value: %r -> %r", retries, new_retries)
287 return new_retries
289 def get_backoff_time(self) -> float:
290 """Formula for computing the current backoff
292 :rtype: float
293 """
294 # We want to consider only the last consecutive errors sequence (Ignore redirects).
295 consecutive_errors_len = len(
296 list(
297 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
298 )
299 )
300 if consecutive_errors_len <= 1:
301 return 0
303 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
304 if self.backoff_jitter != 0.0:
305 backoff_value += random.random() * self.backoff_jitter
306 return float(max(0, min(self.backoff_max, backoff_value)))
308 def parse_retry_after(self, retry_after: str) -> float:
309 seconds: float
310 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
311 if re.match(r"^\s*[0-9]+\s*$", retry_after):
312 seconds = int(retry_after)
313 else:
314 retry_date_tuple = email.utils.parsedate_tz(retry_after)
315 if retry_date_tuple is None:
316 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
318 retry_date = email.utils.mktime_tz(retry_date_tuple)
319 seconds = retry_date - time.time()
321 seconds = max(seconds, 0)
323 return seconds
325 def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
326 """Get the value of Retry-After in seconds."""
328 retry_after = response.headers.get("Retry-After")
330 if retry_after is None:
331 return None
333 return self.parse_retry_after(retry_after)
335 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
336 retry_after = self.get_retry_after(response)
337 if retry_after:
338 time.sleep(retry_after)
339 return True
341 return False
343 def _sleep_backoff(self) -> None:
344 backoff = self.get_backoff_time()
345 if backoff <= 0:
346 return
347 time.sleep(backoff)
349 def sleep(self, response: BaseHTTPResponse | None = None) -> None:
350 """Sleep between retry attempts.
352 This method will respect a server's ``Retry-After`` response header
353 and sleep the duration of the time requested. If that is not present, it
354 will use an exponential backoff. By default, the backoff factor is 0 and
355 this method will return immediately.
356 """
358 if self.respect_retry_after_header and response:
359 slept = self.sleep_for_retry(response)
360 if slept:
361 return
363 self._sleep_backoff()
365 def _is_connection_error(self, err: Exception) -> bool:
366 """Errors when we're fairly sure that the server did not receive the
367 request, so it should be safe to retry.
368 """
369 if isinstance(err, ProxyError):
370 err = err.original_error
371 return isinstance(err, ConnectTimeoutError)
373 def _is_read_error(self, err: Exception) -> bool:
374 """Errors that occur after the request has been started, so we should
375 assume that the server began processing it.
376 """
377 return isinstance(err, (ReadTimeoutError, ProtocolError))
379 def _is_method_retryable(self, method: str) -> bool:
380 """Checks if a given HTTP method should be retried upon, depending if
381 it is included in the allowed_methods
382 """
383 if self.allowed_methods and method.upper() not in self.allowed_methods:
384 return False
385 return True
387 def is_retry(
388 self, method: str, status_code: int, has_retry_after: bool = False
389 ) -> bool:
390 """Is this method/status code retryable? (Based on allowlists and control
391 variables such as the number of total retries to allow, whether to
392 respect the Retry-After header, whether this header is present, and
393 whether the returned status code is on the list of status codes to
394 be retried upon on the presence of the aforementioned header)
395 """
396 if not self._is_method_retryable(method):
397 return False
399 if self.status_forcelist and status_code in self.status_forcelist:
400 return True
402 return bool(
403 self.total
404 and self.respect_retry_after_header
405 and has_retry_after
406 and (status_code in self.RETRY_AFTER_STATUS_CODES)
407 )
409 def is_exhausted(self) -> bool:
410 """Are we out of retries?"""
411 retry_counts = [
412 x
413 for x in (
414 self.total,
415 self.connect,
416 self.read,
417 self.redirect,
418 self.status,
419 self.other,
420 )
421 if x
422 ]
423 if not retry_counts:
424 return False
426 return min(retry_counts) < 0
428 def increment(
429 self,
430 method: str | None = None,
431 url: str | None = None,
432 response: BaseHTTPResponse | None = None,
433 error: Exception | None = None,
434 _pool: ConnectionPool | None = None,
435 _stacktrace: TracebackType | None = None,
436 ) -> Self:
437 """Return a new Retry object with incremented retry counters.
439 :param response: A response object, or None, if the server did not
440 return a response.
441 :type response: :class:`~urllib3.response.BaseHTTPResponse`
442 :param Exception error: An error encountered during the request, or
443 None if the response was received successfully.
445 :return: A new ``Retry`` object.
446 """
447 if self.total is False and error:
448 # Disabled, indicate to re-raise the error.
449 raise reraise(type(error), error, _stacktrace)
451 total = self.total
452 if total is not None:
453 total -= 1
455 connect = self.connect
456 read = self.read
457 redirect = self.redirect
458 status_count = self.status
459 other = self.other
460 cause = "unknown"
461 status = None
462 redirect_location = None
464 if error and self._is_connection_error(error):
465 # Connect retry?
466 if connect is False:
467 raise reraise(type(error), error, _stacktrace)
468 elif connect is not None:
469 connect -= 1
471 elif error and self._is_read_error(error):
472 # Read retry?
473 if read is False or method is None or not self._is_method_retryable(method):
474 raise reraise(type(error), error, _stacktrace)
475 elif read is not None:
476 read -= 1
478 elif error:
479 # Other retry?
480 if other is not None:
481 other -= 1
483 elif response and response.get_redirect_location():
484 # Redirect retry?
485 if redirect is not None:
486 redirect -= 1
487 cause = "too many redirects"
488 response_redirect_location = response.get_redirect_location()
489 if response_redirect_location:
490 redirect_location = response_redirect_location
491 status = response.status
493 else:
494 # Incrementing because of a server error like a 500 in
495 # status_forcelist and the given method is in the allowed_methods
496 cause = ResponseError.GENERIC_ERROR
497 if response and response.status:
498 if status_count is not None:
499 status_count -= 1
500 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
501 status = response.status
503 history = self.history + (
504 RequestHistory(method, url, error, status, redirect_location),
505 )
507 new_retry = self.new(
508 total=total,
509 connect=connect,
510 read=read,
511 redirect=redirect,
512 status=status_count,
513 other=other,
514 history=history,
515 )
517 if new_retry.is_exhausted():
518 reason = error or ResponseError(cause)
519 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
521 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
523 return new_retry
525 def __repr__(self) -> str:
526 return (
527 f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
528 f"read={self.read}, redirect={self.redirect}, status={self.status})"
529 )
532# For backwards compatibility (equivalent to pre-v1.9):
533Retry.DEFAULT = Retry(3)