Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py: 34%
169 statements
« prev ^ index » next coverage.py v7.2.0, created at 2023-02-23 06:30 +0000
1from __future__ import annotations
3import email
4import logging
5import re
6import time
7import typing
8from itertools import takewhile
9from types import TracebackType
11from ..exceptions import (
12 ConnectTimeoutError,
13 InvalidHeader,
14 MaxRetryError,
15 ProtocolError,
16 ProxyError,
17 ReadTimeoutError,
18 ResponseError,
19)
20from .util import reraise
22if typing.TYPE_CHECKING:
23 from ..connectionpool import ConnectionPool
24 from ..response import BaseHTTPResponse
26log = logging.getLogger(__name__)
29# Data structure for representing the metadata of requests that result in a retry.
30class RequestHistory(typing.NamedTuple):
31 method: str | None
32 url: str | None
33 error: Exception | None
34 status: int | None
35 redirect_location: str | None
38class Retry:
39 """Retry configuration.
41 Each retry attempt will create a new Retry object with updated values, so
42 they can be safely reused.
44 Retries can be defined as a default for a pool:
46 .. code-block:: python
48 retries = Retry(connect=5, read=2, redirect=5)
49 http = PoolManager(retries=retries)
50 response = http.request("GET", "https://example.com/")
52 Or per-request (which overrides the default for the pool):
54 .. code-block:: python
56 response = http.request("GET", "https://example.com/", retries=Retry(10))
58 Retries can be disabled by passing ``False``:
60 .. code-block:: python
62 response = http.request("GET", "https://example.com/", retries=False)
64 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
65 retries are disabled, in which case the causing exception will be raised.
67 :param int total:
68 Total number of retries to allow. Takes precedence over other counts.
70 Set to ``None`` to remove this constraint and fall back on other
71 counts.
73 Set to ``0`` to fail on the first retry.
75 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
77 :param int connect:
78 How many connection-related errors to retry on.
80 These are errors raised before the request is sent to the remote server,
81 which we assume has not triggered the server to process the request.
83 Set to ``0`` to fail on the first retry of this type.
85 :param int read:
86 How many times to retry on read errors.
88 These errors are raised after the request was sent to the server, so the
89 request may have side-effects.
91 Set to ``0`` to fail on the first retry of this type.
93 :param int redirect:
94 How many redirects to perform. Limit this to avoid infinite redirect
95 loops.
97 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
98 308.
100 Set to ``0`` to fail on the first retry of this type.
102 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
104 :param int status:
105 How many times to retry on bad status codes.
107 These are retries made on responses, where status code matches
108 ``status_forcelist``.
110 Set to ``0`` to fail on the first retry of this type.
112 :param int other:
113 How many times to retry on other errors.
115 Other errors are errors that are not connect, read, redirect or status errors.
116 These errors might be raised after the request was sent to the server, so the
117 request might have side-effects.
119 Set to ``0`` to fail on the first retry of this type.
121 If ``total`` is not set, it's a good idea to set this to 0 to account
122 for unexpected edge cases and avoid infinite retry loops.
124 :param Collection allowed_methods:
125 Set of uppercased HTTP method verbs that we should retry on.
127 By default, we only retry on methods which are considered to be
128 idempotent (multiple requests with the same parameters end with the
129 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
131 Set to a ``None`` value to retry on any verb.
133 :param Collection status_forcelist:
134 A set of integer HTTP status codes that we should force a retry on.
135 A retry is initiated if the request method is in ``allowed_methods``
136 and the response status code is in ``status_forcelist``.
138 By default, this is disabled with ``None``.
140 :param float backoff_factor:
141 A backoff factor to apply between attempts after the second try
142 (most errors are resolved immediately by a second try without a
143 delay). urllib3 will sleep for::
145 {backoff factor} * (2 ** ({number of total retries} - 1))
147 seconds. If the backoff_factor is 0.1, then :func:`Retry.sleep` will sleep
148 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
149 than `backoff_max`.
151 By default, backoff is disabled (set to 0).
153 :param bool raise_on_redirect: Whether, if the number of redirects is
154 exhausted, to raise a MaxRetryError, or to return a response with a
155 response code in the 3xx range.
157 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
158 whether we should raise an exception, or return a response,
159 if status falls in ``status_forcelist`` range and retries have
160 been exhausted.
162 :param tuple history: The history of the request encountered during
163 each call to :meth:`~Retry.increment`. The list is in the order
164 the requests occurred. Each list item is of class :class:`RequestHistory`.
166 :param bool respect_retry_after_header:
167 Whether to respect Retry-After header on status codes defined as
168 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
170 :param Collection remove_headers_on_redirect:
171 Sequence of headers to remove from the request when a response
172 indicating a redirect is returned before firing off the redirected
173 request.
174 """
176 #: Default methods to be used for ``allowed_methods``
177 DEFAULT_ALLOWED_METHODS = frozenset(
178 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
179 )
181 #: Default status codes to be used for ``status_forcelist``
182 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
184 #: Default headers to be used for ``remove_headers_on_redirect``
185 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
187 #: Default maximum backoff time.
188 DEFAULT_BACKOFF_MAX = 120
190 # Backward compatibility; assigned outside of the class.
191 DEFAULT: typing.ClassVar[Retry]
193 def __init__(
194 self,
195 total: bool | int | None = 10,
196 connect: int | None = None,
197 read: int | None = None,
198 redirect: bool | int | None = None,
199 status: int | None = None,
200 other: int | None = None,
201 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
202 status_forcelist: typing.Collection[int] | None = None,
203 backoff_factor: float = 0,
204 backoff_max: float = DEFAULT_BACKOFF_MAX,
205 raise_on_redirect: bool = True,
206 raise_on_status: bool = True,
207 history: tuple[RequestHistory, ...] | None = None,
208 respect_retry_after_header: bool = True,
209 remove_headers_on_redirect: typing.Collection[
210 str
211 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
212 ) -> None:
213 self.total = total
214 self.connect = connect
215 self.read = read
216 self.status = status
217 self.other = other
219 if redirect is False or total is False:
220 redirect = 0
221 raise_on_redirect = False
223 self.redirect = redirect
224 self.status_forcelist = status_forcelist or set()
225 self.allowed_methods = allowed_methods
226 self.backoff_factor = backoff_factor
227 self.backoff_max = backoff_max
228 self.raise_on_redirect = raise_on_redirect
229 self.raise_on_status = raise_on_status
230 self.history = history or ()
231 self.respect_retry_after_header = respect_retry_after_header
232 self.remove_headers_on_redirect = frozenset(
233 h.lower() for h in remove_headers_on_redirect
234 )
236 def new(self, **kw: typing.Any) -> Retry:
237 params = dict(
238 total=self.total,
239 connect=self.connect,
240 read=self.read,
241 redirect=self.redirect,
242 status=self.status,
243 other=self.other,
244 allowed_methods=self.allowed_methods,
245 status_forcelist=self.status_forcelist,
246 backoff_factor=self.backoff_factor,
247 backoff_max=self.backoff_max,
248 raise_on_redirect=self.raise_on_redirect,
249 raise_on_status=self.raise_on_status,
250 history=self.history,
251 remove_headers_on_redirect=self.remove_headers_on_redirect,
252 respect_retry_after_header=self.respect_retry_after_header,
253 )
255 params.update(kw)
256 return type(self)(**params) # type: ignore[arg-type]
258 @classmethod
259 def from_int(
260 cls,
261 retries: Retry | bool | int | None,
262 redirect: bool | int | None = True,
263 default: Retry | bool | int | None = None,
264 ) -> Retry:
265 """Backwards-compatibility for the old retries format."""
266 if retries is None:
267 retries = default if default is not None else cls.DEFAULT
269 if isinstance(retries, Retry):
270 return retries
272 redirect = bool(redirect) and None
273 new_retries = cls(retries, redirect=redirect)
274 log.debug("Converted retries value: %r -> %r", retries, new_retries)
275 return new_retries
277 def get_backoff_time(self) -> float:
278 """Formula for computing the current backoff
280 :rtype: float
281 """
282 # We want to consider only the last consecutive errors sequence (Ignore redirects).
283 consecutive_errors_len = len(
284 list(
285 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
286 )
287 )
288 if consecutive_errors_len <= 1:
289 return 0
291 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
292 return float(min(self.backoff_max, backoff_value))
294 def parse_retry_after(self, retry_after: str) -> float:
295 seconds: float
296 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
297 if re.match(r"^\s*[0-9]+\s*$", retry_after):
298 seconds = int(retry_after)
299 else:
300 retry_date_tuple = email.utils.parsedate_tz(retry_after)
301 if retry_date_tuple is None:
302 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
304 retry_date = email.utils.mktime_tz(retry_date_tuple)
305 seconds = retry_date - time.time()
307 seconds = max(seconds, 0)
309 return seconds
311 def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
312 """Get the value of Retry-After in seconds."""
314 retry_after = response.headers.get("Retry-After")
316 if retry_after is None:
317 return None
319 return self.parse_retry_after(retry_after)
321 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
322 retry_after = self.get_retry_after(response)
323 if retry_after:
324 time.sleep(retry_after)
325 return True
327 return False
329 def _sleep_backoff(self) -> None:
330 backoff = self.get_backoff_time()
331 if backoff <= 0:
332 return
333 time.sleep(backoff)
335 def sleep(self, response: BaseHTTPResponse | None = None) -> None:
336 """Sleep between retry attempts.
338 This method will respect a server's ``Retry-After`` response header
339 and sleep the duration of the time requested. If that is not present, it
340 will use an exponential backoff. By default, the backoff factor is 0 and
341 this method will return immediately.
342 """
344 if self.respect_retry_after_header and response:
345 slept = self.sleep_for_retry(response)
346 if slept:
347 return
349 self._sleep_backoff()
351 def _is_connection_error(self, err: Exception) -> bool:
352 """Errors when we're fairly sure that the server did not receive the
353 request, so it should be safe to retry.
354 """
355 if isinstance(err, ProxyError):
356 err = err.original_error
357 return isinstance(err, ConnectTimeoutError)
359 def _is_read_error(self, err: Exception) -> bool:
360 """Errors that occur after the request has been started, so we should
361 assume that the server began processing it.
362 """
363 return isinstance(err, (ReadTimeoutError, ProtocolError))
365 def _is_method_retryable(self, method: str) -> bool:
366 """Checks if a given HTTP method should be retried upon, depending if
367 it is included in the allowed_methods
368 """
369 if self.allowed_methods and method.upper() not in self.allowed_methods:
370 return False
371 return True
373 def is_retry(
374 self, method: str, status_code: int, has_retry_after: bool = False
375 ) -> bool:
376 """Is this method/status code retryable? (Based on allowlists and control
377 variables such as the number of total retries to allow, whether to
378 respect the Retry-After header, whether this header is present, and
379 whether the returned status code is on the list of status codes to
380 be retried upon on the presence of the aforementioned header)
381 """
382 if not self._is_method_retryable(method):
383 return False
385 if self.status_forcelist and status_code in self.status_forcelist:
386 return True
388 return bool(
389 self.total
390 and self.respect_retry_after_header
391 and has_retry_after
392 and (status_code in self.RETRY_AFTER_STATUS_CODES)
393 )
395 def is_exhausted(self) -> bool:
396 """Are we out of retries?"""
397 retry_counts = [
398 x
399 for x in (
400 self.total,
401 self.connect,
402 self.read,
403 self.redirect,
404 self.status,
405 self.other,
406 )
407 if x
408 ]
409 if not retry_counts:
410 return False
412 return min(retry_counts) < 0
414 def increment(
415 self,
416 method: str | None = None,
417 url: str | None = None,
418 response: BaseHTTPResponse | None = None,
419 error: Exception | None = None,
420 _pool: ConnectionPool | None = None,
421 _stacktrace: TracebackType | None = None,
422 ) -> Retry:
423 """Return a new Retry object with incremented retry counters.
425 :param response: A response object, or None, if the server did not
426 return a response.
427 :type response: :class:`~urllib3.response.BaseHTTPResponse`
428 :param Exception error: An error encountered during the request, or
429 None if the response was received successfully.
431 :return: A new ``Retry`` object.
432 """
433 if self.total is False and error:
434 # Disabled, indicate to re-raise the error.
435 raise reraise(type(error), error, _stacktrace)
437 total = self.total
438 if total is not None:
439 total -= 1
441 connect = self.connect
442 read = self.read
443 redirect = self.redirect
444 status_count = self.status
445 other = self.other
446 cause = "unknown"
447 status = None
448 redirect_location = None
450 if error and self._is_connection_error(error):
451 # Connect retry?
452 if connect is False:
453 raise reraise(type(error), error, _stacktrace)
454 elif connect is not None:
455 connect -= 1
457 elif error and self._is_read_error(error):
458 # Read retry?
459 if read is False or method is None or not self._is_method_retryable(method):
460 raise reraise(type(error), error, _stacktrace)
461 elif read is not None:
462 read -= 1
464 elif error:
465 # Other retry?
466 if other is not None:
467 other -= 1
469 elif response and response.get_redirect_location():
470 # Redirect retry?
471 if redirect is not None:
472 redirect -= 1
473 cause = "too many redirects"
474 response_redirect_location = response.get_redirect_location()
475 if response_redirect_location:
476 redirect_location = response_redirect_location
477 status = response.status
479 else:
480 # Incrementing because of a server error like a 500 in
481 # status_forcelist and the given method is in the allowed_methods
482 cause = ResponseError.GENERIC_ERROR
483 if response and response.status:
484 if status_count is not None:
485 status_count -= 1
486 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
487 status = response.status
489 history = self.history + (
490 RequestHistory(method, url, error, status, redirect_location),
491 )
493 new_retry = self.new(
494 total=total,
495 connect=connect,
496 read=read,
497 redirect=redirect,
498 status=status_count,
499 other=other,
500 history=history,
501 )
503 if new_retry.is_exhausted():
504 reason = error or ResponseError(cause)
505 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
507 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
509 return new_retry
511 def __repr__(self) -> str:
512 return (
513 f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
514 f"read={self.read}, redirect={self.redirect}, status={self.status})"
515 )
518# For backwards compatibility (equivalent to pre-v1.9):
519Retry.DEFAULT = Retry(3)