Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import email
4import logging
5import random
6import re
7import time
8import typing
9from itertools import takewhile
10from types import TracebackType
12from ..exceptions import (
13 ConnectTimeoutError,
14 InvalidHeader,
15 MaxRetryError,
16 ProtocolError,
17 ProxyError,
18 ReadTimeoutError,
19 ResponseError,
20)
21from .util import reraise
23if typing.TYPE_CHECKING:
24 from typing_extensions import Self
26 from ..connectionpool import ConnectionPool
27 from ..response import BaseHTTPResponse
29log = logging.getLogger(__name__)
32# Data structure for representing the metadata of requests that result in a retry.
33class RequestHistory(typing.NamedTuple):
34 method: str | None
35 url: str | None
36 error: Exception | None
37 status: int | None
38 redirect_location: str | None
41class Retry:
42 """Retry configuration.
44 Each retry attempt will create a new Retry object with updated values, so
45 they can be safely reused.
47 Retries can be defined as a default for a pool:
49 .. code-block:: python
51 retries = Retry(connect=5, read=2, redirect=5)
52 http = PoolManager(retries=retries)
53 response = http.request("GET", "https://example.com/")
55 Or per-request (which overrides the default for the pool):
57 .. code-block:: python
59 response = http.request("GET", "https://example.com/", retries=Retry(10))
61 Retries can be disabled by passing ``False``:
63 .. code-block:: python
65 response = http.request("GET", "https://example.com/", retries=False)
67 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
68 retries are disabled, in which case the causing exception will be raised.
70 :param int total:
71 Total number of retries to allow. Takes precedence over other counts.
73 Set to ``None`` to remove this constraint and fall back on other
74 counts.
76 Set to ``0`` to fail on the first retry.
78 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
80 :param int connect:
81 How many connection-related errors to retry on.
83 These are errors raised before the request is sent to the remote server,
84 which we assume has not triggered the server to process the request.
86 Set to ``0`` to fail on the first retry of this type.
88 :param int read:
89 How many times to retry on read errors.
91 These errors are raised after the request was sent to the server, so the
92 request may have side-effects.
94 Set to ``0`` to fail on the first retry of this type.
96 :param int redirect:
97 How many redirects to perform. Limit this to avoid infinite redirect
98 loops.
100 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
101 308.
103 Set to ``0`` to fail on the first retry of this type.
105 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
107 :param int status:
108 How many times to retry on bad status codes.
110 These are retries made on responses, where status code matches
111 ``status_forcelist``.
113 Set to ``0`` to fail on the first retry of this type.
115 :param int other:
116 How many times to retry on other errors.
118 Other errors are errors that are not connect, read, redirect or status errors.
119 These errors might be raised after the request was sent to the server, so the
120 request might have side-effects.
122 Set to ``0`` to fail on the first retry of this type.
124 If ``total`` is not set, it's a good idea to set this to 0 to account
125 for unexpected edge cases and avoid infinite retry loops.
127 :param Collection allowed_methods:
128 Set of uppercased HTTP method verbs that we should retry on.
130 By default, we only retry on methods which are considered to be
131 idempotent (multiple requests with the same parameters end with the
132 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
134 Set to a ``None`` value to retry on any verb.
136 :param Collection status_forcelist:
137 A set of integer HTTP status codes that we should force a retry on.
138 A retry is initiated if the request method is in ``allowed_methods``
139 and the response status code is in ``status_forcelist``.
141 By default, this is disabled with ``None``.
143 :param float backoff_factor:
144 A backoff factor to apply between attempts after the second try
145 (most errors are resolved immediately by a second try without a
146 delay). urllib3 will sleep for::
148 {backoff factor} * (2 ** ({number of previous retries}))
150 seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
152 random.uniform(0, {backoff jitter})
154 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
155 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
156 be longer than `backoff_max`.
158 By default, backoff is disabled (factor set to 0).
160 :param bool raise_on_redirect: Whether, if the number of redirects is
161 exhausted, to raise a MaxRetryError, or to return a response with a
162 response code in the 3xx range.
164 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
165 whether we should raise an exception, or return a response,
166 if status falls in ``status_forcelist`` range and retries have
167 been exhausted.
169 :param tuple history: The history of the request encountered during
170 each call to :meth:`~Retry.increment`. The list is in the order
171 the requests occurred. Each list item is of class :class:`RequestHistory`.
173 :param bool respect_retry_after_header:
174 Whether to respect Retry-After header on status codes defined as
175 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
177 :param Collection remove_headers_on_redirect:
178 Sequence of headers to remove from the request when a response
179 indicating a redirect is returned before firing off the redirected
180 request.
182 :param int retry_after_max: Number of seconds to allow as the maximum for
183 Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`.
184 Any Retry-After headers larger than this value will be limited to this
185 value.
186 """
188 #: Default methods to be used for ``allowed_methods``
189 DEFAULT_ALLOWED_METHODS = frozenset(
190 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
191 )
193 #: Default status codes to be used for ``status_forcelist``
194 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
196 #: Default headers to be used for ``remove_headers_on_redirect``
197 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
198 ["Cookie", "Authorization", "Proxy-Authorization"]
199 )
201 #: Default maximum backoff time.
202 DEFAULT_BACKOFF_MAX = 120
204 # This is undocumented in the RFC. Setting to 6 hours matches other popular libraries.
205 #: Default maximum allowed value for Retry-After headers in seconds
206 DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600
208 # Backward compatibility; assigned outside of the class.
209 DEFAULT: typing.ClassVar[Retry]
211 def __init__(
212 self,
213 total: bool | int | None = 10,
214 connect: int | None = None,
215 read: int | None = None,
216 redirect: bool | int | None = None,
217 status: int | None = None,
218 other: int | None = None,
219 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
220 status_forcelist: typing.Collection[int] | None = None,
221 backoff_factor: float = 0,
222 backoff_max: float = DEFAULT_BACKOFF_MAX,
223 raise_on_redirect: bool = True,
224 raise_on_status: bool = True,
225 history: tuple[RequestHistory, ...] | None = None,
226 respect_retry_after_header: bool = True,
227 remove_headers_on_redirect: typing.Collection[
228 str
229 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
230 backoff_jitter: float = 0.0,
231 retry_after_max: int = DEFAULT_RETRY_AFTER_MAX,
232 ) -> None:
233 self.total = total
234 self.connect = connect
235 self.read = read
236 self.status = status
237 self.other = other
239 if redirect is False or total is False:
240 redirect = 0
241 raise_on_redirect = False
243 self.redirect = redirect
244 self.status_forcelist = status_forcelist or set()
245 self.allowed_methods = allowed_methods
246 self.backoff_factor = backoff_factor
247 self.backoff_max = backoff_max
248 self.retry_after_max = retry_after_max
249 self.raise_on_redirect = raise_on_redirect
250 self.raise_on_status = raise_on_status
251 self.history = history or ()
252 self.respect_retry_after_header = respect_retry_after_header
253 self.remove_headers_on_redirect = frozenset(
254 h.lower() for h in remove_headers_on_redirect
255 )
256 self.backoff_jitter = backoff_jitter
258 def new(self, **kw: typing.Any) -> Self:
259 params = dict(
260 total=self.total,
261 connect=self.connect,
262 read=self.read,
263 redirect=self.redirect,
264 status=self.status,
265 other=self.other,
266 allowed_methods=self.allowed_methods,
267 status_forcelist=self.status_forcelist,
268 backoff_factor=self.backoff_factor,
269 backoff_max=self.backoff_max,
270 retry_after_max=self.retry_after_max,
271 raise_on_redirect=self.raise_on_redirect,
272 raise_on_status=self.raise_on_status,
273 history=self.history,
274 remove_headers_on_redirect=self.remove_headers_on_redirect,
275 respect_retry_after_header=self.respect_retry_after_header,
276 backoff_jitter=self.backoff_jitter,
277 )
279 params.update(kw)
280 return type(self)(**params) # type: ignore[arg-type]
282 @classmethod
283 def from_int(
284 cls,
285 retries: Retry | bool | int | None,
286 redirect: bool | int | None = True,
287 default: Retry | bool | int | None = None,
288 ) -> Retry:
289 """Backwards-compatibility for the old retries format."""
290 if retries is None:
291 retries = default if default is not None else cls.DEFAULT
293 if isinstance(retries, Retry):
294 return retries
296 redirect = bool(redirect) and None
297 new_retries = cls(retries, redirect=redirect)
298 log.debug("Converted retries value: %r -> %r", retries, new_retries)
299 return new_retries
301 def get_backoff_time(self) -> float:
302 """Formula for computing the current backoff
304 :rtype: float
305 """
306 # We want to consider only the last consecutive errors sequence (Ignore redirects).
307 consecutive_errors_len = len(
308 list(
309 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
310 )
311 )
312 if consecutive_errors_len <= 1:
313 return 0
315 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
316 if self.backoff_jitter != 0.0:
317 backoff_value += random.random() * self.backoff_jitter
318 return float(max(0, min(self.backoff_max, backoff_value)))
320 def parse_retry_after(self, retry_after: str) -> float:
321 seconds: float
322 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
323 if re.match(r"^\s*[0-9]+\s*$", retry_after):
324 seconds = int(retry_after)
325 else:
326 retry_date_tuple = email.utils.parsedate_tz(retry_after)
327 if retry_date_tuple is None:
328 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
330 retry_date = email.utils.mktime_tz(retry_date_tuple)
331 seconds = retry_date - time.time()
333 seconds = max(seconds, 0)
335 # Check the seconds do not exceed the specified maximum
336 if seconds > self.retry_after_max:
337 seconds = self.retry_after_max
339 return seconds
341 def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
342 """Get the value of Retry-After in seconds."""
344 retry_after = response.headers.get("Retry-After")
346 if retry_after is None:
347 return None
349 return self.parse_retry_after(retry_after)
351 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
352 retry_after = self.get_retry_after(response)
353 if retry_after:
354 time.sleep(retry_after)
355 return True
357 return False
359 def _sleep_backoff(self) -> None:
360 backoff = self.get_backoff_time()
361 if backoff <= 0:
362 return
363 time.sleep(backoff)
365 def sleep(self, response: BaseHTTPResponse | None = None) -> None:
366 """Sleep between retry attempts.
368 This method will respect a server's ``Retry-After`` response header
369 and sleep the duration of the time requested. If that is not present, it
370 will use an exponential backoff. By default, the backoff factor is 0 and
371 this method will return immediately.
372 """
374 if self.respect_retry_after_header and response:
375 slept = self.sleep_for_retry(response)
376 if slept:
377 return
379 self._sleep_backoff()
381 def _is_connection_error(self, err: Exception) -> bool:
382 """Errors when we're fairly sure that the server did not receive the
383 request, so it should be safe to retry.
384 """
385 if isinstance(err, ProxyError):
386 err = err.original_error
387 return isinstance(err, ConnectTimeoutError)
389 def _is_read_error(self, err: Exception) -> bool:
390 """Errors that occur after the request has been started, so we should
391 assume that the server began processing it.
392 """
393 return isinstance(err, (ReadTimeoutError, ProtocolError))
395 def _is_method_retryable(self, method: str) -> bool:
396 """Checks if a given HTTP method should be retried upon, depending if
397 it is included in the allowed_methods
398 """
399 if self.allowed_methods and method.upper() not in self.allowed_methods:
400 return False
401 return True
403 def is_retry(
404 self, method: str, status_code: int, has_retry_after: bool = False
405 ) -> bool:
406 """Is this method/status code retryable? (Based on allowlists and control
407 variables such as the number of total retries to allow, whether to
408 respect the Retry-After header, whether this header is present, and
409 whether the returned status code is on the list of status codes to
410 be retried upon on the presence of the aforementioned header)
411 """
412 if not self._is_method_retryable(method):
413 return False
415 if self.status_forcelist and status_code in self.status_forcelist:
416 return True
418 return bool(
419 self.total
420 and self.respect_retry_after_header
421 and has_retry_after
422 and (status_code in self.RETRY_AFTER_STATUS_CODES)
423 )
425 def is_exhausted(self) -> bool:
426 """Are we out of retries?"""
427 retry_counts = [
428 x
429 for x in (
430 self.total,
431 self.connect,
432 self.read,
433 self.redirect,
434 self.status,
435 self.other,
436 )
437 if x
438 ]
439 if not retry_counts:
440 return False
442 return min(retry_counts) < 0
444 def increment(
445 self,
446 method: str | None = None,
447 url: str | None = None,
448 response: BaseHTTPResponse | None = None,
449 error: Exception | None = None,
450 _pool: ConnectionPool | None = None,
451 _stacktrace: TracebackType | None = None,
452 ) -> Self:
453 """Return a new Retry object with incremented retry counters.
455 :param response: A response object, or None, if the server did not
456 return a response.
457 :type response: :class:`~urllib3.response.BaseHTTPResponse`
458 :param Exception error: An error encountered during the request, or
459 None if the response was received successfully.
461 :return: A new ``Retry`` object.
462 """
463 if self.total is False and error:
464 # Disabled, indicate to re-raise the error.
465 raise reraise(type(error), error, _stacktrace)
467 total = self.total
468 if total is not None:
469 total -= 1
471 connect = self.connect
472 read = self.read
473 redirect = self.redirect
474 status_count = self.status
475 other = self.other
476 cause = "unknown"
477 status = None
478 redirect_location = None
480 if error and self._is_connection_error(error):
481 # Connect retry?
482 if connect is False:
483 raise reraise(type(error), error, _stacktrace)
484 elif connect is not None:
485 connect -= 1
487 elif error and self._is_read_error(error):
488 # Read retry?
489 if read is False or method is None or not self._is_method_retryable(method):
490 raise reraise(type(error), error, _stacktrace)
491 elif read is not None:
492 read -= 1
494 elif error:
495 # Other retry?
496 if other is not None:
497 other -= 1
499 elif response and response.get_redirect_location():
500 # Redirect retry?
501 if redirect is not None:
502 redirect -= 1
503 cause = "too many redirects"
504 response_redirect_location = response.get_redirect_location()
505 if response_redirect_location:
506 redirect_location = response_redirect_location
507 status = response.status
509 else:
510 # Incrementing because of a server error like a 500 in
511 # status_forcelist and the given method is in the allowed_methods
512 cause = ResponseError.GENERIC_ERROR
513 if response and response.status:
514 if status_count is not None:
515 status_count -= 1
516 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
517 status = response.status
519 history = self.history + (
520 RequestHistory(method, url, error, status, redirect_location),
521 )
523 new_retry = self.new(
524 total=total,
525 connect=connect,
526 read=read,
527 redirect=redirect,
528 status=status_count,
529 other=other,
530 history=history,
531 )
533 if new_retry.is_exhausted():
534 reason = error or ResponseError(cause)
535 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
537 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
539 return new_retry
541 def __repr__(self) -> str:
542 return (
543 f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
544 f"read={self.read}, redirect={self.redirect}, status={self.status})"
545 )
548# For backwards compatibility (equivalent to pre-v1.9):
549Retry.DEFAULT = Retry(3)