Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py: 36%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import absolute_import
3import email
4import logging
5import re
6import time
7import warnings
8from collections import namedtuple
9from itertools import takewhile
11from ..exceptions import (
12 ConnectTimeoutError,
13 InvalidHeader,
14 MaxRetryError,
15 ProtocolError,
16 ProxyError,
17 ReadTimeoutError,
18 ResponseError,
19)
20from ..packages import six
22log = logging.getLogger(__name__)
25# Data structure for representing the metadata of requests that result in a retry.
26RequestHistory = namedtuple(
27 "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
28)
31# TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
32_Default = object()
35class _RetryMeta(type):
36 @property
37 def DEFAULT_METHOD_WHITELIST(cls):
38 warnings.warn(
39 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
40 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
41 DeprecationWarning,
42 )
43 return cls.DEFAULT_ALLOWED_METHODS
45 @DEFAULT_METHOD_WHITELIST.setter
46 def DEFAULT_METHOD_WHITELIST(cls, value):
47 warnings.warn(
48 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
49 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
50 DeprecationWarning,
51 )
52 cls.DEFAULT_ALLOWED_METHODS = value
54 @property
55 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
56 warnings.warn(
57 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
58 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
59 DeprecationWarning,
60 )
61 return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
63 @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
64 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
65 warnings.warn(
66 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
67 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
68 DeprecationWarning,
69 )
70 cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
72 @property
73 def BACKOFF_MAX(cls):
74 warnings.warn(
75 "Using 'Retry.BACKOFF_MAX' is deprecated and "
76 "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
77 DeprecationWarning,
78 )
79 return cls.DEFAULT_BACKOFF_MAX
81 @BACKOFF_MAX.setter
82 def BACKOFF_MAX(cls, value):
83 warnings.warn(
84 "Using 'Retry.BACKOFF_MAX' is deprecated and "
85 "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
86 DeprecationWarning,
87 )
88 cls.DEFAULT_BACKOFF_MAX = value
91@six.add_metaclass(_RetryMeta)
92class Retry(object):
93 """Retry configuration.
95 Each retry attempt will create a new Retry object with updated values, so
96 they can be safely reused.
98 Retries can be defined as a default for a pool::
100 retries = Retry(connect=5, read=2, redirect=5)
101 http = PoolManager(retries=retries)
102 response = http.request('GET', 'http://example.com/')
104 Or per-request (which overrides the default for the pool)::
106 response = http.request('GET', 'http://example.com/', retries=Retry(10))
108 Retries can be disabled by passing ``False``::
110 response = http.request('GET', 'http://example.com/', retries=False)
112 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
113 retries are disabled, in which case the causing exception will be raised.
115 :param int total:
116 Total number of retries to allow. Takes precedence over other counts.
118 Set to ``None`` to remove this constraint and fall back on other
119 counts.
121 Set to ``0`` to fail on the first retry.
123 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
125 :param int connect:
126 How many connection-related errors to retry on.
128 These are errors raised before the request is sent to the remote server,
129 which we assume has not triggered the server to process the request.
131 Set to ``0`` to fail on the first retry of this type.
133 :param int read:
134 How many times to retry on read errors.
136 These errors are raised after the request was sent to the server, so the
137 request may have side-effects.
139 Set to ``0`` to fail on the first retry of this type.
141 :param int redirect:
142 How many redirects to perform. Limit this to avoid infinite redirect
143 loops.
145 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
146 308.
148 Set to ``0`` to fail on the first retry of this type.
150 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
152 :param int status:
153 How many times to retry on bad status codes.
155 These are retries made on responses, where status code matches
156 ``status_forcelist``.
158 Set to ``0`` to fail on the first retry of this type.
160 :param int other:
161 How many times to retry on other errors.
163 Other errors are errors that are not connect, read, redirect or status errors.
164 These errors might be raised after the request was sent to the server, so the
165 request might have side-effects.
167 Set to ``0`` to fail on the first retry of this type.
169 If ``total`` is not set, it's a good idea to set this to 0 to account
170 for unexpected edge cases and avoid infinite retry loops.
172 :param iterable allowed_methods:
173 Set of uppercased HTTP method verbs that we should retry on.
175 By default, we only retry on methods which are considered to be
176 idempotent (multiple requests with the same parameters end with the
177 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
179 Set to a ``False`` value to retry on any verb.
181 .. warning::
183 Previously this parameter was named ``method_whitelist``, that
184 usage is deprecated in v1.26.0 and will be removed in v2.0.
186 :param iterable status_forcelist:
187 A set of integer HTTP status codes that we should force a retry on.
188 A retry is initiated if the request method is in ``allowed_methods``
189 and the response status code is in ``status_forcelist``.
191 By default, this is disabled with ``None``.
193 :param float backoff_factor:
194 A backoff factor to apply between attempts after the second try
195 (most errors are resolved immediately by a second try without a
196 delay). urllib3 will sleep for::
198 {backoff factor} * (2 ** ({number of total retries} - 1))
200 seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
201 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
202 than :attr:`Retry.DEFAULT_BACKOFF_MAX`.
204 By default, backoff is disabled (set to 0).
206 :param bool raise_on_redirect: Whether, if the number of redirects is
207 exhausted, to raise a MaxRetryError, or to return a response with a
208 response code in the 3xx range.
210 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
211 whether we should raise an exception, or return a response,
212 if status falls in ``status_forcelist`` range and retries have
213 been exhausted.
215 :param tuple history: The history of the request encountered during
216 each call to :meth:`~Retry.increment`. The list is in the order
217 the requests occurred. Each list item is of class :class:`RequestHistory`.
219 :param bool respect_retry_after_header:
220 Whether to respect Retry-After header on status codes defined as
221 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
223 :param iterable remove_headers_on_redirect:
224 Sequence of headers to remove from the request when a response
225 indicating a redirect is returned before firing off the redirected
226 request.
227 """
229 #: Default methods to be used for ``allowed_methods``
230 DEFAULT_ALLOWED_METHODS = frozenset(
231 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
232 )
234 #: Default status codes to be used for ``status_forcelist``
235 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
237 #: Default headers to be used for ``remove_headers_on_redirect``
238 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
239 ["Cookie", "Authorization", "Proxy-Authorization"]
240 )
242 #: Maximum backoff time.
243 DEFAULT_BACKOFF_MAX = 120
245 def __init__(
246 self,
247 total=10,
248 connect=None,
249 read=None,
250 redirect=None,
251 status=None,
252 other=None,
253 allowed_methods=_Default,
254 status_forcelist=None,
255 backoff_factor=0,
256 raise_on_redirect=True,
257 raise_on_status=True,
258 history=None,
259 respect_retry_after_header=True,
260 remove_headers_on_redirect=_Default,
261 # TODO: Deprecated, remove in v2.0
262 method_whitelist=_Default,
263 ):
265 if method_whitelist is not _Default:
266 if allowed_methods is not _Default:
267 raise ValueError(
268 "Using both 'allowed_methods' and "
269 "'method_whitelist' together is not allowed. "
270 "Instead only use 'allowed_methods'"
271 )
272 warnings.warn(
273 "Using 'method_whitelist' with Retry is deprecated and "
274 "will be removed in v2.0. Use 'allowed_methods' instead",
275 DeprecationWarning,
276 stacklevel=2,
277 )
278 allowed_methods = method_whitelist
279 if allowed_methods is _Default:
280 allowed_methods = self.DEFAULT_ALLOWED_METHODS
281 if remove_headers_on_redirect is _Default:
282 remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
284 self.total = total
285 self.connect = connect
286 self.read = read
287 self.status = status
288 self.other = other
290 if redirect is False or total is False:
291 redirect = 0
292 raise_on_redirect = False
294 self.redirect = redirect
295 self.status_forcelist = status_forcelist or set()
296 self.allowed_methods = allowed_methods
297 self.backoff_factor = backoff_factor
298 self.raise_on_redirect = raise_on_redirect
299 self.raise_on_status = raise_on_status
300 self.history = history or tuple()
301 self.respect_retry_after_header = respect_retry_after_header
302 self.remove_headers_on_redirect = frozenset(
303 [h.lower() for h in remove_headers_on_redirect]
304 )
306 def new(self, **kw):
307 params = dict(
308 total=self.total,
309 connect=self.connect,
310 read=self.read,
311 redirect=self.redirect,
312 status=self.status,
313 other=self.other,
314 status_forcelist=self.status_forcelist,
315 backoff_factor=self.backoff_factor,
316 raise_on_redirect=self.raise_on_redirect,
317 raise_on_status=self.raise_on_status,
318 history=self.history,
319 remove_headers_on_redirect=self.remove_headers_on_redirect,
320 respect_retry_after_header=self.respect_retry_after_header,
321 )
323 # TODO: If already given in **kw we use what's given to us
324 # If not given we need to figure out what to pass. We decide
325 # based on whether our class has the 'method_whitelist' property
326 # and if so we pass the deprecated 'method_whitelist' otherwise
327 # we use 'allowed_methods'. Remove in v2.0
328 if "method_whitelist" not in kw and "allowed_methods" not in kw:
329 if "method_whitelist" in self.__dict__:
330 warnings.warn(
331 "Using 'method_whitelist' with Retry is deprecated and "
332 "will be removed in v2.0. Use 'allowed_methods' instead",
333 DeprecationWarning,
334 )
335 params["method_whitelist"] = self.allowed_methods
336 else:
337 params["allowed_methods"] = self.allowed_methods
339 params.update(kw)
340 return type(self)(**params)
342 @classmethod
343 def from_int(cls, retries, redirect=True, default=None):
344 """Backwards-compatibility for the old retries format."""
345 if retries is None:
346 retries = default if default is not None else cls.DEFAULT
348 if isinstance(retries, Retry):
349 return retries
351 redirect = bool(redirect) and None
352 new_retries = cls(retries, redirect=redirect)
353 log.debug("Converted retries value: %r -> %r", retries, new_retries)
354 return new_retries
356 def get_backoff_time(self):
357 """Formula for computing the current backoff
359 :rtype: float
360 """
361 # We want to consider only the last consecutive errors sequence (Ignore redirects).
362 consecutive_errors_len = len(
363 list(
364 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
365 )
366 )
367 if consecutive_errors_len <= 1:
368 return 0
370 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
371 return min(self.DEFAULT_BACKOFF_MAX, backoff_value)
373 def parse_retry_after(self, retry_after):
374 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
375 if re.match(r"^\s*[0-9]+\s*$", retry_after):
376 seconds = int(retry_after)
377 else:
378 retry_date_tuple = email.utils.parsedate_tz(retry_after)
379 if retry_date_tuple is None:
380 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
381 if retry_date_tuple[9] is None: # Python 2
382 # Assume UTC if no timezone was specified
383 # On Python2.7, parsedate_tz returns None for a timezone offset
384 # instead of 0 if no timezone is given, where mktime_tz treats
385 # a None timezone offset as local time.
386 retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
388 retry_date = email.utils.mktime_tz(retry_date_tuple)
389 seconds = retry_date - time.time()
391 if seconds < 0:
392 seconds = 0
394 return seconds
396 def get_retry_after(self, response):
397 """Get the value of Retry-After in seconds."""
399 retry_after = response.headers.get("Retry-After")
401 if retry_after is None:
402 return None
404 return self.parse_retry_after(retry_after)
406 def sleep_for_retry(self, response=None):
407 retry_after = self.get_retry_after(response)
408 if retry_after:
409 time.sleep(retry_after)
410 return True
412 return False
414 def _sleep_backoff(self):
415 backoff = self.get_backoff_time()
416 if backoff <= 0:
417 return
418 time.sleep(backoff)
420 def sleep(self, response=None):
421 """Sleep between retry attempts.
423 This method will respect a server's ``Retry-After`` response header
424 and sleep the duration of the time requested. If that is not present, it
425 will use an exponential backoff. By default, the backoff factor is 0 and
426 this method will return immediately.
427 """
429 if self.respect_retry_after_header and response:
430 slept = self.sleep_for_retry(response)
431 if slept:
432 return
434 self._sleep_backoff()
436 def _is_connection_error(self, err):
437 """Errors when we're fairly sure that the server did not receive the
438 request, so it should be safe to retry.
439 """
440 if isinstance(err, ProxyError):
441 err = err.original_error
442 return isinstance(err, ConnectTimeoutError)
444 def _is_read_error(self, err):
445 """Errors that occur after the request has been started, so we should
446 assume that the server began processing it.
447 """
448 return isinstance(err, (ReadTimeoutError, ProtocolError))
450 def _is_method_retryable(self, method):
451 """Checks if a given HTTP method should be retried upon, depending if
452 it is included in the allowed_methods
453 """
454 # TODO: For now favor if the Retry implementation sets its own method_whitelist
455 # property outside of our constructor to avoid breaking custom implementations.
456 if "method_whitelist" in self.__dict__:
457 warnings.warn(
458 "Using 'method_whitelist' with Retry is deprecated and "
459 "will be removed in v2.0. Use 'allowed_methods' instead",
460 DeprecationWarning,
461 )
462 allowed_methods = self.method_whitelist
463 else:
464 allowed_methods = self.allowed_methods
466 if allowed_methods and method.upper() not in allowed_methods:
467 return False
468 return True
470 def is_retry(self, method, status_code, has_retry_after=False):
471 """Is this method/status code retryable? (Based on allowlists and control
472 variables such as the number of total retries to allow, whether to
473 respect the Retry-After header, whether this header is present, and
474 whether the returned status code is on the list of status codes to
475 be retried upon on the presence of the aforementioned header)
476 """
477 if not self._is_method_retryable(method):
478 return False
480 if self.status_forcelist and status_code in self.status_forcelist:
481 return True
483 return (
484 self.total
485 and self.respect_retry_after_header
486 and has_retry_after
487 and (status_code in self.RETRY_AFTER_STATUS_CODES)
488 )
490 def is_exhausted(self):
491 """Are we out of retries?"""
492 retry_counts = (
493 self.total,
494 self.connect,
495 self.read,
496 self.redirect,
497 self.status,
498 self.other,
499 )
500 retry_counts = list(filter(None, retry_counts))
501 if not retry_counts:
502 return False
504 return min(retry_counts) < 0
506 def increment(
507 self,
508 method=None,
509 url=None,
510 response=None,
511 error=None,
512 _pool=None,
513 _stacktrace=None,
514 ):
515 """Return a new Retry object with incremented retry counters.
517 :param response: A response object, or None, if the server did not
518 return a response.
519 :type response: :class:`~urllib3.response.HTTPResponse`
520 :param Exception error: An error encountered during the request, or
521 None if the response was received successfully.
523 :return: A new ``Retry`` object.
524 """
525 if self.total is False and error:
526 # Disabled, indicate to re-raise the error.
527 raise six.reraise(type(error), error, _stacktrace)
529 total = self.total
530 if total is not None:
531 total -= 1
533 connect = self.connect
534 read = self.read
535 redirect = self.redirect
536 status_count = self.status
537 other = self.other
538 cause = "unknown"
539 status = None
540 redirect_location = None
542 if error and self._is_connection_error(error):
543 # Connect retry?
544 if connect is False:
545 raise six.reraise(type(error), error, _stacktrace)
546 elif connect is not None:
547 connect -= 1
549 elif error and self._is_read_error(error):
550 # Read retry?
551 if read is False or not self._is_method_retryable(method):
552 raise six.reraise(type(error), error, _stacktrace)
553 elif read is not None:
554 read -= 1
556 elif error:
557 # Other retry?
558 if other is not None:
559 other -= 1
561 elif response and response.get_redirect_location():
562 # Redirect retry?
563 if redirect is not None:
564 redirect -= 1
565 cause = "too many redirects"
566 redirect_location = response.get_redirect_location()
567 status = response.status
569 else:
570 # Incrementing because of a server error like a 500 in
571 # status_forcelist and the given method is in the allowed_methods
572 cause = ResponseError.GENERIC_ERROR
573 if response and response.status:
574 if status_count is not None:
575 status_count -= 1
576 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
577 status = response.status
579 history = self.history + (
580 RequestHistory(method, url, error, status, redirect_location),
581 )
583 new_retry = self.new(
584 total=total,
585 connect=connect,
586 read=read,
587 redirect=redirect,
588 status=status_count,
589 other=other,
590 history=history,
591 )
593 if new_retry.is_exhausted():
594 raise MaxRetryError(_pool, url, error or ResponseError(cause))
596 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
598 return new_retry
600 def __repr__(self):
601 return (
602 "{cls.__name__}(total={self.total}, connect={self.connect}, "
603 "read={self.read}, redirect={self.redirect}, status={self.status})"
604 ).format(cls=type(self), self=self)
606 def __getattr__(self, item):
607 if item == "method_whitelist":
608 # TODO: Remove this deprecated alias in v2.0
609 warnings.warn(
610 "Using 'method_whitelist' with Retry is deprecated and "
611 "will be removed in v2.0. Use 'allowed_methods' instead",
612 DeprecationWarning,
613 )
614 return self.allowed_methods
615 try:
616 return getattr(super(Retry, self), item)
617 except AttributeError:
618 return getattr(Retry, item)
621# For backwards compatibility (equivalent to pre-v1.9):
622Retry.DEFAULT = Retry(3)