Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/api_core/retry.py: 70%
81 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 06:03 +0000
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Helpers for retrying functions with exponential back-off.
17The :class:`Retry` decorator can be used to retry functions that raise
18exceptions using exponential backoff. Because a exponential sleep algorithm is
19used, the retry is limited by a `deadline`. The deadline is the maxmimum amount
20of time a method can block. This is used instead of total number of retries
21because it is difficult to ascertain the amount of time a function can block
22when using total number of retries and exponential backoff.
24By default, this decorator will retry transient
25API errors (see :func:`if_transient_error`). For example:
27.. code-block:: python
29 @retry.Retry()
30 def call_flaky_rpc():
31 return client.flaky_rpc()
33 # Will retry flaky_rpc() if it raises transient API errors.
34 result = call_flaky_rpc()
36You can pass a custom predicate to retry on different exceptions, such as
37waiting for an eventually consistent item to be available:
39.. code-block:: python
41 @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
42 def check_if_exists():
43 return client.does_thing_exist()
45 is_available = check_if_exists()
47Some client library methods apply retry automatically. These methods can accept
48a ``retry`` parameter that allows you to configure the behavior:
50.. code-block:: python
52 my_retry = retry.Retry(deadline=60)
53 result = client.some_method(retry=my_retry)
55"""
57from __future__ import unicode_literals
59import datetime
60import functools
61import logging
62import random
63import time
65import requests.exceptions
67from google.api_core import datetime_helpers
68from google.api_core import exceptions
69from google.auth import exceptions as auth_exceptions
71_LOGGER = logging.getLogger(__name__)
72_DEFAULT_INITIAL_DELAY = 1.0 # seconds
73_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
74_DEFAULT_DELAY_MULTIPLIER = 2.0
75_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
78def if_exception_type(*exception_types):
79 """Creates a predicate to check if the exception is of a given type.
81 Args:
82 exception_types (Sequence[:func:`type`]): The exception types to check
83 for.
85 Returns:
86 Callable[Exception]: A predicate that returns True if the provided
87 exception is of the given type(s).
88 """
90 def if_exception_type_predicate(exception):
91 """Bound predicate for checking an exception type."""
92 return isinstance(exception, exception_types)
94 return if_exception_type_predicate
97# pylint: disable=invalid-name
98# Pylint sees this as a constant, but it is also an alias that should be
99# considered a function.
100if_transient_error = if_exception_type(
101 exceptions.InternalServerError,
102 exceptions.TooManyRequests,
103 exceptions.ServiceUnavailable,
104 requests.exceptions.ConnectionError,
105 requests.exceptions.ChunkedEncodingError,
106 auth_exceptions.TransportError,
107)
108"""A predicate that checks if an exception is a transient API error.
110The following server errors are considered transient:
112- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
113 ``INTERNAL(13)`` and its subclasses.
114- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
115- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
116- :class:`requests.exceptions.ConnectionError`
117- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
118 chunked encoding but sent an invalid chunk.
119- :class:`google.auth.exceptions.TransportError` - Used to indicate an
120 error occurred during an HTTP request.
121"""
122# pylint: enable=invalid-name
125def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER):
126 """Generates sleep intervals based on the exponential back-off algorithm.
128 This implements the `Truncated Exponential Back-off`_ algorithm.
130 .. _Truncated Exponential Back-off:
131 https://cloud.google.com/storage/docs/exponential-backoff
133 Args:
134 initial (float): The minimum amount of time to delay. This must
135 be greater than 0.
136 maximum (float): The maximum amount of time to delay.
137 multiplier (float): The multiplier applied to the delay.
139 Yields:
140 float: successive sleep intervals.
141 """
142 delay = min(initial, maximum)
143 while True:
144 yield random.uniform(0.0, delay)
145 delay = min(delay * multiplier, maximum)
148def retry_target(
149 target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs
150):
151 """Call a function and retry if it fails.
153 This is the lowest-level retry helper. Generally, you'll use the
154 higher-level retry helper :class:`Retry`.
156 Args:
157 target(Callable): The function to call and retry. This must be a
158 nullary function - apply arguments with `functools.partial`.
159 predicate (Callable[Exception]): A callable used to determine if an
160 exception raised by the target should be considered retryable.
161 It should return True to retry or False otherwise.
162 sleep_generator (Iterable[float]): An infinite iterator that determines
163 how long to sleep between retries.
164 timeout (float): How long to keep retrying the target.
165 on_error (Callable[Exception]): A function to call while processing a
166 retryable exception. Any error raised by this function will *not*
167 be caught.
168 deadline (float): DEPRECATED: use ``timeout`` instead. For backward
169 compatibility, if specified it will override ``timeout`` parameter.
171 Returns:
172 Any: the return value of the target function.
174 Raises:
175 google.api_core.RetryError: If the deadline is exceeded while retrying.
176 ValueError: If the sleep generator stops yielding values.
177 Exception: If the target raises a method that isn't retryable.
178 """
180 timeout = kwargs.get("deadline", timeout)
182 if timeout is not None:
183 deadline = datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout)
184 else:
185 deadline = None
187 last_exc = None
189 for sleep in sleep_generator:
190 try:
191 return target()
193 # pylint: disable=broad-except
194 # This function explicitly must deal with broad exceptions.
195 except Exception as exc:
196 if not predicate(exc):
197 raise
198 last_exc = exc
199 if on_error is not None:
200 on_error(exc)
202 if deadline is not None:
203 next_attempt_time = datetime_helpers.utcnow() + datetime.timedelta(
204 seconds=sleep
205 )
206 if deadline < next_attempt_time:
207 raise exceptions.RetryError(
208 "Deadline of {:.1f}s exceeded while calling target function".format(
209 timeout
210 ),
211 last_exc,
212 ) from last_exc
214 _LOGGER.debug(
215 "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
216 )
217 time.sleep(sleep)
219 raise ValueError("Sleep generator stopped yielding sleep values.")
222class Retry(object):
223 """Exponential retry decorator.
225 This class is a decorator used to add retry or polling behavior to an RPC
226 call.
228 Although the default behavior is to retry transient API errors, a
229 different predicate can be provided to retry other exceptions.
231 There two important concepts that retry/polling behavior may operate on,
232 Deadline and Timeout, which need to be properly defined for the correct
233 usage of this class and the rest of the library.
235 Deadline: a fixed point in time by which a certain operation must
236 terminate. For example, if a certain operation has a deadline
237 "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
238 error) by that time, regardless of when it was started or whether it
239 was started at all.
241 Timeout: the maximum duration of time after which a certain operation
242 must terminate (successfully or with an error). The countdown begins right
243 after an operation was started. For example, if an operation was started at
244 09:24:00 with timeout of 75 seconds, it must terminate no later than
245 09:25:15.
247 Unfortunately, in the past this class (and the api-core library as a whole) has not been
248 properly distinguishing the concepts of "timeout" and "deadline", and the
249 ``deadline`` parameter has meant ``timeout``. That is why
250 ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
251 ``deadline`` parameter is set, it will override the ``timeout`` parameter. In other words,
252 ``retry.deadline`` should be treated as just a deprecated alias for
253 ``retry.timeout``.
255 Said another way, it is safe to assume that this class and the rest of this
256 library operate in terms of timeouts (not deadlines) unless explicitly
257 noted the usage of deadline semantics.
259 It is also important to
260 understand the three most common applications of the Timeout concept in the
261 context of this library.
263 Usually the generic Timeout term may stand for one of the following actual
264 timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
266 RPC Timeout: a value supplied by the client to the server so
267 that the server side knows the maximum amount of time it is expected to
268 spend handling that specifc RPC. For example, in the case of gRPC transport,
269 RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
270 request. The `timeout` property of this class normally never represents the
271 RPC Timeout as it is handled separately by the ``google.api_core.timeout``
272 module of this library.
274 Retry Timeout: this is the most common meaning of the ``timeout`` property
275 of this class, and defines how long a certain RPC may be retried in case
276 the server returns an error.
278 Polling Timeout: defines how long the
279 client side is allowed to call the polling RPC repeatedly to check a status of a
280 long-running operation. Each polling RPC is
281 expected to succeed (its errors are supposed to be handled by the retry
282 logic). The decision as to whether a new polling attempt needs to be made is based
283 not on the RPC status code but on the status of the returned
284 status of an operation. In other words: we will poll a long-running operation until the operation is done or the polling timeout expires. Each poll will inform us of the status of the operation. The poll consists of an RPC to the server that may itself be retried as per the poll-specific retry settings in case of errors. The operation-level retry settings do NOT apply to polling-RPC retries.
286 With the actual timeout types being defined above, the client libraries
287 often refer to just Timeout without clarifying which type specifically
288 that is. In that case the actual timeout type (sometimes also refered to as
289 Logical Timeout) can be determined from the context. If it is a unary rpc
290 call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
291 provided directly as a standalone value) or Retry Timeout (if provided as
292 ``retry.timeout`` property of the unary RPC's retry config). For
293 ``Operation`` or ``PollingFuture`` in general Timeout stands for
294 Polling Timeout.
296 Args:
297 predicate (Callable[Exception]): A callable that should return ``True``
298 if the given exception is retryable.
299 initial (float): The minimum amount of time to delay in seconds. This
300 must be greater than 0.
301 maximum (float): The maximum amount of time to delay in seconds.
302 multiplier (float): The multiplier applied to the delay.
303 timeout (float): How long to keep retrying, in seconds.
304 deadline (float): DEPRECATED: use `timeout` instead. For backward
305 compatibility, if specified it will override the ``timeout`` parameter.
306 """
308 def __init__(
309 self,
310 predicate=if_transient_error,
311 initial=_DEFAULT_INITIAL_DELAY,
312 maximum=_DEFAULT_MAXIMUM_DELAY,
313 multiplier=_DEFAULT_DELAY_MULTIPLIER,
314 timeout=_DEFAULT_DEADLINE,
315 on_error=None,
316 **kwargs
317 ):
318 self._predicate = predicate
319 self._initial = initial
320 self._multiplier = multiplier
321 self._maximum = maximum
322 self._timeout = kwargs.get("deadline", timeout)
323 self._deadline = self._timeout
324 self._on_error = on_error
326 def __call__(self, func, on_error=None):
327 """Wrap a callable with retry behavior.
329 Args:
330 func (Callable): The callable to add retry behavior to.
331 on_error (Callable[Exception]): A function to call while processing
332 a retryable exception. Any error raised by this function will
333 *not* be caught.
335 Returns:
336 Callable: A callable that will invoke ``func`` with retry
337 behavior.
338 """
339 if self._on_error is not None:
340 on_error = self._on_error
342 @functools.wraps(func)
343 def retry_wrapped_func(*args, **kwargs):
344 """A wrapper that calls target function with retry."""
345 target = functools.partial(func, *args, **kwargs)
346 sleep_generator = exponential_sleep_generator(
347 self._initial, self._maximum, multiplier=self._multiplier
348 )
349 return retry_target(
350 target,
351 self._predicate,
352 sleep_generator,
353 self._timeout,
354 on_error=on_error,
355 )
357 return retry_wrapped_func
359 @property
360 def deadline(self):
361 """
362 DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
363 documentation for details.
364 """
365 return self._timeout
367 @property
368 def timeout(self):
369 return self._timeout
371 def with_deadline(self, deadline):
372 """Return a copy of this retry with the given timeout.
374 DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
375 documentation for details.
377 Args:
378 deadline (float): How long to keep retrying in seconds.
380 Returns:
381 Retry: A new retry instance with the given timeout.
382 """
383 return self.with_timeout(timeout=deadline)
385 def with_timeout(self, timeout):
386 """Return a copy of this retry with the given timeout.
388 Args:
389 timeout (float): How long to keep retrying, in seconds.
391 Returns:
392 Retry: A new retry instance with the given timeout.
393 """
394 return Retry(
395 predicate=self._predicate,
396 initial=self._initial,
397 maximum=self._maximum,
398 multiplier=self._multiplier,
399 timeout=timeout,
400 on_error=self._on_error,
401 )
403 def with_predicate(self, predicate):
404 """Return a copy of this retry with the given predicate.
406 Args:
407 predicate (Callable[Exception]): A callable that should return
408 ``True`` if the given exception is retryable.
410 Returns:
411 Retry: A new retry instance with the given predicate.
412 """
413 return Retry(
414 predicate=predicate,
415 initial=self._initial,
416 maximum=self._maximum,
417 multiplier=self._multiplier,
418 timeout=self._timeout,
419 on_error=self._on_error,
420 )
422 def with_delay(self, initial=None, maximum=None, multiplier=None):
423 """Return a copy of this retry with the given delay options.
425 Args:
426 initial (float): The minimum amount of time to delay. This must
427 be greater than 0.
428 maximum (float): The maximum amount of time to delay.
429 multiplier (float): The multiplier applied to the delay.
431 Returns:
432 Retry: A new retry instance with the given predicate.
433 """
434 return Retry(
435 predicate=self._predicate,
436 initial=initial if initial is not None else self._initial,
437 maximum=maximum if maximum is not None else self._maximum,
438 multiplier=multiplier if multiplier is not None else self._multiplier,
439 timeout=self._timeout,
440 on_error=self._on_error,
441 )
443 def __str__(self):
444 return (
445 "<Retry predicate={}, initial={:.1f}, maximum={:.1f}, "
446 "multiplier={:.1f}, timeout={}, on_error={}>".format(
447 self._predicate,
448 self._initial,
449 self._maximum,
450 self._multiplier,
451 self._timeout, # timeout can be None, thus no {:.1f}
452 self._on_error,
453 )
454 )