Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/api_core/retry.py: 49%
95 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:45 +0000
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Helpers for retrying functions with exponential back-off.
17The :class:`Retry` decorator can be used to retry functions that raise
18exceptions using exponential backoff. Because a exponential sleep algorithm is
19used, the retry is limited by a `deadline`. The deadline is the maximum amount
20of time a method can block. This is used instead of total number of retries
21because it is difficult to ascertain the amount of time a function can block
22when using total number of retries and exponential backoff.
24By default, this decorator will retry transient
25API errors (see :func:`if_transient_error`). For example:
27.. code-block:: python
29 @retry.Retry()
30 def call_flaky_rpc():
31 return client.flaky_rpc()
33 # Will retry flaky_rpc() if it raises transient API errors.
34 result = call_flaky_rpc()
36You can pass a custom predicate to retry on different exceptions, such as
37waiting for an eventually consistent item to be available:
39.. code-block:: python
41 @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
42 def check_if_exists():
43 return client.does_thing_exist()
45 is_available = check_if_exists()
47Some client library methods apply retry automatically. These methods can accept
48a ``retry`` parameter that allows you to configure the behavior:
50.. code-block:: python
52 my_retry = retry.Retry(deadline=60)
53 result = client.some_method(retry=my_retry)
55"""
57from __future__ import annotations
59import datetime
60import functools
61import logging
62import random
63import sys
64import time
65import inspect
66import warnings
67from typing import Any, Callable, TypeVar, TYPE_CHECKING
69import requests.exceptions
71from google.api_core import datetime_helpers
72from google.api_core import exceptions
73from google.auth import exceptions as auth_exceptions
75if TYPE_CHECKING:
76 if sys.version_info >= (3, 10):
77 from typing import ParamSpec
78 else:
79 from typing_extensions import ParamSpec
81 _P = ParamSpec("_P")
82 _R = TypeVar("_R")
84_LOGGER = logging.getLogger(__name__)
85_DEFAULT_INITIAL_DELAY = 1.0 # seconds
86_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
87_DEFAULT_DELAY_MULTIPLIER = 2.0
88_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
89_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead."
92def if_exception_type(
93 *exception_types: type[BaseException],
94) -> Callable[[BaseException], bool]:
95 """Creates a predicate to check if the exception is of a given type.
97 Args:
98 exception_types (Sequence[:func:`type`]): The exception types to check
99 for.
101 Returns:
102 Callable[Exception]: A predicate that returns True if the provided
103 exception is of the given type(s).
104 """
106 def if_exception_type_predicate(exception: BaseException) -> bool:
107 """Bound predicate for checking an exception type."""
108 return isinstance(exception, exception_types)
110 return if_exception_type_predicate
113# pylint: disable=invalid-name
114# Pylint sees this as a constant, but it is also an alias that should be
115# considered a function.
116if_transient_error = if_exception_type(
117 exceptions.InternalServerError,
118 exceptions.TooManyRequests,
119 exceptions.ServiceUnavailable,
120 requests.exceptions.ConnectionError,
121 requests.exceptions.ChunkedEncodingError,
122 auth_exceptions.TransportError,
123)
124"""A predicate that checks if an exception is a transient API error.
126The following server errors are considered transient:
128- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
129 ``INTERNAL(13)`` and its subclasses.
130- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
131- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
132- :class:`requests.exceptions.ConnectionError`
133- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
134 chunked encoding but sent an invalid chunk.
135- :class:`google.auth.exceptions.TransportError` - Used to indicate an
136 error occurred during an HTTP request.
137"""
138# pylint: enable=invalid-name
141def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER):
142 """Generates sleep intervals based on the exponential back-off algorithm.
144 This implements the `Truncated Exponential Back-off`_ algorithm.
146 .. _Truncated Exponential Back-off:
147 https://cloud.google.com/storage/docs/exponential-backoff
149 Args:
150 initial (float): The minimum amount of time to delay. This must
151 be greater than 0.
152 maximum (float): The maximum amount of time to delay.
153 multiplier (float): The multiplier applied to the delay.
155 Yields:
156 float: successive sleep intervals.
157 """
158 delay = min(initial, maximum)
159 while True:
160 yield random.uniform(0.0, delay)
161 delay = min(delay * multiplier, maximum)
164def retry_target(
165 target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs
166):
167 """Call a function and retry if it fails.
169 This is the lowest-level retry helper. Generally, you'll use the
170 higher-level retry helper :class:`Retry`.
172 Args:
173 target(Callable): The function to call and retry. This must be a
174 nullary function - apply arguments with `functools.partial`.
175 predicate (Callable[Exception]): A callable used to determine if an
176 exception raised by the target should be considered retryable.
177 It should return True to retry or False otherwise.
178 sleep_generator (Iterable[float]): An infinite iterator that determines
179 how long to sleep between retries.
180 timeout (float): How long to keep retrying the target.
181 on_error (Callable[Exception]): A function to call while processing a
182 retryable exception. Any error raised by this function will *not*
183 be caught.
184 deadline (float): DEPRECATED: use ``timeout`` instead. For backward
185 compatibility, if specified it will override ``timeout`` parameter.
187 Returns:
188 Any: the return value of the target function.
190 Raises:
191 google.api_core.RetryError: If the deadline is exceeded while retrying.
192 ValueError: If the sleep generator stops yielding values.
193 Exception: If the target raises a method that isn't retryable.
194 """
196 timeout = kwargs.get("deadline", timeout)
198 if timeout is not None:
199 deadline = datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout)
200 else:
201 deadline = None
203 last_exc = None
205 for sleep in sleep_generator:
206 try:
207 result = target()
208 if inspect.isawaitable(result):
209 warnings.warn(_ASYNC_RETRY_WARNING)
210 return result
212 # pylint: disable=broad-except
213 # This function explicitly must deal with broad exceptions.
214 except Exception as exc:
215 if not predicate(exc):
216 raise
217 last_exc = exc
218 if on_error is not None:
219 on_error(exc)
221 if deadline is not None:
222 next_attempt_time = datetime_helpers.utcnow() + datetime.timedelta(
223 seconds=sleep
224 )
225 if deadline < next_attempt_time:
226 raise exceptions.RetryError(
227 "Deadline of {:.1f}s exceeded while calling target function".format(
228 timeout
229 ),
230 last_exc,
231 ) from last_exc
233 _LOGGER.debug(
234 "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
235 )
236 time.sleep(sleep)
238 raise ValueError("Sleep generator stopped yielding sleep values.")
241class Retry(object):
242 """Exponential retry decorator.
244 This class is a decorator used to add retry or polling behavior to an RPC
245 call.
247 Although the default behavior is to retry transient API errors, a
248 different predicate can be provided to retry other exceptions.
250 There two important concepts that retry/polling behavior may operate on,
251 Deadline and Timeout, which need to be properly defined for the correct
252 usage of this class and the rest of the library.
254 Deadline: a fixed point in time by which a certain operation must
255 terminate. For example, if a certain operation has a deadline
256 "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
257 error) by that time, regardless of when it was started or whether it
258 was started at all.
260 Timeout: the maximum duration of time after which a certain operation
261 must terminate (successfully or with an error). The countdown begins right
262 after an operation was started. For example, if an operation was started at
263 09:24:00 with timeout of 75 seconds, it must terminate no later than
264 09:25:15.
266 Unfortunately, in the past this class (and the api-core library as a whole) has not been
267 properly distinguishing the concepts of "timeout" and "deadline", and the
268 ``deadline`` parameter has meant ``timeout``. That is why
269 ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
270 ``deadline`` parameter is set, it will override the ``timeout`` parameter. In other words,
271 ``retry.deadline`` should be treated as just a deprecated alias for
272 ``retry.timeout``.
274 Said another way, it is safe to assume that this class and the rest of this
275 library operate in terms of timeouts (not deadlines) unless explicitly
276 noted the usage of deadline semantics.
278 It is also important to
279 understand the three most common applications of the Timeout concept in the
280 context of this library.
282 Usually the generic Timeout term may stand for one of the following actual
283 timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
285 RPC Timeout: a value supplied by the client to the server so
286 that the server side knows the maximum amount of time it is expected to
287 spend handling that specific RPC. For example, in the case of gRPC transport,
288 RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
289 request. The `timeout` property of this class normally never represents the
290 RPC Timeout as it is handled separately by the ``google.api_core.timeout``
291 module of this library.
293 Retry Timeout: this is the most common meaning of the ``timeout`` property
294 of this class, and defines how long a certain RPC may be retried in case
295 the server returns an error.
297 Polling Timeout: defines how long the
298 client side is allowed to call the polling RPC repeatedly to check a status of a
299 long-running operation. Each polling RPC is
300 expected to succeed (its errors are supposed to be handled by the retry
301 logic). The decision as to whether a new polling attempt needs to be made is based
302 not on the RPC status code but on the status of the returned
303 status of an operation. In other words: we will poll a long-running operation until the operation is done or the polling timeout expires. Each poll will inform us of the status of the operation. The poll consists of an RPC to the server that may itself be retried as per the poll-specific retry settings in case of errors. The operation-level retry settings do NOT apply to polling-RPC retries.
305 With the actual timeout types being defined above, the client libraries
306 often refer to just Timeout without clarifying which type specifically
307 that is. In that case the actual timeout type (sometimes also referred to as
308 Logical Timeout) can be determined from the context. If it is a unary rpc
309 call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
310 provided directly as a standalone value) or Retry Timeout (if provided as
311 ``retry.timeout`` property of the unary RPC's retry config). For
312 ``Operation`` or ``PollingFuture`` in general Timeout stands for
313 Polling Timeout.
315 Args:
316 predicate (Callable[Exception]): A callable that should return ``True``
317 if the given exception is retryable.
318 initial (float): The minimum amount of time to delay in seconds. This
319 must be greater than 0.
320 maximum (float): The maximum amount of time to delay in seconds.
321 multiplier (float): The multiplier applied to the delay.
322 timeout (float): How long to keep retrying, in seconds.
323 deadline (float): DEPRECATED: use `timeout` instead. For backward
324 compatibility, if specified it will override the ``timeout`` parameter.
325 """
327 def __init__(
328 self,
329 predicate: Callable[[BaseException], bool] = if_transient_error,
330 initial: float = _DEFAULT_INITIAL_DELAY,
331 maximum: float = _DEFAULT_MAXIMUM_DELAY,
332 multiplier: float = _DEFAULT_DELAY_MULTIPLIER,
333 timeout: float = _DEFAULT_DEADLINE,
334 on_error: Callable[[BaseException], Any] | None = None,
335 **kwargs: Any,
336 ) -> None:
337 self._predicate = predicate
338 self._initial = initial
339 self._multiplier = multiplier
340 self._maximum = maximum
341 self._timeout = kwargs.get("deadline", timeout)
342 self._deadline = self._timeout
343 self._on_error = on_error
345 def __call__(
346 self,
347 func: Callable[_P, _R],
348 on_error: Callable[[BaseException], Any] | None = None,
349 ) -> Callable[_P, _R]:
350 """Wrap a callable with retry behavior.
352 Args:
353 func (Callable): The callable to add retry behavior to.
354 on_error (Callable[Exception]): A function to call while processing
355 a retryable exception. Any error raised by this function will
356 *not* be caught.
358 Returns:
359 Callable: A callable that will invoke ``func`` with retry
360 behavior.
361 """
362 if self._on_error is not None:
363 on_error = self._on_error
365 @functools.wraps(func)
366 def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:
367 """A wrapper that calls target function with retry."""
368 target = functools.partial(func, *args, **kwargs)
369 sleep_generator = exponential_sleep_generator(
370 self._initial, self._maximum, multiplier=self._multiplier
371 )
372 return retry_target(
373 target,
374 self._predicate,
375 sleep_generator,
376 self._timeout,
377 on_error=on_error,
378 )
380 return retry_wrapped_func
382 @property
383 def deadline(self):
384 """
385 DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
386 documentation for details.
387 """
388 return self._timeout
390 @property
391 def timeout(self):
392 return self._timeout
394 def with_deadline(self, deadline):
395 """Return a copy of this retry with the given timeout.
397 DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
398 documentation for details.
400 Args:
401 deadline (float): How long to keep retrying in seconds.
403 Returns:
404 Retry: A new retry instance with the given timeout.
405 """
406 return self.with_timeout(timeout=deadline)
408 def with_timeout(self, timeout):
409 """Return a copy of this retry with the given timeout.
411 Args:
412 timeout (float): How long to keep retrying, in seconds.
414 Returns:
415 Retry: A new retry instance with the given timeout.
416 """
417 return Retry(
418 predicate=self._predicate,
419 initial=self._initial,
420 maximum=self._maximum,
421 multiplier=self._multiplier,
422 timeout=timeout,
423 on_error=self._on_error,
424 )
426 def with_predicate(self, predicate):
427 """Return a copy of this retry with the given predicate.
429 Args:
430 predicate (Callable[Exception]): A callable that should return
431 ``True`` if the given exception is retryable.
433 Returns:
434 Retry: A new retry instance with the given predicate.
435 """
436 return Retry(
437 predicate=predicate,
438 initial=self._initial,
439 maximum=self._maximum,
440 multiplier=self._multiplier,
441 timeout=self._timeout,
442 on_error=self._on_error,
443 )
445 def with_delay(self, initial=None, maximum=None, multiplier=None):
446 """Return a copy of this retry with the given delay options.
448 Args:
449 initial (float): The minimum amount of time to delay. This must
450 be greater than 0.
451 maximum (float): The maximum amount of time to delay.
452 multiplier (float): The multiplier applied to the delay.
454 Returns:
455 Retry: A new retry instance with the given predicate.
456 """
457 return Retry(
458 predicate=self._predicate,
459 initial=initial if initial is not None else self._initial,
460 maximum=maximum if maximum is not None else self._maximum,
461 multiplier=multiplier if multiplier is not None else self._multiplier,
462 timeout=self._timeout,
463 on_error=self._on_error,
464 )
466 def __str__(self):
467 return (
468 "<Retry predicate={}, initial={:.1f}, maximum={:.1f}, "
469 "multiplier={:.1f}, timeout={}, on_error={}>".format(
470 self._predicate,
471 self._initial,
472 self._maximum,
473 self._multiplier,
474 self._timeout, # timeout can be None, thus no {:.1f}
475 self._on_error,
476 )
477 )