Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Helpers for retrying functions with exponential back-off.
17The :class:`Retry` decorator can be used to retry functions that raise
18exceptions using exponential backoff. Because a exponential sleep algorithm is
19used, the retry is limited by a `timeout`. The timeout determines the window
20in which retries will be attempted. This is used instead of total number of retries
21because it is difficult to ascertain the amount of time a function can block
22when using total number of retries and exponential backoff.
24By default, this decorator will retry transient
25API errors (see :func:`if_transient_error`). For example:
27.. code-block:: python
29 @retry.Retry()
30 def call_flaky_rpc():
31 return client.flaky_rpc()
33 # Will retry flaky_rpc() if it raises transient API errors.
34 result = call_flaky_rpc()
36You can pass a custom predicate to retry on different exceptions, such as
37waiting for an eventually consistent item to be available:
39.. code-block:: python
41 @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
42 def check_if_exists():
43 return client.does_thing_exist()
45 is_available = check_if_exists()
47Some client library methods apply retry automatically. These methods can accept
48a ``retry`` parameter that allows you to configure the behavior:
50.. code-block:: python
52 my_retry = retry.Retry(timeout=60)
53 result = client.some_method(retry=my_retry)
55"""
57from __future__ import annotations
59import functools
60import inspect
61import sys
62import time
63import warnings
64from typing import TYPE_CHECKING, Any, Callable, Iterable, TypeVar
66from google.api_core.retry.retry_base import (
67 RetryFailureReason,
68 _BaseRetry,
69 _retry_error_helper,
70 build_retry_error,
71 exponential_sleep_generator,
72)
74if TYPE_CHECKING:
75 if sys.version_info >= (3, 10):
76 from typing import ParamSpec
77 else:
78 from typing_extensions import ParamSpec
80 _P = ParamSpec("_P") # target function call parameters
81 _R = TypeVar("_R") # target function returned value
83_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead."
86def retry_target(
87 target: Callable[[], _R],
88 predicate: Callable[[Exception], bool],
89 sleep_generator: Iterable[float],
90 timeout: float | None = None,
91 on_error: Callable[[Exception], None] | None = None,
92 exception_factory: Callable[
93 [list[Exception], RetryFailureReason, float | None],
94 tuple[Exception, Exception | None],
95 ] = build_retry_error,
96 **kwargs,
97):
98 """Call a function and retry if it fails.
100 This is the lowest-level retry helper. Generally, you'll use the
101 higher-level retry helper :class:`Retry`.
103 Args:
104 target(Callable): The function to call and retry. This must be a
105 nullary function - apply arguments with `functools.partial`.
106 predicate (Callable[Exception]): A callable used to determine if an
107 exception raised by the target should be considered retryable.
108 It should return True to retry or False otherwise.
109 sleep_generator (Iterable[float]): An infinite iterator that determines
110 how long to sleep between retries.
111 timeout (Optional[float]): How long to keep retrying the target.
112 Note: timeout is only checked before initiating a retry, so the target may
113 run past the timeout value as long as it is healthy.
114 on_error (Optional[Callable[Exception]]): If given, the on_error
115 callback will be called with each retryable exception raised by the
116 target. Any error raised by this function will *not* be caught.
117 exception_factory: A function that is called when the retryable reaches
118 a terminal failure state, used to construct an exception to be raised.
119 It takes a list of all exceptions encountered, a retry.RetryFailureReason
120 enum indicating the failure cause, and the original timeout value
121 as arguments. It should return a tuple of the exception to be raised,
122 along with the cause exception if any. The default implementation will raise
123 a RetryError on timeout, or the last exception encountered otherwise.
124 deadline (float): DEPRECATED: use ``timeout`` instead. For backward
125 compatibility, if specified it will override ``timeout`` parameter.
127 Returns:
128 Any: the return value of the target function.
130 Raises:
131 ValueError: If the sleep generator stops yielding values.
132 Exception: a custom exception specified by the exception_factory if provided.
133 If no exception_factory is provided:
134 google.api_core.RetryError: If the timeout is exceeded while retrying.
135 Exception: If the target raises an error that isn't retryable.
136 """
138 timeout = kwargs.get("deadline", timeout)
140 deadline = time.monotonic() + timeout if timeout is not None else None
141 error_list: list[Exception] = []
142 sleep_iter = iter(sleep_generator)
144 # continue trying until an attempt completes, or a terminal exception is raised in _retry_error_helper
145 # TODO: support max_attempts argument: https://github.com/googleapis/python-api-core/issues/535
146 while True:
147 try:
148 result = target()
149 if inspect.isawaitable(result):
150 warnings.warn(_ASYNC_RETRY_WARNING)
151 return result
153 # pylint: disable=broad-except
154 # This function explicitly must deal with broad exceptions.
155 except Exception as exc:
156 # defer to shared logic for handling errors
157 next_sleep = _retry_error_helper(
158 exc,
159 deadline,
160 sleep_iter,
161 error_list,
162 predicate,
163 on_error,
164 exception_factory,
165 timeout,
166 )
167 # if exception not raised, sleep before next attempt
168 time.sleep(next_sleep)
171class Retry(_BaseRetry):
172 """Exponential retry decorator for unary synchronous RPCs.
174 This class is a decorator used to add retry or polling behavior to an RPC
175 call.
177 Although the default behavior is to retry transient API errors, a
178 different predicate can be provided to retry other exceptions.
180 There are two important concepts that retry/polling behavior may operate on,
181 Deadline and Timeout, which need to be properly defined for the correct
182 usage of this class and the rest of the library.
184 Deadline: a fixed point in time by which a certain operation must
185 terminate. For example, if a certain operation has a deadline
186 "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
187 error) by that time, regardless of when it was started or whether it
188 was started at all.
190 Timeout: the maximum duration of time after which a certain operation
191 must terminate (successfully or with an error). The countdown begins right
192 after an operation was started. For example, if an operation was started at
193 09:24:00 with timeout of 75 seconds, it must terminate no later than
194 09:25:15.
196 Unfortunately, in the past this class (and the api-core library as a whole) has not
197 been properly distinguishing the concepts of "timeout" and "deadline", and the
198 ``deadline`` parameter has meant ``timeout``. That is why
199 ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
200 ``deadline`` parameter is set, it will override the ``timeout`` parameter.
201 In other words, ``retry.deadline`` should be treated as just a deprecated alias for
202 ``retry.timeout``.
204 Said another way, it is safe to assume that this class and the rest of this
205 library operate in terms of timeouts (not deadlines) unless explicitly
206 noted the usage of deadline semantics.
208 It is also important to
209 understand the three most common applications of the Timeout concept in the
210 context of this library.
212 Usually the generic Timeout term may stand for one of the following actual
213 timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
215 RPC Timeout: a value supplied by the client to the server so
216 that the server side knows the maximum amount of time it is expected to
217 spend handling that specific RPC. For example, in the case of gRPC transport,
218 RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
219 request. The `timeout` property of this class normally never represents the
220 RPC Timeout as it is handled separately by the ``google.api_core.timeout``
221 module of this library.
223 Retry Timeout: this is the most common meaning of the ``timeout`` property
224 of this class, and defines how long a certain RPC may be retried in case
225 the server returns an error.
227 Polling Timeout: defines how long the
228 client side is allowed to call the polling RPC repeatedly to check a status of a
229 long-running operation. Each polling RPC is
230 expected to succeed (its errors are supposed to be handled by the retry
231 logic). The decision as to whether a new polling attempt needs to be made is based
232 not on the RPC status code but on the status of the returned
233 status of an operation. In other words: we will poll a long-running operation until
234 the operation is done or the polling timeout expires. Each poll will inform us of
235 the status of the operation. The poll consists of an RPC to the server that may
236 itself be retried as per the poll-specific retry settings in case of errors. The
237 operation-level retry settings do NOT apply to polling-RPC retries.
239 With the actual timeout types being defined above, the client libraries
240 often refer to just Timeout without clarifying which type specifically
241 that is. In that case the actual timeout type (sometimes also referred to as
242 Logical Timeout) can be determined from the context. If it is a unary rpc
243 call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
244 provided directly as a standalone value) or Retry Timeout (if provided as
245 ``retry.timeout`` property of the unary RPC's retry config). For
246 ``Operation`` or ``PollingFuture`` in general Timeout stands for
247 Polling Timeout.
249 Args:
250 predicate (Callable[Exception]): A callable that should return ``True``
251 if the given exception is retryable.
252 initial (float): The minimum amount of time to delay in seconds. This
253 must be greater than 0.
254 maximum (float): The maximum amount of time to delay in seconds.
255 multiplier (float): The multiplier applied to the delay.
256 timeout (Optional[float]): How long to keep retrying, in seconds.
257 Note: timeout is only checked before initiating a retry, so the target may
258 run past the timeout value as long as it is healthy.
259 on_error (Callable[Exception]): A function to call while processing
260 a retryable exception. Any error raised by this function will
261 *not* be caught.
262 deadline (float): DEPRECATED: use `timeout` instead. For backward
263 compatibility, if specified it will override the ``timeout`` parameter.
264 """
266 def __call__(
267 self,
268 func: Callable[_P, _R],
269 on_error: Callable[[Exception], Any] | None = None,
270 ) -> Callable[_P, _R]:
271 """Wrap a callable with retry behavior.
273 Args:
274 func (Callable): The callable to add retry behavior to.
275 on_error (Optional[Callable[Exception]]): If given, the
276 on_error callback will be called with each retryable exception
277 raised by the wrapped function. Any error raised by this
278 function will *not* be caught. If on_error was specified in the
279 constructor, this value will be ignored.
281 Returns:
282 Callable: A callable that will invoke ``func`` with retry
283 behavior.
284 """
285 if self._on_error is not None:
286 on_error = self._on_error
288 @functools.wraps(func)
289 def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:
290 """A wrapper that calls target function with retry."""
291 target = functools.partial(func, *args, **kwargs)
292 sleep_generator = exponential_sleep_generator(
293 self._initial, self._maximum, multiplier=self._multiplier
294 )
295 return retry_target(
296 target,
297 self._predicate,
298 sleep_generator,
299 timeout=self._timeout,
300 on_error=on_error,
301 )
303 return retry_wrapped_func