Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Helpers for retrying functions with exponential back-off.
17The :class:`Retry` decorator can be used to retry functions that raise
18exceptions using exponential backoff. Because a exponential sleep algorithm is
19used, the retry is limited by a `timeout`. The timeout determines the window
20in which retries will be attempted. This is used instead of total number of retries
21because it is difficult to ascertain the amount of time a function can block
22when using total number of retries and exponential backoff.
24By default, this decorator will retry transient
25API errors (see :func:`if_transient_error`). For example:
27.. code-block:: python
29 @retry.Retry()
30 def call_flaky_rpc():
31 return client.flaky_rpc()
33 # Will retry flaky_rpc() if it raises transient API errors.
34 result = call_flaky_rpc()
36You can pass a custom predicate to retry on different exceptions, such as
37waiting for an eventually consistent item to be available:
39.. code-block:: python
41 @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
42 def check_if_exists():
43 return client.does_thing_exist()
45 is_available = check_if_exists()
47Some client library methods apply retry automatically. These methods can accept
48a ``retry`` parameter that allows you to configure the behavior:
50.. code-block:: python
52 my_retry = retry.Retry(timeout=60)
53 result = client.some_method(retry=my_retry)
55"""
57from __future__ import annotations
59import functools
60import sys
61import time
62import inspect
63import warnings
64from typing import Any, Callable, Iterable, TypeVar, TYPE_CHECKING
66from google.api_core.retry.retry_base import _BaseRetry
67from google.api_core.retry.retry_base import _retry_error_helper
68from google.api_core.retry.retry_base import exponential_sleep_generator
69from google.api_core.retry.retry_base import build_retry_error
70from google.api_core.retry.retry_base import RetryFailureReason
73if TYPE_CHECKING:
74 if sys.version_info >= (3, 10):
75 from typing import ParamSpec
76 else:
77 from typing_extensions import ParamSpec
79 _P = ParamSpec("_P") # target function call parameters
80 _R = TypeVar("_R") # target function returned value
82_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead."
85def retry_target(
86 target: Callable[_P, _R],
87 predicate: Callable[[Exception], bool],
88 sleep_generator: Iterable[float],
89 timeout: float | None = None,
90 on_error: Callable[[Exception], None] | None = None,
91 exception_factory: Callable[
92 [list[Exception], RetryFailureReason, float | None],
93 tuple[Exception, Exception | None],
94 ] = build_retry_error,
95 **kwargs,
96):
97 """Call a function and retry if it fails.
99 This is the lowest-level retry helper. Generally, you'll use the
100 higher-level retry helper :class:`Retry`.
102 Args:
103 target(Callable): The function to call and retry. This must be a
104 nullary function - apply arguments with `functools.partial`.
105 predicate (Callable[Exception]): A callable used to determine if an
106 exception raised by the target should be considered retryable.
107 It should return True to retry or False otherwise.
108 sleep_generator (Iterable[float]): An infinite iterator that determines
109 how long to sleep between retries.
110 timeout (Optional[float]): How long to keep retrying the target.
111 Note: timeout is only checked before initiating a retry, so the target may
112 run past the timeout value as long as it is healthy.
113 on_error (Optional[Callable[Exception]]): If given, the on_error
114 callback will be called with each retryable exception raised by the
115 target. Any error raised by this function will *not* be caught.
116 exception_factory: A function that is called when the retryable reaches
117 a terminal failure state, used to construct an exception to be raised.
118 It takes a list of all exceptions encountered, a retry.RetryFailureReason
119 enum indicating the failure cause, and the original timeout value
120 as arguments. It should return a tuple of the exception to be raised,
121 along with the cause exception if any. The default implementation will raise
122 a RetryError on timeout, or the last exception encountered otherwise.
123 deadline (float): DEPRECATED: use ``timeout`` instead. For backward
124 compatibility, if specified it will override ``timeout`` parameter.
126 Returns:
127 Any: the return value of the target function.
129 Raises:
130 ValueError: If the sleep generator stops yielding values.
131 Exception: a custom exception specified by the exception_factory if provided.
132 If no exception_factory is provided:
133 google.api_core.RetryError: If the timeout is exceeded while retrying.
134 Exception: If the target raises an error that isn't retryable.
135 """
137 timeout = kwargs.get("deadline", timeout)
139 deadline = time.monotonic() + timeout if timeout is not None else None
140 error_list: list[Exception] = []
142 for sleep in sleep_generator:
143 try:
144 result = target()
145 if inspect.isawaitable(result):
146 warnings.warn(_ASYNC_RETRY_WARNING)
147 return result
149 # pylint: disable=broad-except
150 # This function explicitly must deal with broad exceptions.
151 except Exception as exc:
152 # defer to shared logic for handling errors
153 _retry_error_helper(
154 exc,
155 deadline,
156 sleep,
157 error_list,
158 predicate,
159 on_error,
160 exception_factory,
161 timeout,
162 )
163 # if exception not raised, sleep before next attempt
164 time.sleep(sleep)
166 raise ValueError("Sleep generator stopped yielding sleep values.")
169class Retry(_BaseRetry):
170 """Exponential retry decorator for unary synchronous RPCs.
172 This class is a decorator used to add retry or polling behavior to an RPC
173 call.
175 Although the default behavior is to retry transient API errors, a
176 different predicate can be provided to retry other exceptions.
178 There are two important concepts that retry/polling behavior may operate on,
179 Deadline and Timeout, which need to be properly defined for the correct
180 usage of this class and the rest of the library.
182 Deadline: a fixed point in time by which a certain operation must
183 terminate. For example, if a certain operation has a deadline
184 "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
185 error) by that time, regardless of when it was started or whether it
186 was started at all.
188 Timeout: the maximum duration of time after which a certain operation
189 must terminate (successfully or with an error). The countdown begins right
190 after an operation was started. For example, if an operation was started at
191 09:24:00 with timeout of 75 seconds, it must terminate no later than
192 09:25:15.
194 Unfortunately, in the past this class (and the api-core library as a whole) has not
195 been properly distinguishing the concepts of "timeout" and "deadline", and the
196 ``deadline`` parameter has meant ``timeout``. That is why
197 ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
198 ``deadline`` parameter is set, it will override the ``timeout`` parameter.
199 In other words, ``retry.deadline`` should be treated as just a deprecated alias for
200 ``retry.timeout``.
202 Said another way, it is safe to assume that this class and the rest of this
203 library operate in terms of timeouts (not deadlines) unless explicitly
204 noted the usage of deadline semantics.
206 It is also important to
207 understand the three most common applications of the Timeout concept in the
208 context of this library.
210 Usually the generic Timeout term may stand for one of the following actual
211 timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
213 RPC Timeout: a value supplied by the client to the server so
214 that the server side knows the maximum amount of time it is expected to
215 spend handling that specific RPC. For example, in the case of gRPC transport,
216 RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
217 request. The `timeout` property of this class normally never represents the
218 RPC Timeout as it is handled separately by the ``google.api_core.timeout``
219 module of this library.
221 Retry Timeout: this is the most common meaning of the ``timeout`` property
222 of this class, and defines how long a certain RPC may be retried in case
223 the server returns an error.
225 Polling Timeout: defines how long the
226 client side is allowed to call the polling RPC repeatedly to check a status of a
227 long-running operation. Each polling RPC is
228 expected to succeed (its errors are supposed to be handled by the retry
229 logic). The decision as to whether a new polling attempt needs to be made is based
230 not on the RPC status code but on the status of the returned
231 status of an operation. In other words: we will poll a long-running operation until
232 the operation is done or the polling timeout expires. Each poll will inform us of
233 the status of the operation. The poll consists of an RPC to the server that may
234 itself be retried as per the poll-specific retry settings in case of errors. The
235 operation-level retry settings do NOT apply to polling-RPC retries.
237 With the actual timeout types being defined above, the client libraries
238 often refer to just Timeout without clarifying which type specifically
239 that is. In that case the actual timeout type (sometimes also referred to as
240 Logical Timeout) can be determined from the context. If it is a unary rpc
241 call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
242 provided directly as a standalone value) or Retry Timeout (if provided as
243 ``retry.timeout`` property of the unary RPC's retry config). For
244 ``Operation`` or ``PollingFuture`` in general Timeout stands for
245 Polling Timeout.
247 Args:
248 predicate (Callable[Exception]): A callable that should return ``True``
249 if the given exception is retryable.
250 initial (float): The minimum amount of time to delay in seconds. This
251 must be greater than 0.
252 maximum (float): The maximum amount of time to delay in seconds.
253 multiplier (float): The multiplier applied to the delay.
254 timeout (Optional[float]): How long to keep retrying, in seconds.
255 Note: timeout is only checked before initiating a retry, so the target may
256 run past the timeout value as long as it is healthy.
257 on_error (Callable[Exception]): A function to call while processing
258 a retryable exception. Any error raised by this function will
259 *not* be caught.
260 deadline (float): DEPRECATED: use `timeout` instead. For backward
261 compatibility, if specified it will override the ``timeout`` parameter.
262 """
264 def __call__(
265 self,
266 func: Callable[_P, _R],
267 on_error: Callable[[Exception], Any] | None = None,
268 ) -> Callable[_P, _R]:
269 """Wrap a callable with retry behavior.
271 Args:
272 func (Callable): The callable to add retry behavior to.
273 on_error (Optional[Callable[Exception]]): If given, the
274 on_error callback will be called with each retryable exception
275 raised by the wrapped function. Any error raised by this
276 function will *not* be caught. If on_error was specified in the
277 constructor, this value will be ignored.
279 Returns:
280 Callable: A callable that will invoke ``func`` with retry
281 behavior.
282 """
283 if self._on_error is not None:
284 on_error = self._on_error
286 @functools.wraps(func)
287 def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:
288 """A wrapper that calls target function with retry."""
289 target = functools.partial(func, *args, **kwargs)
290 sleep_generator = exponential_sleep_generator(
291 self._initial, self._maximum, multiplier=self._multiplier
292 )
293 return retry_target(
294 target,
295 self._predicate,
296 sleep_generator,
297 timeout=self._timeout,
298 on_error=on_error,
299 )
301 return retry_wrapped_func