Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared classes and functions for retrying requests.
17:class:`_BaseRetry` is the base class for :class:`Retry`,
18:class:`AsyncRetry`, :class:`StreamingRetry`, and :class:`AsyncStreamingRetry`.
19"""
21from __future__ import annotations
23import logging
24import random
25import time
26from enum import Enum
27from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional
29import requests.exceptions
30from google.auth import exceptions as auth_exceptions
32from google.api_core import exceptions
34if TYPE_CHECKING:
35 import sys
37 if sys.version_info >= (3, 11):
38 from typing import Self
39 else:
40 from typing_extensions import Self
42_DEFAULT_INITIAL_DELAY = 1.0 # seconds
43_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
44_DEFAULT_DELAY_MULTIPLIER = 2.0
45_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
47_LOGGER = logging.getLogger("google.api_core.retry")
50def if_exception_type(
51 *exception_types: type[Exception],
52) -> Callable[[Exception], bool]:
53 """Creates a predicate to check if the exception is of a given type.
55 Args:
56 exception_types (Sequence[:func:`type`]): The exception types to check
57 for.
59 Returns:
60 Callable[Exception]: A predicate that returns True if the provided
61 exception is of the given type(s).
62 """
64 def if_exception_type_predicate(exception: Exception) -> bool:
65 """Bound predicate for checking an exception type."""
66 return isinstance(exception, exception_types)
68 return if_exception_type_predicate
71# pylint: disable=invalid-name
72# Pylint sees this as a constant, but it is also an alias that should be
73# considered a function.
74if_transient_error = if_exception_type(
75 exceptions.InternalServerError,
76 exceptions.TooManyRequests,
77 exceptions.ServiceUnavailable,
78 requests.exceptions.ConnectionError,
79 requests.exceptions.ChunkedEncodingError,
80 auth_exceptions.TransportError,
81)
82"""A predicate that checks if an exception is a transient API error.
84The following server errors are considered transient:
86- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
87 ``INTERNAL(13)`` and its subclasses.
88- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
89- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
90- :class:`requests.exceptions.ConnectionError`
91- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
92 chunked encoding but sent an invalid chunk.
93- :class:`google.auth.exceptions.TransportError` - Used to indicate an
94 error occurred during an HTTP request.
95"""
96# pylint: enable=invalid-name
99def exponential_sleep_generator(
100 initial: float, maximum: float, multiplier: float = _DEFAULT_DELAY_MULTIPLIER
101):
102 """Generates sleep intervals based on the exponential back-off algorithm.
104 This implements the `Truncated Exponential Back-off`_ algorithm.
106 .. _Truncated Exponential Back-off:
107 https://cloud.google.com/storage/docs/exponential-backoff
109 Args:
110 initial (float): The minimum amount of time to delay. This must
111 be greater than 0.
112 maximum (float): The maximum amount of time to delay.
113 multiplier (float): The multiplier applied to the delay.
115 Yields:
116 float: successive sleep intervals.
117 """
118 max_delay = min(initial, maximum)
119 while True:
120 yield random.uniform(0.0, max_delay)
121 max_delay = min(max_delay * multiplier, maximum)
124class RetryFailureReason(Enum):
125 """
126 The cause of a failed retry, used when building exceptions
127 """
129 TIMEOUT = 0
130 NON_RETRYABLE_ERROR = 1
133def build_retry_error(
134 exc_list: list[Exception],
135 reason: RetryFailureReason,
136 timeout_val: float | None,
137 **kwargs: Any,
138) -> tuple[Exception, Exception | None]:
139 """
140 Default exception_factory implementation.
142 Returns a RetryError if the failure is due to a timeout, otherwise
143 returns the last exception encountered.
145 Args:
146 - exc_list: list of exceptions that occurred during the retry
147 - reason: reason for the retry failure.
148 Can be TIMEOUT or NON_RETRYABLE_ERROR
149 - timeout_val: the original timeout value for the retry (in seconds), for use in the exception message
151 Returns:
152 - tuple: a tuple of the exception to be raised, and the cause exception if any
153 """
154 if reason == RetryFailureReason.TIMEOUT:
155 # return RetryError with the most recent exception as the cause
156 src_exc = exc_list[-1] if exc_list else None
157 timeout_val_str = f"of {timeout_val:0.1f}s " if timeout_val is not None else ""
158 return (
159 exceptions.RetryError(
160 f"Timeout {timeout_val_str}exceeded",
161 src_exc,
162 ),
163 src_exc,
164 )
165 elif exc_list:
166 # return most recent exception encountered and its cause
167 final_exc = exc_list[-1]
168 cause = getattr(final_exc, "__cause__", None)
169 return final_exc, cause
170 else:
171 # no exceptions were given in exc_list. Raise generic RetryError
172 return exceptions.RetryError("Unknown error", None), None
175def _retry_error_helper(
176 exc: Exception,
177 deadline: float | None,
178 sleep_iterator: Iterator[float],
179 error_list: list[Exception],
180 predicate_fn: Callable[[Exception], bool],
181 on_error_fn: Callable[[Exception], None] | None,
182 exc_factory_fn: Callable[
183 [list[Exception], RetryFailureReason, float | None],
184 tuple[Exception, Exception | None],
185 ],
186 original_timeout: float | None,
187) -> float:
188 """
189 Shared logic for handling an error for all retry implementations
191 - Raises an error on timeout or non-retryable error
192 - Calls on_error_fn if provided
193 - Logs the error
195 Args:
196 - exc: the exception that was raised
197 - deadline: the deadline for the retry, calculated as a diff from time.monotonic()
198 - sleep_iterator: iterator to draw the next backoff value from
199 - error_list: the list of exceptions that have been raised so far
200 - predicate_fn: takes `exc` and returns true if the operation should be retried
201 - on_error_fn: callback to execute when a retryable error occurs
202 - exc_factory_fn: callback used to build the exception to be raised on terminal failure
203 - original_timeout_val: the original timeout value for the retry (in seconds),
204 to be passed to the exception factory for building an error message
205 Returns:
206 - the sleep value chosen before the next attempt
207 """
208 error_list.append(exc)
209 if not predicate_fn(exc):
210 final_exc, source_exc = exc_factory_fn(
211 error_list,
212 RetryFailureReason.NON_RETRYABLE_ERROR,
213 original_timeout,
214 )
215 raise final_exc from source_exc
216 if on_error_fn is not None:
217 on_error_fn(exc)
218 # next_sleep is fetched after the on_error callback, to allow clients
219 # to update sleep_iterator values dynamically in response to errors
220 try:
221 next_sleep = next(sleep_iterator)
222 except StopIteration:
223 raise ValueError("Sleep generator stopped yielding sleep values.") from exc
224 if deadline is not None and time.monotonic() + next_sleep > deadline:
225 final_exc, source_exc = exc_factory_fn(
226 error_list,
227 RetryFailureReason.TIMEOUT,
228 original_timeout,
229 )
230 raise final_exc from source_exc
231 _LOGGER.debug(
232 "Retrying due to {}, sleeping {:.1f}s ...".format(error_list[-1], next_sleep)
233 )
234 return next_sleep
237class _BaseRetry(object):
238 """
239 Base class for retry configuration objects. This class is intended to capture retry
240 and backoff configuration that is common to both synchronous and asynchronous retries,
241 for both unary and streaming RPCs. It is not intended to be instantiated directly,
242 but rather to be subclassed by the various retry configuration classes.
243 """
245 def __init__(
246 self,
247 predicate: Callable[[Exception], bool] = if_transient_error,
248 initial: float = _DEFAULT_INITIAL_DELAY,
249 maximum: float = _DEFAULT_MAXIMUM_DELAY,
250 multiplier: float = _DEFAULT_DELAY_MULTIPLIER,
251 timeout: Optional[float] = _DEFAULT_DEADLINE,
252 on_error: Optional[Callable[[Exception], Any]] = None,
253 **kwargs: Any,
254 ) -> None:
255 self._predicate = predicate
256 self._initial = initial
257 self._multiplier = multiplier
258 self._maximum = maximum
259 self._timeout = kwargs.get("deadline", timeout)
260 self._deadline = self._timeout
261 self._on_error = on_error
263 def __call__(self, *args, **kwargs) -> Any:
264 raise NotImplementedError("Not implemented in base class")
266 @property
267 def deadline(self) -> float | None:
268 """
269 DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
270 documentation for details.
271 """
272 return self._timeout
274 @property
275 def timeout(self) -> float | None:
276 return self._timeout
278 def with_deadline(self, deadline: float | None) -> Self:
279 """Return a copy of this retry with the given timeout.
281 DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
282 documentation for details.
284 Args:
285 deadline (float|None): How long to keep retrying, in seconds. If None,
286 no timeout is enforced.
288 Returns:
289 Retry: A new retry instance with the given timeout.
290 """
291 return self.with_timeout(deadline)
293 def with_timeout(self, timeout: float | None) -> Self:
294 """Return a copy of this retry with the given timeout.
296 Args:
297 timeout (float): How long to keep retrying, in seconds. If None,
298 no timeout will be enforced.
300 Returns:
301 Retry: A new retry instance with the given timeout.
302 """
303 return type(self)(
304 predicate=self._predicate,
305 initial=self._initial,
306 maximum=self._maximum,
307 multiplier=self._multiplier,
308 timeout=timeout,
309 on_error=self._on_error,
310 )
312 def with_predicate(self, predicate: Callable[[Exception], bool]) -> Self:
313 """Return a copy of this retry with the given predicate.
315 Args:
316 predicate (Callable[Exception]): A callable that should return
317 ``True`` if the given exception is retryable.
319 Returns:
320 Retry: A new retry instance with the given predicate.
321 """
322 return type(self)(
323 predicate=predicate,
324 initial=self._initial,
325 maximum=self._maximum,
326 multiplier=self._multiplier,
327 timeout=self._timeout,
328 on_error=self._on_error,
329 )
331 def with_delay(
332 self,
333 initial: Optional[float] = None,
334 maximum: Optional[float] = None,
335 multiplier: Optional[float] = None,
336 ) -> Self:
337 """Return a copy of this retry with the given delay options.
339 Args:
340 initial (float): The minimum amount of time to delay (in seconds). This must
341 be greater than 0. If None, the current value is used.
342 maximum (float): The maximum amount of time to delay (in seconds). If None, the
343 current value is used.
344 multiplier (float): The multiplier applied to the delay. If None, the current
345 value is used.
347 Returns:
348 Retry: A new retry instance with the given delay options.
349 """
350 return type(self)(
351 predicate=self._predicate,
352 initial=initial if initial is not None else self._initial,
353 maximum=maximum if maximum is not None else self._maximum,
354 multiplier=multiplier if multiplier is not None else self._multiplier,
355 timeout=self._timeout,
356 on_error=self._on_error,
357 )
359 def __str__(self) -> str:
360 return (
361 "<{} predicate={}, initial={:.1f}, maximum={:.1f}, "
362 "multiplier={:.1f}, timeout={}, on_error={}>".format(
363 type(self).__name__,
364 self._predicate,
365 self._initial,
366 self._maximum,
367 self._multiplier,
368 self._timeout, # timeout can be None, thus no {:.1f}
369 self._on_error,
370 )
371 )