Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared classes and functions for retrying requests.
17:class:`_BaseRetry` is the base class for :class:`Retry`,
18:class:`AsyncRetry`, :class:`StreamingRetry`, and :class:`AsyncStreamingRetry`.
19"""
21from __future__ import annotations
23import logging
24import random
25import time
27from enum import Enum
28from typing import Any, Callable, Optional, Iterator, TYPE_CHECKING
30import requests.exceptions
32from google.api_core import exceptions
33from google.auth import exceptions as auth_exceptions
35if TYPE_CHECKING:
36 import sys
38 if sys.version_info >= (3, 11):
39 from typing import Self
40 else:
41 from typing_extensions import Self
43_DEFAULT_INITIAL_DELAY = 1.0 # seconds
44_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
45_DEFAULT_DELAY_MULTIPLIER = 2.0
46_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
48_LOGGER = logging.getLogger("google.api_core.retry")
51def if_exception_type(
52 *exception_types: type[Exception],
53) -> Callable[[Exception], bool]:
54 """Creates a predicate to check if the exception is of a given type.
56 Args:
57 exception_types (Sequence[:func:`type`]): The exception types to check
58 for.
60 Returns:
61 Callable[Exception]: A predicate that returns True if the provided
62 exception is of the given type(s).
63 """
65 def if_exception_type_predicate(exception: Exception) -> bool:
66 """Bound predicate for checking an exception type."""
67 return isinstance(exception, exception_types)
69 return if_exception_type_predicate
72# pylint: disable=invalid-name
73# Pylint sees this as a constant, but it is also an alias that should be
74# considered a function.
75if_transient_error = if_exception_type(
76 exceptions.InternalServerError,
77 exceptions.TooManyRequests,
78 exceptions.ServiceUnavailable,
79 requests.exceptions.ConnectionError,
80 requests.exceptions.ChunkedEncodingError,
81 auth_exceptions.TransportError,
82)
83"""A predicate that checks if an exception is a transient API error.
85The following server errors are considered transient:
87- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
88 ``INTERNAL(13)`` and its subclasses.
89- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
90- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
91- :class:`requests.exceptions.ConnectionError`
92- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
93 chunked encoding but sent an invalid chunk.
94- :class:`google.auth.exceptions.TransportError` - Used to indicate an
95 error occurred during an HTTP request.
96"""
97# pylint: enable=invalid-name
100def exponential_sleep_generator(
101 initial: float, maximum: float, multiplier: float = _DEFAULT_DELAY_MULTIPLIER
102):
103 """Generates sleep intervals based on the exponential back-off algorithm.
105 This implements the `Truncated Exponential Back-off`_ algorithm.
107 .. _Truncated Exponential Back-off:
108 https://cloud.google.com/storage/docs/exponential-backoff
110 Args:
111 initial (float): The minimum amount of time to delay. This must
112 be greater than 0.
113 maximum (float): The maximum amount of time to delay.
114 multiplier (float): The multiplier applied to the delay.
116 Yields:
117 float: successive sleep intervals.
118 """
119 max_delay = min(initial, maximum)
120 while True:
121 yield random.uniform(0.0, max_delay)
122 max_delay = min(max_delay * multiplier, maximum)
125class RetryFailureReason(Enum):
126 """
127 The cause of a failed retry, used when building exceptions
128 """
130 TIMEOUT = 0
131 NON_RETRYABLE_ERROR = 1
134def build_retry_error(
135 exc_list: list[Exception],
136 reason: RetryFailureReason,
137 timeout_val: float | None,
138 **kwargs: Any,
139) -> tuple[Exception, Exception | None]:
140 """
141 Default exception_factory implementation.
143 Returns a RetryError if the failure is due to a timeout, otherwise
144 returns the last exception encountered.
146 Args:
147 - exc_list: list of exceptions that occurred during the retry
148 - reason: reason for the retry failure.
149 Can be TIMEOUT or NON_RETRYABLE_ERROR
150 - timeout_val: the original timeout value for the retry (in seconds), for use in the exception message
152 Returns:
153 - tuple: a tuple of the exception to be raised, and the cause exception if any
154 """
155 if reason == RetryFailureReason.TIMEOUT:
156 # return RetryError with the most recent exception as the cause
157 src_exc = exc_list[-1] if exc_list else None
158 timeout_val_str = f"of {timeout_val:0.1f}s " if timeout_val is not None else ""
159 return (
160 exceptions.RetryError(
161 f"Timeout {timeout_val_str}exceeded",
162 src_exc,
163 ),
164 src_exc,
165 )
166 elif exc_list:
167 # return most recent exception encountered and its cause
168 final_exc = exc_list[-1]
169 cause = getattr(final_exc, "__cause__", None)
170 return final_exc, cause
171 else:
172 # no exceptions were given in exc_list. Raise generic RetryError
173 return exceptions.RetryError("Unknown error", None), None
176def _retry_error_helper(
177 exc: Exception,
178 deadline: float | None,
179 sleep_iterator: Iterator[float],
180 error_list: list[Exception],
181 predicate_fn: Callable[[Exception], bool],
182 on_error_fn: Callable[[Exception], None] | None,
183 exc_factory_fn: Callable[
184 [list[Exception], RetryFailureReason, float | None],
185 tuple[Exception, Exception | None],
186 ],
187 original_timeout: float | None,
188) -> float:
189 """
190 Shared logic for handling an error for all retry implementations
192 - Raises an error on timeout or non-retryable error
193 - Calls on_error_fn if provided
194 - Logs the error
196 Args:
197 - exc: the exception that was raised
198 - deadline: the deadline for the retry, calculated as a diff from time.monotonic()
199 - sleep_iterator: iterator to draw the next backoff value from
200 - error_list: the list of exceptions that have been raised so far
201 - predicate_fn: takes `exc` and returns true if the operation should be retried
202 - on_error_fn: callback to execute when a retryable error occurs
203 - exc_factory_fn: callback used to build the exception to be raised on terminal failure
204 - original_timeout_val: the original timeout value for the retry (in seconds),
205 to be passed to the exception factory for building an error message
206 Returns:
207 - the sleep value chosen before the next attempt
208 """
209 error_list.append(exc)
210 if not predicate_fn(exc):
211 final_exc, source_exc = exc_factory_fn(
212 error_list,
213 RetryFailureReason.NON_RETRYABLE_ERROR,
214 original_timeout,
215 )
216 raise final_exc from source_exc
217 if on_error_fn is not None:
218 on_error_fn(exc)
219 # next_sleep is fetched after the on_error callback, to allow clients
220 # to update sleep_iterator values dynamically in response to errors
221 try:
222 next_sleep = next(sleep_iterator)
223 except StopIteration:
224 raise ValueError("Sleep generator stopped yielding sleep values.") from exc
225 if deadline is not None and time.monotonic() + next_sleep > deadline:
226 final_exc, source_exc = exc_factory_fn(
227 error_list,
228 RetryFailureReason.TIMEOUT,
229 original_timeout,
230 )
231 raise final_exc from source_exc
232 _LOGGER.debug(
233 "Retrying due to {}, sleeping {:.1f}s ...".format(error_list[-1], next_sleep)
234 )
235 return next_sleep
238class _BaseRetry(object):
239 """
240 Base class for retry configuration objects. This class is intended to capture retry
241 and backoff configuration that is common to both synchronous and asynchronous retries,
242 for both unary and streaming RPCs. It is not intended to be instantiated directly,
243 but rather to be subclassed by the various retry configuration classes.
244 """
246 def __init__(
247 self,
248 predicate: Callable[[Exception], bool] = if_transient_error,
249 initial: float = _DEFAULT_INITIAL_DELAY,
250 maximum: float = _DEFAULT_MAXIMUM_DELAY,
251 multiplier: float = _DEFAULT_DELAY_MULTIPLIER,
252 timeout: Optional[float] = _DEFAULT_DEADLINE,
253 on_error: Optional[Callable[[Exception], Any]] = None,
254 **kwargs: Any,
255 ) -> None:
256 self._predicate = predicate
257 self._initial = initial
258 self._multiplier = multiplier
259 self._maximum = maximum
260 self._timeout = kwargs.get("deadline", timeout)
261 self._deadline = self._timeout
262 self._on_error = on_error
264 def __call__(self, *args, **kwargs) -> Any:
265 raise NotImplementedError("Not implemented in base class")
267 @property
268 def deadline(self) -> float | None:
269 """
270 DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
271 documentation for details.
272 """
273 return self._timeout
275 @property
276 def timeout(self) -> float | None:
277 return self._timeout
279 def with_deadline(self, deadline: float | None) -> Self:
280 """Return a copy of this retry with the given timeout.
282 DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
283 documentation for details.
285 Args:
286 deadline (float|None): How long to keep retrying, in seconds. If None,
287 no timeout is enforced.
289 Returns:
290 Retry: A new retry instance with the given timeout.
291 """
292 return self.with_timeout(deadline)
294 def with_timeout(self, timeout: float | None) -> Self:
295 """Return a copy of this retry with the given timeout.
297 Args:
298 timeout (float): How long to keep retrying, in seconds. If None,
299 no timeout will be enforced.
301 Returns:
302 Retry: A new retry instance with the given timeout.
303 """
304 return type(self)(
305 predicate=self._predicate,
306 initial=self._initial,
307 maximum=self._maximum,
308 multiplier=self._multiplier,
309 timeout=timeout,
310 on_error=self._on_error,
311 )
313 def with_predicate(self, predicate: Callable[[Exception], bool]) -> Self:
314 """Return a copy of this retry with the given predicate.
316 Args:
317 predicate (Callable[Exception]): A callable that should return
318 ``True`` if the given exception is retryable.
320 Returns:
321 Retry: A new retry instance with the given predicate.
322 """
323 return type(self)(
324 predicate=predicate,
325 initial=self._initial,
326 maximum=self._maximum,
327 multiplier=self._multiplier,
328 timeout=self._timeout,
329 on_error=self._on_error,
330 )
332 def with_delay(
333 self,
334 initial: Optional[float] = None,
335 maximum: Optional[float] = None,
336 multiplier: Optional[float] = None,
337 ) -> Self:
338 """Return a copy of this retry with the given delay options.
340 Args:
341 initial (float): The minimum amount of time to delay (in seconds). This must
342 be greater than 0. If None, the current value is used.
343 maximum (float): The maximum amount of time to delay (in seconds). If None, the
344 current value is used.
345 multiplier (float): The multiplier applied to the delay. If None, the current
346 value is used.
348 Returns:
349 Retry: A new retry instance with the given delay options.
350 """
351 return type(self)(
352 predicate=self._predicate,
353 initial=initial if initial is not None else self._initial,
354 maximum=maximum if maximum is not None else self._maximum,
355 multiplier=multiplier if multiplier is not None else self._multiplier,
356 timeout=self._timeout,
357 on_error=self._on_error,
358 )
360 def __str__(self) -> str:
361 return (
362 "<{} predicate={}, initial={:.1f}, maximum={:.1f}, "
363 "multiplier={:.1f}, timeout={}, on_error={}>".format(
364 type(self).__name__,
365 self._predicate,
366 self._initial,
367 self._maximum,
368 self._multiplier,
369 self._timeout, # timeout can be None, thus no {:.1f}
370 self._on_error,
371 )
372 )