Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Common utilities for Google Media Downloads and Resumable Uploads.
17Includes custom exception types, useful constants and shared helpers.
18"""
20import http.client
22_SLEEP_RETRY_ERROR_MSG = (
23 "At most one of `max_cumulative_retry` and `max_retries` " "can be specified."
24)
26UPLOAD_CHUNK_SIZE = 262144 # 256 * 1024
27"""int: Chunks in a resumable upload must come in multiples of 256 KB."""
29PERMANENT_REDIRECT = http.client.PERMANENT_REDIRECT # type: ignore
30"""int: Permanent redirect status code.
32.. note::
33 This is a backward-compatibility alias.
35It is used by Google services to indicate some (but not all) of
36a resumable upload has been completed.
38For more information, see `RFC 7238`_.
40.. _RFC 7238: https://tools.ietf.org/html/rfc7238
41"""
43TOO_MANY_REQUESTS = http.client.TOO_MANY_REQUESTS
44"""int: Status code indicating rate-limiting.
46.. note::
47 This is a backward-compatibility alias.
49For more information, see `RFC 6585`_.
51.. _RFC 6585: https://tools.ietf.org/html/rfc6585#section-4
52"""
54MAX_SLEEP = 64.0
55"""float: Maximum amount of time allowed between requests.
57Used during the retry process for sleep after a failed request.
58Chosen since it is the power of two nearest to one minute.
59"""
61MAX_CUMULATIVE_RETRY = 600.0
62"""float: Maximum total sleep time allowed during retry process.
64This is provided (10 minutes) as a default. When the cumulative sleep
65exceeds this limit, no more retries will occur.
66"""
68RETRYABLE = (
69 http.client.TOO_MANY_REQUESTS, # 429
70 http.client.REQUEST_TIMEOUT, # 408
71 http.client.INTERNAL_SERVER_ERROR, # 500
72 http.client.BAD_GATEWAY, # 502
73 http.client.SERVICE_UNAVAILABLE, # 503
74 http.client.GATEWAY_TIMEOUT, # 504
75)
76"""iterable: HTTP status codes that indicate a retryable error.
78Connection errors are also retried, but are not listed as they are
79exceptions, not status codes.
80"""
83class InvalidResponse(Exception):
84 """Error class for responses which are not in the correct state.
86 Args:
87 response (object): The HTTP response which caused the failure.
88 args (tuple): The positional arguments typically passed to an
89 exception class.
90 """
92 def __init__(self, response, *args):
93 super(InvalidResponse, self).__init__(*args)
94 self.response = response
95 """object: The HTTP response object that caused the failure."""
98class DataCorruption(Exception):
99 """Error class for corrupt media transfers.
101 Args:
102 response (object): The HTTP response which caused the failure.
103 args (tuple): The positional arguments typically passed to an
104 exception class.
105 """
107 def __init__(self, response, *args):
108 super(DataCorruption, self).__init__(*args)
109 self.response = response
110 """object: The HTTP response object that caused the failure."""
113class RetryStrategy(object):
114 """Configuration class for retrying failed requests.
116 At most one of ``max_cumulative_retry`` and ``max_retries`` can be
117 specified (they are both caps on the total number of retries). If
118 neither are specified, then ``max_cumulative_retry`` is set as
119 :data:`MAX_CUMULATIVE_RETRY`.
121 Args:
122 max_sleep (Optional[float]): The maximum amount of time to sleep after
123 a failed request. Default is :attr:`MAX_SLEEP`.
124 max_cumulative_retry (Optional[float]): The maximum **total** amount of
125 time to sleep during retry process.
126 max_retries (Optional[int]): The number of retries to attempt.
127 initial_delay (Optional[float]): The initial delay. Default 1.0 second.
128 muiltiplier (Optional[float]): Exponent of the backoff. Default is 2.0.
130 Attributes:
131 max_sleep (float): Maximum amount of time allowed between requests.
132 max_cumulative_retry (Optional[float]): Maximum total sleep time
133 allowed during retry process.
134 max_retries (Optional[int]): The number retries to attempt.
135 initial_delay (Optional[float]): The initial delay. Default 1.0 second.
136 muiltiplier (Optional[float]): Exponent of the backoff. Default is 2.0.
138 Raises:
139 ValueError: If both of ``max_cumulative_retry`` and ``max_retries``
140 are passed.
141 """
143 def __init__(
144 self,
145 max_sleep=MAX_SLEEP,
146 max_cumulative_retry=None,
147 max_retries=None,
148 initial_delay=1.0,
149 multiplier=2.0,
150 ):
151 if max_cumulative_retry is not None and max_retries is not None:
152 raise ValueError(_SLEEP_RETRY_ERROR_MSG)
153 if max_cumulative_retry is None and max_retries is None:
154 max_cumulative_retry = MAX_CUMULATIVE_RETRY
156 self.max_sleep = max_sleep
157 self.max_cumulative_retry = max_cumulative_retry
158 self.max_retries = max_retries
159 self.initial_delay = initial_delay
160 self.multiplier = multiplier
162 def retry_allowed(self, total_sleep, num_retries):
163 """Check if another retry is allowed.
165 Args:
166 total_sleep (float): With another retry, the amount of sleep that
167 will be accumulated by the caller.
168 num_retries (int): With another retry, the number of retries that
169 will be attempted by the caller.
171 Returns:
172 bool: Indicating if another retry is allowed (depending on either
173 the cumulative sleep allowed or the maximum number of retries
174 allowed.
175 """
176 if self.max_cumulative_retry is None:
177 return num_retries <= self.max_retries
178 else:
179 return total_sleep <= self.max_cumulative_retry