Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared utilities used by both downloads and uploads.
17This utilities are explicitly catered to ``requests``-like transports.
18"""
20import http.client
21import time
23import requests.exceptions
24import urllib3.exceptions # type: ignore
26from google.resumable_media import _helpers, common
28_DEFAULT_RETRY_STRATEGY = common.RetryStrategy()
29_SINGLE_GET_CHUNK_SIZE = 8192
30# The number of seconds to wait to establish a connection
31# (connect() call on socket). Avoid setting this to a multiple of 3 to not
32# Align with TCP Retransmission timing. (typically 2.5-3s)
33_DEFAULT_CONNECT_TIMEOUT = 61
34# The number of seconds to wait between bytes sent from the server.
35_DEFAULT_READ_TIMEOUT = 60
37_CONNECTION_ERROR_CLASSES = (
38 http.client.BadStatusLine,
39 http.client.IncompleteRead,
40 http.client.ResponseNotReady,
41 requests.exceptions.ConnectionError,
42 requests.exceptions.ChunkedEncodingError,
43 requests.exceptions.Timeout,
44 urllib3.exceptions.PoolError,
45 urllib3.exceptions.ProtocolError,
46 urllib3.exceptions.SSLError,
47 urllib3.exceptions.TimeoutError,
48 ConnectionError, # Python 3.x only, superclass of ConnectionResetError.
49)
52class RequestsMixin(object):
53 """Mix-in class implementing ``requests``-specific behavior.
55 These are methods that are more general purpose, with implementations
56 specific to the types defined in ``requests``.
57 """
59 @staticmethod
60 def _get_status_code(response):
61 """Access the status code from an HTTP response.
63 Args:
64 response (~requests.Response): The HTTP response object.
66 Returns:
67 int: The status code.
68 """
69 return response.status_code
71 @staticmethod
72 def _get_headers(response):
73 """Access the headers from an HTTP response.
75 Args:
76 response (~requests.Response): The HTTP response object.
78 Returns:
79 ~requests.structures.CaseInsensitiveDict: The header mapping (keys
80 are case-insensitive).
81 """
82 return response.headers
84 @staticmethod
85 def _get_body(response):
86 """Access the response body from an HTTP response.
88 Args:
89 response (~requests.Response): The HTTP response object.
91 Returns:
92 bytes: The body of the ``response``.
93 """
94 return response.content
97class RawRequestsMixin(RequestsMixin):
98 @staticmethod
99 def _get_body(response):
100 """Access the response body from an HTTP response.
102 Args:
103 response (~requests.Response): The HTTP response object.
105 Returns:
106 bytes: The body of the ``response``.
107 """
108 if response._content is False:
109 response._content = b"".join(
110 response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False)
111 )
112 response._content_consumed = True
113 return response._content
116def wait_and_retry(func, get_status_code, retry_strategy):
117 """Attempts to retry a call to ``func`` until success.
119 Expects ``func`` to return an HTTP response and uses ``get_status_code``
120 to check if the response is retry-able.
122 ``func`` is expected to raise a failure status code as a
123 common.InvalidResponse, at which point this method will check the code
124 against the common.RETRIABLE list of retriable status codes.
126 Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
127 ``retry_strategy``) returns :data:`False`. Uses
128 :func:`_helpers.calculate_retry_wait` to double the wait time (with jitter)
129 after each attempt.
131 Args:
132 func (Callable): A callable that takes no arguments and produces
133 an HTTP response which will be checked as retry-able.
134 get_status_code (Callable[Any, int]): Helper to get a status code
135 from a response.
136 retry_strategy (~google.resumable_media.common.RetryStrategy): The
137 strategy to use if the request fails and must be retried.
139 Returns:
140 object: The return value of ``func``.
141 """
142 total_sleep = 0.0
143 num_retries = 0
144 # base_wait will be multiplied by the multiplier on the first retry.
145 base_wait = float(retry_strategy.initial_delay) / retry_strategy.multiplier
147 # Set the retriable_exception_type if possible. We expect requests to be
148 # present here and the transport to be using requests.exceptions errors,
149 # but due to loose coupling with the transport layer we can't guarantee it.
151 while True: # return on success or when retries exhausted.
152 error = None
153 try:
154 response = func()
155 except _CONNECTION_ERROR_CLASSES as e:
156 error = e # Fall through to retry, if there are retries left.
157 except common.InvalidResponse as e:
158 # An InvalidResponse is only retriable if its status code matches.
159 # The `process_response()` method on a Download or Upload method
160 # will convert the status code into an exception.
161 if get_status_code(e.response) in common.RETRYABLE:
162 error = e # Fall through to retry, if there are retries left.
163 else:
164 raise # If the status code is not retriable, raise w/o retry.
165 else:
166 return response
168 base_wait, wait_time = _helpers.calculate_retry_wait(
169 base_wait, retry_strategy.max_sleep, retry_strategy.multiplier
170 )
171 num_retries += 1
172 total_sleep += wait_time
174 # Check if (another) retry is allowed. If retries are exhausted and
175 # no acceptable response was received, raise the retriable error.
176 if not retry_strategy.retry_allowed(total_sleep, num_retries):
177 raise error
179 time.sleep(wait_time)