Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/resumable_media/requests/_request_helpers.py: 46%
48 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-26 06:07 +0000
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared utilities used by both downloads and uploads.
17This utilities are explicitly catered to ``requests``-like transports.
18"""
20import requests.exceptions
21import urllib3.exceptions # type: ignore
23import time
25from google.resumable_media import common
26from google.resumable_media import _helpers
28_DEFAULT_RETRY_STRATEGY = common.RetryStrategy()
29_SINGLE_GET_CHUNK_SIZE = 8192
30# The number of seconds to wait to establish a connection
31# (connect() call on socket). Avoid setting this to a multiple of 3 to not
32# Align with TCP Retransmission timing. (typically 2.5-3s)
33_DEFAULT_CONNECT_TIMEOUT = 61
34# The number of seconds to wait between bytes sent from the server.
35_DEFAULT_READ_TIMEOUT = 60
37_CONNECTION_ERROR_CLASSES = (
38 requests.exceptions.ConnectionError,
39 requests.exceptions.ChunkedEncodingError,
40 requests.exceptions.Timeout,
41 urllib3.exceptions.ProtocolError,
42 ConnectionError, # Python 3.x only, superclass of ConnectionResetError.
43)
46class RequestsMixin(object):
47 """Mix-in class implementing ``requests``-specific behavior.
49 These are methods that are more general purpose, with implementations
50 specific to the types defined in ``requests``.
51 """
53 @staticmethod
54 def _get_status_code(response):
55 """Access the status code from an HTTP response.
57 Args:
58 response (~requests.Response): The HTTP response object.
60 Returns:
61 int: The status code.
62 """
63 return response.status_code
65 @staticmethod
66 def _get_headers(response):
67 """Access the headers from an HTTP response.
69 Args:
70 response (~requests.Response): The HTTP response object.
72 Returns:
73 ~requests.structures.CaseInsensitiveDict: The header mapping (keys
74 are case-insensitive).
75 """
76 return response.headers
78 @staticmethod
79 def _get_body(response):
80 """Access the response body from an HTTP response.
82 Args:
83 response (~requests.Response): The HTTP response object.
85 Returns:
86 bytes: The body of the ``response``.
87 """
88 return response.content
91class RawRequestsMixin(RequestsMixin):
92 @staticmethod
93 def _get_body(response):
94 """Access the response body from an HTTP response.
96 Args:
97 response (~requests.Response): The HTTP response object.
99 Returns:
100 bytes: The body of the ``response``.
101 """
102 if response._content is False:
103 response._content = b"".join(
104 response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False)
105 )
106 response._content_consumed = True
107 return response._content
110def wait_and_retry(func, get_status_code, retry_strategy):
111 """Attempts to retry a call to ``func`` until success.
113 Expects ``func`` to return an HTTP response and uses ``get_status_code``
114 to check if the response is retry-able.
116 ``func`` is expected to raise a failure status code as a
117 common.InvalidResponse, at which point this method will check the code
118 against the common.RETRIABLE list of retriable status codes.
120 Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
121 ``retry_strategy``) returns :data:`False`. Uses
122 :func:`_helpers.calculate_retry_wait` to double the wait time (with jitter)
123 after each attempt.
125 Args:
126 func (Callable): A callable that takes no arguments and produces
127 an HTTP response which will be checked as retry-able.
128 get_status_code (Callable[Any, int]): Helper to get a status code
129 from a response.
130 retry_strategy (~google.resumable_media.common.RetryStrategy): The
131 strategy to use if the request fails and must be retried.
133 Returns:
134 object: The return value of ``func``.
135 """
136 total_sleep = 0.0
137 num_retries = 0
138 # base_wait will be multiplied by the multiplier on the first retry.
139 base_wait = float(retry_strategy.initial_delay) / retry_strategy.multiplier
141 # Set the retriable_exception_type if possible. We expect requests to be
142 # present here and the transport to be using requests.exceptions errors,
143 # but due to loose coupling with the transport layer we can't guarantee it.
145 while True: # return on success or when retries exhausted.
146 error = None
147 try:
148 response = func()
149 except _CONNECTION_ERROR_CLASSES as e:
150 error = e # Fall through to retry, if there are retries left.
151 except common.InvalidResponse as e:
152 # An InvalidResponse is only retriable if its status code matches.
153 # The `process_response()` method on a Download or Upload method
154 # will convert the status code into an exception.
155 if get_status_code(e.response) in common.RETRYABLE:
156 error = e # Fall through to retry, if there are retries left.
157 else:
158 raise # If the status code is not retriable, raise w/o retry.
159 else:
160 return response
162 base_wait, wait_time = _helpers.calculate_retry_wait(
163 base_wait, retry_strategy.max_sleep, retry_strategy.multiplier
164 )
165 num_retries += 1
166 total_sleep += wait_time
168 # Check if (another) retry is allowed. If retries are exhausted and
169 # no acceptable response was received, raise the retriable error.
170 if not retry_strategy.retry_allowed(total_sleep, num_retries):
171 raise error
173 time.sleep(wait_time)