Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/resumable_media/requests/_request_helpers.py: 6%

1# Copyright 2017 Google Inc. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15"""Shared utilities used by both downloads and uploads. 

16 

17This utilities are explicitly catered to ``requests``-like transports. 

18""" 

19 

20import http.client 

21import requests.exceptions 

22import urllib3.exceptions # type: ignore 

23 

24import time 

25 

26from google.resumable_media import common 

27from google.resumable_media import _helpers 

28 

29_DEFAULT_RETRY_STRATEGY = common.RetryStrategy() 

30_SINGLE_GET_CHUNK_SIZE = 8192 

31# The number of seconds to wait to establish a connection 

32# (connect() call on socket). Avoid setting this to a multiple of 3 to not 

33# Align with TCP Retransmission timing. (typically 2.5-3s) 

34_DEFAULT_CONNECT_TIMEOUT = 61 

35# The number of seconds to wait between bytes sent from the server. 

36_DEFAULT_READ_TIMEOUT = 60 

37 

38_CONNECTION_ERROR_CLASSES = ( 

39 http.client.BadStatusLine, 

40 http.client.IncompleteRead, 

41 http.client.ResponseNotReady, 

42 requests.exceptions.ConnectionError, 

43 requests.exceptions.ChunkedEncodingError, 

44 requests.exceptions.Timeout, 

45 urllib3.exceptions.PoolError, 

46 urllib3.exceptions.ProtocolError, 

47 urllib3.exceptions.SSLError, 

48 urllib3.exceptions.TimeoutError, 

49 ConnectionError, # Python 3.x only, superclass of ConnectionResetError. 

50) 

51 

52 

53class RequestsMixin(object): 

54 """Mix-in class implementing ``requests``-specific behavior. 

55 

56 These are methods that are more general purpose, with implementations 

57 specific to the types defined in ``requests``. 

58 """ 

59 

60 @staticmethod 

61 def _get_status_code(response): 

62 """Access the status code from an HTTP response. 

63 

64 Args: 

65 response (~requests.Response): The HTTP response object. 

66 

67 Returns: 

68 int: The status code. 

69 """ 

70 return response.status_code 

71 

72 @staticmethod 

73 def _get_headers(response): 

74 """Access the headers from an HTTP response. 

75 

76 Args: 

77 response (~requests.Response): The HTTP response object. 

78 

79 Returns: 

80 ~requests.structures.CaseInsensitiveDict: The header mapping (keys 

81 are case-insensitive). 

82 """ 

83 return response.headers 

84 

85 @staticmethod 

86 def _get_body(response): 

87 """Access the response body from an HTTP response. 

88 

89 Args: 

90 response (~requests.Response): The HTTP response object. 

91 

92 Returns: 

93 bytes: The body of the ``response``. 

94 """ 

95 return response.content 

96 

97 

98class RawRequestsMixin(RequestsMixin): 

99 @staticmethod 

100 def _get_body(response): 

101 """Access the response body from an HTTP response. 

102 

103 Args: 

104 response (~requests.Response): The HTTP response object. 

105 

106 Returns: 

107 bytes: The body of the ``response``. 

108 """ 

109 if response._content is False: 

110 response._content = b"".join( 

111 response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False) 

112 ) 

113 response._content_consumed = True 

114 return response._content 

115 

116 

117def wait_and_retry(func, get_status_code, retry_strategy): 

118 """Attempts to retry a call to ``func`` until success. 

119 

120 Expects ``func`` to return an HTTP response and uses ``get_status_code`` 

121 to check if the response is retry-able. 

122 

123 ``func`` is expected to raise a failure status code as a 

124 common.InvalidResponse, at which point this method will check the code 

125 against the common.RETRIABLE list of retriable status codes. 

126 

127 Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current 

128 ``retry_strategy``) returns :data:`False`. Uses 

129 :func:`_helpers.calculate_retry_wait` to double the wait time (with jitter) 

130 after each attempt. 

131 

132 Args: 

133 func (Callable): A callable that takes no arguments and produces 

134 an HTTP response which will be checked as retry-able. 

135 get_status_code (Callable[Any, int]): Helper to get a status code 

136 from a response. 

137 retry_strategy (~google.resumable_media.common.RetryStrategy): The 

138 strategy to use if the request fails and must be retried. 

139 

140 Returns: 

141 object: The return value of ``func``. 

142 """ 

143 total_sleep = 0.0 

144 num_retries = 0 

145 # base_wait will be multiplied by the multiplier on the first retry. 

146 base_wait = float(retry_strategy.initial_delay) / retry_strategy.multiplier 

147 

148 # Set the retriable_exception_type if possible. We expect requests to be 

149 # present here and the transport to be using requests.exceptions errors, 

150 # but due to loose coupling with the transport layer we can't guarantee it. 

151 

152 while True: # return on success or when retries exhausted. 

153 error = None 

154 try: 

155 response = func() 

156 except _CONNECTION_ERROR_CLASSES as e: 

157 error = e # Fall through to retry, if there are retries left. 

158 except common.InvalidResponse as e: 

159 # An InvalidResponse is only retriable if its status code matches. 

160 # The `process_response()` method on a Download or Upload method 

161 # will convert the status code into an exception. 

162 if get_status_code(e.response) in common.RETRYABLE: 

163 error = e # Fall through to retry, if there are retries left. 

164 else: 

165 raise # If the status code is not retriable, raise w/o retry. 

166 else: 

167 return response 

168 

169 base_wait, wait_time = _helpers.calculate_retry_wait( 

170 base_wait, retry_strategy.max_sleep, retry_strategy.multiplier 

171 ) 

172 num_retries += 1 

173 total_sleep += wait_time 

174 

175 # Check if (another) retry is allowed. If retries are exhausted and 

176 # no acceptable response was received, raise the retriable error. 

177 if not retry_strategy.retry_allowed(total_sleep, num_retries): 

178 raise error 

179 

180 time.sleep(wait_time)