Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/google/cloud/storage/retry.py: 60%

1# Copyright 2020 Google LLC 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15"""Helpers for configuring retries with exponential back-off. 

16 

17See [Retry Strategy for Google Cloud Storage](https://cloud.google.com/storage/docs/retry-strategy#client-libraries) 

18""" 

19 

20import http 

21 

22import requests 

23import requests.exceptions as requests_exceptions 

24import urllib3 

25 

26from google.api_core import exceptions as api_exceptions 

27from google.api_core import retry 

28from google.auth import exceptions as auth_exceptions 

29from google.cloud.storage.exceptions import InvalidResponse 

30 

31 

32_RETRYABLE_TYPES = ( 

33 api_exceptions.TooManyRequests, # 429 

34 api_exceptions.InternalServerError, # 500 

35 api_exceptions.BadGateway, # 502 

36 api_exceptions.ServiceUnavailable, # 503 

37 api_exceptions.GatewayTimeout, # 504 

38 ConnectionError, 

39 requests.ConnectionError, 

40 requests_exceptions.ChunkedEncodingError, 

41 requests_exceptions.Timeout, 

42 http.client.BadStatusLine, 

43 http.client.IncompleteRead, 

44 http.client.ResponseNotReady, 

45 urllib3.exceptions.PoolError, 

46 urllib3.exceptions.ProtocolError, 

47 urllib3.exceptions.SSLError, 

48 urllib3.exceptions.TimeoutError, 

49) 

50 

51 

52_RETRYABLE_STATUS_CODES = ( 

53 http.client.TOO_MANY_REQUESTS, # 429 

54 http.client.REQUEST_TIMEOUT, # 408 

55 http.client.INTERNAL_SERVER_ERROR, # 500 

56 http.client.BAD_GATEWAY, # 502 

57 http.client.SERVICE_UNAVAILABLE, # 503 

58 http.client.GATEWAY_TIMEOUT, # 504 

59) 

60 

61 

62def _should_retry(exc): 

63 """Predicate for determining when to retry.""" 

64 if isinstance(exc, _RETRYABLE_TYPES): 

65 return True 

66 elif isinstance(exc, api_exceptions.GoogleAPICallError): 

67 return exc.code in _RETRYABLE_STATUS_CODES 

68 elif isinstance(exc, InvalidResponse): 

69 return exc.response.status_code in _RETRYABLE_STATUS_CODES 

70 elif isinstance(exc, auth_exceptions.TransportError): 

71 return _should_retry(exc.args[0]) 

72 else: 

73 return False 

74 

75 

76DEFAULT_RETRY = retry.Retry(predicate=_should_retry) 

77"""The default retry object. 

78 

79This retry setting will retry all _RETRYABLE_TYPES and any status codes from 

80_ADDITIONAL_RETRYABLE_STATUS_CODES. 

81 

82To modify the default retry behavior, create a new retry object modeled after 

83this one by calling it a ``with_XXX`` method. For example, to create a copy of 

84DEFAULT_RETRY with a deadline of 30 seconds, pass 

85``retry=DEFAULT_RETRY.with_deadline(30)``. See google-api-core reference 

86(https://googleapis.dev/python/google-api-core/latest/retry.html) for details. 

87""" 

88 

89 

90class ConditionalRetryPolicy(object): 

91 """A class for use when an API call is only conditionally safe to retry. 

92 

93 This class is intended for use in inspecting the API call parameters of an 

94 API call to verify that any flags necessary to make the API call idempotent 

95 (such as specifying an ``if_generation_match`` or related flag) are present. 

96 

97 It can be used in place of a ``retry.Retry`` object, in which case 

98 ``_http.Connection.api_request`` will pass the requested api call keyword 

99 arguments into the ``conditional_predicate`` and return the ``retry_policy`` 

100 if the conditions are met. 

101 

102 :type retry_policy: class:`google.api_core.retry.Retry` 

103 :param retry_policy: A retry object defining timeouts, persistence and which 

104 exceptions to retry. 

105 

106 :type conditional_predicate: callable 

107 :param conditional_predicate: A callable that accepts exactly the number of 

108 arguments in ``required_kwargs``, in order, and returns True if the 

109 arguments have sufficient data to determine that the call is safe to 

110 retry (idempotent). 

111 

112 :type required_kwargs: list(str) 

113 :param required_kwargs: 

114 A list of keyword argument keys that will be extracted from the API call 

115 and passed into the ``conditional predicate`` in order. For example, 

116 ``["query_params"]`` is commmonly used for preconditions in query_params. 

117 """ 

118 

119 def __init__(self, retry_policy, conditional_predicate, required_kwargs): 

120 self.retry_policy = retry_policy 

121 self.conditional_predicate = conditional_predicate 

122 self.required_kwargs = required_kwargs 

123 

124 def get_retry_policy_if_conditions_met(self, **kwargs): 

125 if self.conditional_predicate(*[kwargs[key] for key in self.required_kwargs]): 

126 return self.retry_policy 

127 return None 

128 

129 

130def is_generation_specified(query_params): 

131 """Return True if generation or if_generation_match is specified.""" 

132 generation = query_params.get("generation") is not None 

133 if_generation_match = query_params.get("ifGenerationMatch") is not None 

134 return generation or if_generation_match 

135 

136 

137def is_metageneration_specified(query_params): 

138 """Return True if if_metageneration_match is specified.""" 

139 if_metageneration_match = query_params.get("ifMetagenerationMatch") is not None 

140 return if_metageneration_match 

141 

142 

143def is_etag_in_data(data): 

144 """Return True if an etag is contained in the request body. 

145 

146 :type data: dict or None 

147 :param data: A dict representing the request JSON body. If not passed, returns False. 

148 """ 

149 return data is not None and "etag" in data 

150 

151 

152def is_etag_in_json(data): 

153 """ 

154 ``is_etag_in_json`` is supported for backwards-compatibility reasons only; 

155 please use ``is_etag_in_data`` instead. 

156 """ 

157 return is_etag_in_data(data) 

158 

159 

160DEFAULT_RETRY_IF_GENERATION_SPECIFIED = ConditionalRetryPolicy( 

161 DEFAULT_RETRY, is_generation_specified, ["query_params"] 

162) 

163"""Conditional wrapper for the default retry object. 

164 

165This retry setting will retry all _RETRYABLE_TYPES and any status codes from 

166_ADDITIONAL_RETRYABLE_STATUS_CODES, but only if the request included an 

167``ifGenerationMatch`` header. 

168""" 

169 

170DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED = ConditionalRetryPolicy( 

171 DEFAULT_RETRY, is_metageneration_specified, ["query_params"] 

172) 

173"""Conditional wrapper for the default retry object. 

174 

175This retry setting will retry all _RETRYABLE_TYPES and any status codes from 

176_ADDITIONAL_RETRYABLE_STATUS_CODES, but only if the request included an 

177``ifMetagenerationMatch`` header. 

178""" 

179 

180DEFAULT_RETRY_IF_ETAG_IN_JSON = ConditionalRetryPolicy( 

181 DEFAULT_RETRY, is_etag_in_json, ["data"] 

182) 

183"""Conditional wrapper for the default retry object. 

184 

185This retry setting will retry all _RETRYABLE_TYPES and any status codes from 

186_ADDITIONAL_RETRYABLE_STATUS_CODES, but only if the request included an 

187``ETAG`` entry in its payload. 

188"""