Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared utilities used by both downloads and uploads."""
17from __future__ import absolute_import
19import base64
20import hashlib
21import logging
22import random
23import warnings
24from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
26from google.resumable_media import common
28RANGE_HEADER = "range"
29CONTENT_RANGE_HEADER = "content-range"
30CONTENT_ENCODING_HEADER = "content-encoding"
32_SLOW_CRC32C_WARNING = (
33 "Currently using crcmod in pure python form. This is a slow "
34 "implementation. Python 3 has a faster implementation, `google-crc32c`, "
35 "which will be used if it is installed."
36)
37_GENERATION_HEADER = "x-goog-generation"
38_HASH_HEADER = "x-goog-hash"
39_STORED_CONTENT_ENCODING_HEADER = "x-goog-stored-content-encoding"
41_MISSING_CHECKSUM = """\
42No {checksum_type} checksum was returned from the service while downloading {}
43(which happens for composite objects), so client-side content integrity
44checking is not being performed."""
45_LOGGER = logging.getLogger(__name__)
48def do_nothing():
49 """Simple default callback."""
52def header_required(response, name, get_headers, callback=do_nothing):
53 """Checks that a specific header is in a headers dictionary.
55 Args:
56 response (object): An HTTP response object, expected to have a
57 ``headers`` attribute that is a ``Mapping[str, str]``.
58 name (str): The name of a required header.
59 get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
60 from an HTTP response.
61 callback (Optional[Callable]): A callback that takes no arguments,
62 to be executed when an exception is being raised.
64 Returns:
65 str: The desired header.
67 Raises:
68 ~google.resumable_media.common.InvalidResponse: If the header
69 is missing.
70 """
71 headers = get_headers(response)
72 if name not in headers:
73 callback()
74 raise common.InvalidResponse(
75 response, "Response headers must contain header", name
76 )
78 return headers[name]
81def require_status_code(response, status_codes, get_status_code, callback=do_nothing):
82 """Require a response has a status code among a list.
84 Args:
85 response (object): The HTTP response object.
86 status_codes (tuple): The acceptable status codes.
87 get_status_code (Callable[Any, int]): Helper to get a status code
88 from a response.
89 callback (Optional[Callable]): A callback that takes no arguments,
90 to be executed when an exception is being raised.
92 Returns:
93 int: The status code.
95 Raises:
96 ~google.resumable_media.common.InvalidResponse: If the status code
97 is not one of the values in ``status_codes``.
98 """
99 status_code = get_status_code(response)
100 if status_code not in status_codes:
101 if status_code not in common.RETRYABLE:
102 callback()
103 raise common.InvalidResponse(
104 response,
105 "Request failed with status code",
106 status_code,
107 "Expected one of",
108 *status_codes,
109 )
110 return status_code
113def calculate_retry_wait(base_wait, max_sleep, multiplier=2.0):
114 """Calculate the amount of time to wait before a retry attempt.
116 Wait time grows exponentially with the number of attempts, until
117 ``max_sleep``.
119 A random amount of jitter (between 0 and 1 seconds) is added to spread out
120 retry attempts from different clients.
122 Args:
123 base_wait (float): The "base" wait time (i.e. without any jitter)
124 that will be multiplied until it reaches the maximum sleep.
125 max_sleep (float): Maximum value that a sleep time is allowed to be.
126 multiplier (float): Multiplier to apply to the base wait.
128 Returns:
129 Tuple[float, float]: The new base wait time as well as the wait time
130 to be applied (with a random amount of jitter between 0 and 1 seconds
131 added).
132 """
133 new_base_wait = multiplier * base_wait
134 if new_base_wait > max_sleep:
135 new_base_wait = max_sleep
137 jitter_ms = random.randint(0, 1000)
138 return new_base_wait, new_base_wait + 0.001 * jitter_ms
141def _get_crc32c_object():
142 """Get crc32c object
143 Attempt to use the Google-CRC32c package. If it isn't available, try
144 to use CRCMod. CRCMod might be using a 'slow' varietal. If so, warn...
145 """
146 try:
147 import google_crc32c # type: ignore
149 crc_obj = google_crc32c.Checksum()
150 except ImportError:
151 try:
152 import crcmod # type: ignore
154 crc_obj = crcmod.predefined.Crc("crc-32c")
155 _is_fast_crcmod()
157 except ImportError:
158 raise ImportError("Failed to import either `google-crc32c` or `crcmod`")
160 return crc_obj
163def _is_fast_crcmod():
164 # Determine if this is using the slow form of crcmod.
165 nested_crcmod = __import__(
166 "crcmod.crcmod",
167 globals(),
168 locals(),
169 ["_usingExtension"],
170 0,
171 )
172 fast_crc = getattr(nested_crcmod, "_usingExtension", False)
173 if not fast_crc:
174 warnings.warn(_SLOW_CRC32C_WARNING, RuntimeWarning, stacklevel=2)
175 return fast_crc
178def _get_metadata_key(checksum_type):
179 if checksum_type == "md5":
180 return "md5Hash"
181 else:
182 return checksum_type
185def prepare_checksum_digest(digest_bytestring):
186 """Convert a checksum object into a digest encoded for an HTTP header.
188 Args:
189 bytes: A checksum digest bytestring.
191 Returns:
192 str: A base64 string representation of the input.
193 """
194 encoded_digest = base64.b64encode(digest_bytestring)
195 # NOTE: ``b64encode`` returns ``bytes``, but HTTP headers expect ``str``.
196 return encoded_digest.decode("utf-8")
199def _get_expected_checksum(response, get_headers, media_url, checksum_type):
200 """Get the expected checksum and checksum object for the download response.
202 Args:
203 response (~requests.Response): The HTTP response object.
204 get_headers (callable: response->dict): returns response headers.
205 media_url (str): The URL containing the media to be downloaded.
206 checksum_type Optional(str): The checksum type to read from the headers,
207 exactly as it will appear in the headers (case-sensitive). Must be
208 "md5", "crc32c" or None.
210 Returns:
211 Tuple (Optional[str], object): The expected checksum of the response,
212 if it can be detected from the ``X-Goog-Hash`` header, and the
213 appropriate checksum object for the expected checksum.
214 """
215 if checksum_type not in ["md5", "crc32c", None]:
216 raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
217 elif checksum_type in ["md5", "crc32c"]:
218 headers = get_headers(response)
219 expected_checksum = _parse_checksum_header(
220 headers.get(_HASH_HEADER), response, checksum_label=checksum_type
221 )
223 if expected_checksum is None:
224 msg = _MISSING_CHECKSUM.format(
225 media_url, checksum_type=checksum_type.upper()
226 )
227 _LOGGER.info(msg)
228 checksum_object = _DoNothingHash()
229 else:
230 if checksum_type == "md5":
231 checksum_object = hashlib.md5()
232 else:
233 checksum_object = _get_crc32c_object()
234 else:
235 expected_checksum = None
236 checksum_object = _DoNothingHash()
238 return (expected_checksum, checksum_object)
241def _get_uploaded_checksum_from_headers(response, get_headers, checksum_type):
242 """Get the computed checksum and checksum object from the response headers.
244 Args:
245 response (~requests.Response): The HTTP response object.
246 get_headers (callable: response->dict): returns response headers.
247 checksum_type Optional(str): The checksum type to read from the headers,
248 exactly as it will appear in the headers (case-sensitive). Must be
249 "md5", "crc32c" or None.
251 Returns:
252 Tuple (Optional[str], object): The checksum of the response,
253 if it can be detected from the ``X-Goog-Hash`` header, and the
254 appropriate checksum object for the expected checksum.
255 """
256 if checksum_type not in ["md5", "crc32c", None]:
257 raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
258 elif checksum_type in ["md5", "crc32c"]:
259 headers = get_headers(response)
260 remote_checksum = _parse_checksum_header(
261 headers.get(_HASH_HEADER), response, checksum_label=checksum_type
262 )
263 else:
264 remote_checksum = None
266 return remote_checksum
269def _parse_checksum_header(header_value, response, checksum_label):
270 """Parses the checksum header from an ``X-Goog-Hash`` value.
272 .. _header reference: https://cloud.google.com/storage/docs/\
273 xml-api/reference-headers#xgooghash
275 Expects ``header_value`` (if not :data:`None`) to be in one of the three
276 following formats:
278 * ``crc32c=n03x6A==``
279 * ``md5=Ojk9c3dhfxgoKVVHYwFbHQ==``
280 * ``crc32c=n03x6A==,md5=Ojk9c3dhfxgoKVVHYwFbHQ==``
282 See the `header reference`_ for more information.
284 Args:
285 header_value (Optional[str]): The ``X-Goog-Hash`` header from
286 a download response.
287 response (~requests.Response): The HTTP response object.
288 checksum_label (str): The label of the header value to read, as in the
289 examples above. Typically "md5" or "crc32c"
291 Returns:
292 Optional[str]: The expected checksum of the response, if it
293 can be detected from the ``X-Goog-Hash`` header; otherwise, None.
295 Raises:
296 ~google.resumable_media.common.InvalidResponse: If there are
297 multiple checksums of the requested type in ``header_value``.
298 """
299 if header_value is None:
300 return None
302 matches = []
303 for checksum in header_value.split(","):
304 name, value = checksum.split("=", 1)
305 # Official docs say "," is the separator, but real-world responses have encountered ", "
306 if name.lstrip() == checksum_label:
307 matches.append(value)
309 if len(matches) == 0:
310 return None
311 elif len(matches) == 1:
312 return matches[0]
313 else:
314 raise common.InvalidResponse(
315 response,
316 "X-Goog-Hash header had multiple ``{}`` values.".format(checksum_label),
317 header_value,
318 matches,
319 )
322def _get_checksum_object(checksum_type):
323 """Respond with a checksum object for a supported type, if not None.
325 Raises ValueError if checksum_type is unsupported.
326 """
327 if checksum_type == "md5":
328 return hashlib.md5()
329 elif checksum_type == "crc32c":
330 return _get_crc32c_object()
331 elif checksum_type is None:
332 return None
333 else:
334 raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
337def _parse_generation_header(response, get_headers):
338 """Parses the generation header from an ``X-Goog-Generation`` value.
340 Args:
341 response (~requests.Response): The HTTP response object.
342 get_headers (callable: response->dict): returns response headers.
344 Returns:
345 Optional[long]: The object generation from the response, if it
346 can be detected from the ``X-Goog-Generation`` header; otherwise, None.
347 """
348 headers = get_headers(response)
349 object_generation = headers.get(_GENERATION_HEADER, None)
351 if object_generation is None:
352 return None
353 else:
354 return int(object_generation)
357def _get_generation_from_url(media_url):
358 """Retrieve the object generation query param specified in the media url.
360 Args:
361 media_url (str): The URL containing the media to be downloaded.
363 Returns:
364 long: The object generation from the media url if exists; otherwise, None.
365 """
367 _, _, _, query, _ = urlsplit(media_url)
368 query_params = parse_qs(query)
369 object_generation = query_params.get("generation", None)
371 if object_generation is None:
372 return None
373 else:
374 return int(object_generation[0])
377def add_query_parameters(media_url, query_params):
378 """Add query parameters to a base url.
380 Args:
381 media_url (str): The URL containing the media to be downloaded.
382 query_params (dict): Names and values of the query parameters to add.
384 Returns:
385 str: URL with additional query strings appended.
386 """
388 if len(query_params) == 0:
389 return media_url
391 scheme, netloc, path, query, frag = urlsplit(media_url)
392 params = parse_qs(query)
393 new_params = {**params, **query_params}
394 query = urlencode(new_params, doseq=True)
395 return urlunsplit((scheme, netloc, path, query, frag))
398def _is_decompressive_transcoding(response, get_headers):
399 """Returns True if the object was served decompressed. This happens when the
400 "x-goog-stored-content-encoding" header is "gzip" and "content-encoding" header
401 is not "gzip". See more at: https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip
402 Args:
403 response (~requests.Response): The HTTP response object.
404 get_headers (callable: response->dict): returns response headers.
405 Returns:
406 bool: Returns True if decompressive transcoding has occurred; otherwise, False.
407 """
408 headers = get_headers(response)
409 return (
410 headers.get(_STORED_CONTENT_ENCODING_HEADER) == "gzip"
411 and headers.get(CONTENT_ENCODING_HEADER) != "gzip"
412 )
415class _DoNothingHash(object):
416 """Do-nothing hash object.
418 Intended as a stand-in for ``hashlib.md5`` or a crc32c checksum
419 implementation in cases where it isn't necessary to compute the hash.
420 """
422 def update(self, unused_chunk):
423 """Do-nothing ``update`` method.
425 Intended to match the interface of ``hashlib.md5`` and other checksums.
427 Args:
428 unused_chunk (bytes): A chunk of data.
429 """