Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Virtual bases classes for downloading media from Google APIs."""
17import http.client
18import re
20from google.resumable_media import _helpers, common
22_CONTENT_RANGE_RE = re.compile(
23 r"bytes (?P<start_byte>\d+)-(?P<end_byte>\d+)/(?P<total_bytes>\d+)",
24 flags=re.IGNORECASE,
25)
26_ACCEPTABLE_STATUS_CODES = (http.client.OK, http.client.PARTIAL_CONTENT)
27_GET = "GET"
28_ZERO_CONTENT_RANGE_HEADER = "bytes */0"
31class DownloadBase(object):
32 """Base class for download helpers.
34 Defines core shared behavior across different download types.
36 Args:
37 media_url (str): The URL containing the media to be downloaded.
38 stream (IO[bytes]): A write-able stream (i.e. file-like object) that
39 the downloaded resource can be written to.
40 start (int): The first byte in a range to be downloaded.
41 end (int): The last byte in a range to be downloaded.
42 headers (Optional[Mapping[str, str]]): Extra headers that should
43 be sent with the request, e.g. headers for encrypted data.
45 Attributes:
46 media_url (str): The URL containing the media to be downloaded.
47 start (Optional[int]): The first byte in a range to be downloaded.
48 end (Optional[int]): The last byte in a range to be downloaded.
49 """
51 def __init__(self, media_url, stream=None, start=None, end=None, headers=None):
52 self.media_url = media_url
53 self._stream = stream
54 self.start = start
55 self.end = end
56 if headers is None:
57 headers = {}
58 self._headers = headers
59 self._finished = False
60 self._retry_strategy = common.RetryStrategy()
62 @property
63 def finished(self):
64 """bool: Flag indicating if the download has completed."""
65 return self._finished
67 @staticmethod
68 def _get_status_code(response):
69 """Access the status code from an HTTP response.
71 Args:
72 response (object): The HTTP response object.
74 Raises:
75 NotImplementedError: Always, since virtual.
76 """
77 raise NotImplementedError("This implementation is virtual.")
79 @staticmethod
80 def _get_headers(response):
81 """Access the headers from an HTTP response.
83 Args:
84 response (object): The HTTP response object.
86 Raises:
87 NotImplementedError: Always, since virtual.
88 """
89 raise NotImplementedError("This implementation is virtual.")
91 @staticmethod
92 def _get_body(response):
93 """Access the response body from an HTTP response.
95 Args:
96 response (object): The HTTP response object.
98 Raises:
99 NotImplementedError: Always, since virtual.
100 """
101 raise NotImplementedError("This implementation is virtual.")
104class Download(DownloadBase):
105 """Helper to manage downloading a resource from a Google API.
107 "Slices" of the resource can be retrieved by specifying a range
108 with ``start`` and / or ``end``. However, in typical usage, neither
109 ``start`` nor ``end`` is expected to be provided.
111 Args:
112 media_url (str): The URL containing the media to be downloaded.
113 stream (IO[bytes]): A write-able stream (i.e. file-like object) that
114 the downloaded resource can be written to.
115 start (int): The first byte in a range to be downloaded. If not
116 provided, but ``end`` is provided, will download from the
117 beginning to ``end`` of the media.
118 end (int): The last byte in a range to be downloaded. If not
119 provided, but ``start`` is provided, will download from the
120 ``start`` to the end of the media.
121 headers (Optional[Mapping[str, str]]): Extra headers that should
122 be sent with the request, e.g. headers for encrypted data.
123 checksum Optional([str]): The type of checksum to compute to verify
124 the integrity of the object. The response headers must contain
125 a checksum of the requested type. If the headers lack an
126 appropriate checksum (for instance in the case of transcoded or
127 ranged downloads where the remote service does not know the
128 correct checksum) an INFO-level log will be emitted. Supported
129 values are "md5", "crc32c" and None.
130 """
132 def __init__(
133 self, media_url, stream=None, start=None, end=None, headers=None, checksum="md5"
134 ):
135 super(Download, self).__init__(
136 media_url, stream=stream, start=start, end=end, headers=headers
137 )
138 self.checksum = checksum
139 self._bytes_downloaded = 0
140 self._expected_checksum = None
141 self._checksum_object = None
142 self._object_generation = None
144 def _prepare_request(self):
145 """Prepare the contents of an HTTP request.
147 This is everything that must be done before a request that doesn't
148 require network I/O (or other I/O). This is based on the `sans-I/O`_
149 philosophy.
151 Returns:
152 Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
154 * HTTP verb for the request (always GET)
155 * the URL for the request
156 * the body of the request (always :data:`None`)
157 * headers for the request
159 Raises:
160 ValueError: If the current :class:`Download` has already
161 finished.
163 .. _sans-I/O: https://sans-io.readthedocs.io/
164 """
165 if self.finished:
166 raise ValueError("A download can only be used once.")
168 add_bytes_range(self.start, self.end, self._headers)
169 return _GET, self.media_url, None, self._headers
171 def _process_response(self, response):
172 """Process the response from an HTTP request.
174 This is everything that must be done after a request that doesn't
175 require network I/O (or other I/O). This is based on the `sans-I/O`_
176 philosophy.
178 Args:
179 response (object): The HTTP response object.
181 .. _sans-I/O: https://sans-io.readthedocs.io/
182 """
183 # Tombstone the current Download so it cannot be used again.
184 self._finished = True
185 _helpers.require_status_code(
186 response, _ACCEPTABLE_STATUS_CODES, self._get_status_code
187 )
189 def consume(self, transport, timeout=None):
190 """Consume the resource to be downloaded.
192 If a ``stream`` is attached to this download, then the downloaded
193 resource will be written to the stream.
195 Args:
196 transport (object): An object which can make authenticated
197 requests.
198 timeout (Optional[Union[float, Tuple[float, float]]]):
199 The number of seconds to wait for the server response.
200 Depending on the retry strategy, a request may be repeated
201 several times using the same timeout each time.
203 Can also be passed as a tuple (connect_timeout, read_timeout).
204 See :meth:`requests.Session.request` documentation for details.
206 Raises:
207 NotImplementedError: Always, since virtual.
208 """
209 raise NotImplementedError("This implementation is virtual.")
212class ChunkedDownload(DownloadBase):
213 """Download a resource in chunks from a Google API.
215 Args:
216 media_url (str): The URL containing the media to be downloaded.
217 chunk_size (int): The number of bytes to be retrieved in each
218 request.
219 stream (IO[bytes]): A write-able stream (i.e. file-like object) that
220 will be used to concatenate chunks of the resource as they are
221 downloaded.
222 start (int): The first byte in a range to be downloaded. If not
223 provided, defaults to ``0``.
224 end (int): The last byte in a range to be downloaded. If not
225 provided, will download to the end of the media.
226 headers (Optional[Mapping[str, str]]): Extra headers that should
227 be sent with each request, e.g. headers for data encryption
228 key headers.
230 Attributes:
231 media_url (str): The URL containing the media to be downloaded.
232 start (Optional[int]): The first byte in a range to be downloaded.
233 end (Optional[int]): The last byte in a range to be downloaded.
234 chunk_size (int): The number of bytes to be retrieved in each request.
236 Raises:
237 ValueError: If ``start`` is negative.
238 """
240 def __init__(self, media_url, chunk_size, stream, start=0, end=None, headers=None):
241 if start < 0:
242 raise ValueError(
243 "On a chunked download the starting value cannot be negative."
244 )
245 super(ChunkedDownload, self).__init__(
246 media_url, stream=stream, start=start, end=end, headers=headers
247 )
248 self.chunk_size = chunk_size
249 self._bytes_downloaded = 0
250 self._total_bytes = None
251 self._invalid = False
253 @property
254 def bytes_downloaded(self):
255 """int: Number of bytes that have been downloaded."""
256 return self._bytes_downloaded
258 @property
259 def total_bytes(self):
260 """Optional[int]: The total number of bytes to be downloaded."""
261 return self._total_bytes
263 @property
264 def invalid(self):
265 """bool: Indicates if the download is in an invalid state.
267 This will occur if a call to :meth:`consume_next_chunk` fails.
268 """
269 return self._invalid
271 def _get_byte_range(self):
272 """Determines the byte range for the next request.
274 Returns:
275 Tuple[int, int]: The pair of begin and end byte for the next
276 chunked request.
277 """
278 curr_start = self.start + self.bytes_downloaded
279 curr_end = curr_start + self.chunk_size - 1
280 # Make sure ``curr_end`` does not exceed ``end``.
281 if self.end is not None:
282 curr_end = min(curr_end, self.end)
283 # Make sure ``curr_end`` does not exceed ``total_bytes - 1``.
284 if self.total_bytes is not None:
285 curr_end = min(curr_end, self.total_bytes - 1)
286 return curr_start, curr_end
288 def _prepare_request(self):
289 """Prepare the contents of an HTTP request.
291 This is everything that must be done before a request that doesn't
292 require network I/O (or other I/O). This is based on the `sans-I/O`_
293 philosophy.
295 .. note:
297 This method will be used multiple times, so ``headers`` will
298 be mutated in between requests. However, we don't make a copy
299 since the same keys are being updated.
301 Returns:
302 Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
304 * HTTP verb for the request (always GET)
305 * the URL for the request
306 * the body of the request (always :data:`None`)
307 * headers for the request
309 Raises:
310 ValueError: If the current download has finished.
311 ValueError: If the current download is invalid.
313 .. _sans-I/O: https://sans-io.readthedocs.io/
314 """
315 if self.finished:
316 raise ValueError("Download has finished.")
317 if self.invalid:
318 raise ValueError("Download is invalid and cannot be re-used.")
320 curr_start, curr_end = self._get_byte_range()
321 add_bytes_range(curr_start, curr_end, self._headers)
322 return _GET, self.media_url, None, self._headers
324 def _make_invalid(self):
325 """Simple setter for ``invalid``.
327 This is intended to be passed along as a callback to helpers that
328 raise an exception so they can mark this instance as invalid before
329 raising.
330 """
331 self._invalid = True
333 def _process_response(self, response):
334 """Process the response from an HTTP request.
336 This is everything that must be done after a request that doesn't
337 require network I/O. This is based on the `sans-I/O`_ philosophy.
339 For the time being, this **does require** some form of I/O to write
340 a chunk to ``stream``. However, this will (almost) certainly not be
341 network I/O.
343 Updates the current state after consuming a chunk. First,
344 increments ``bytes_downloaded`` by the number of bytes in the
345 ``content-length`` header.
347 If ``total_bytes`` is already set, this assumes (but does not check)
348 that we already have the correct value and doesn't bother to check
349 that it agrees with the headers.
351 We expect the **total** length to be in the ``content-range`` header,
352 but this header is only present on requests which sent the ``range``
353 header. This response header should be of the form
354 ``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
355 should be the same as the ``Content-Length``.
357 Args:
358 response (object): The HTTP response object (need headers).
360 Raises:
361 ~google.resumable_media.common.InvalidResponse: If the number
362 of bytes in the body doesn't match the content length header.
364 .. _sans-I/O: https://sans-io.readthedocs.io/
365 """
366 # Verify the response before updating the current instance.
367 if _check_for_zero_content_range(
368 response, self._get_status_code, self._get_headers
369 ):
370 self._finished = True
371 return
373 _helpers.require_status_code(
374 response,
375 _ACCEPTABLE_STATUS_CODES,
376 self._get_status_code,
377 callback=self._make_invalid,
378 )
379 headers = self._get_headers(response)
380 response_body = self._get_body(response)
382 start_byte, end_byte, total_bytes = get_range_info(
383 response, self._get_headers, callback=self._make_invalid
384 )
386 transfer_encoding = headers.get("transfer-encoding")
388 if transfer_encoding is None:
389 content_length = _helpers.header_required(
390 response,
391 "content-length",
392 self._get_headers,
393 callback=self._make_invalid,
394 )
395 num_bytes = int(content_length)
396 if len(response_body) != num_bytes:
397 self._make_invalid()
398 raise common.InvalidResponse(
399 response,
400 "Response is different size than content-length",
401 "Expected",
402 num_bytes,
403 "Received",
404 len(response_body),
405 )
406 else:
407 # 'content-length' header not allowed with chunked encoding.
408 num_bytes = end_byte - start_byte + 1
410 # First update ``bytes_downloaded``.
411 self._bytes_downloaded += num_bytes
412 # If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
413 if self.end is not None and end_byte >= self.end:
414 self._finished = True
415 elif end_byte >= total_bytes - 1:
416 self._finished = True
417 # NOTE: We only use ``total_bytes`` if not already known.
418 if self.total_bytes is None:
419 self._total_bytes = total_bytes
420 # Write the response body to the stream.
421 self._stream.write(response_body)
423 def consume_next_chunk(self, transport, timeout=None):
424 """Consume the next chunk of the resource to be downloaded.
426 Args:
427 transport (object): An object which can make authenticated
428 requests.
429 timeout (Optional[Union[float, Tuple[float, float]]]):
430 The number of seconds to wait for the server response.
431 Depending on the retry strategy, a request may be repeated
432 several times using the same timeout each time.
434 Can also be passed as a tuple (connect_timeout, read_timeout).
435 See :meth:`requests.Session.request` documentation for details.
437 Raises:
438 NotImplementedError: Always, since virtual.
439 """
440 raise NotImplementedError("This implementation is virtual.")
443def add_bytes_range(start, end, headers):
444 """Add a bytes range to a header dictionary.
446 Some possible inputs and the corresponding bytes ranges::
448 >>> headers = {}
449 >>> add_bytes_range(None, None, headers)
450 >>> headers
451 {}
452 >>> add_bytes_range(500, 999, headers)
453 >>> headers['range']
454 'bytes=500-999'
455 >>> add_bytes_range(None, 499, headers)
456 >>> headers['range']
457 'bytes=0-499'
458 >>> add_bytes_range(-500, None, headers)
459 >>> headers['range']
460 'bytes=-500'
461 >>> add_bytes_range(9500, None, headers)
462 >>> headers['range']
463 'bytes=9500-'
465 Args:
466 start (Optional[int]): The first byte in a range. Can be zero,
467 positive, negative or :data:`None`.
468 end (Optional[int]): The last byte in a range. Assumed to be
469 positive.
470 headers (Mapping[str, str]): A headers mapping which can have the
471 bytes range added if at least one of ``start`` or ``end``
472 is not :data:`None`.
473 """
474 if start is None:
475 if end is None:
476 # No range to add.
477 return
478 else:
479 # NOTE: This assumes ``end`` is non-negative.
480 bytes_range = "0-{:d}".format(end)
481 else:
482 if end is None:
483 if start < 0:
484 bytes_range = "{:d}".format(start)
485 else:
486 bytes_range = "{:d}-".format(start)
487 else:
488 # NOTE: This is invalid if ``start < 0``.
489 bytes_range = "{:d}-{:d}".format(start, end)
491 headers[_helpers.RANGE_HEADER] = "bytes=" + bytes_range
494def get_range_info(response, get_headers, callback=_helpers.do_nothing):
495 """Get the start, end and total bytes from a content range header.
497 Args:
498 response (object): An HTTP response object.
499 get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
500 from an HTTP response.
501 callback (Optional[Callable]): A callback that takes no arguments,
502 to be executed when an exception is being raised.
504 Returns:
505 Tuple[int, int, int]: The start byte, end byte and total bytes.
507 Raises:
508 ~google.resumable_media.common.InvalidResponse: If the
509 ``Content-Range`` header is not of the form
510 ``bytes {start}-{end}/{total}``.
511 """
512 content_range = _helpers.header_required(
513 response, _helpers.CONTENT_RANGE_HEADER, get_headers, callback=callback
514 )
515 match = _CONTENT_RANGE_RE.match(content_range)
516 if match is None:
517 callback()
518 raise common.InvalidResponse(
519 response,
520 "Unexpected content-range header",
521 content_range,
522 'Expected to be of the form "bytes {start}-{end}/{total}"',
523 )
525 return (
526 int(match.group("start_byte")),
527 int(match.group("end_byte")),
528 int(match.group("total_bytes")),
529 )
532def _check_for_zero_content_range(response, get_status_code, get_headers):
533 """Validate if response status code is 416 and content range is zero.
535 This is the special case for handling zero bytes files.
537 Args:
538 response (object): An HTTP response object.
539 get_status_code (Callable[Any, int]): Helper to get a status code
540 from a response.
541 get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
542 from an HTTP response.
544 Returns:
545 bool: True if content range total bytes is zero, false otherwise.
546 """
547 if get_status_code(response) == http.client.REQUESTED_RANGE_NOT_SATISFIABLE:
548 content_range = _helpers.header_required(
549 response,
550 _helpers.CONTENT_RANGE_HEADER,
551 get_headers,
552 callback=_helpers.do_nothing,
553 )
554 if content_range == _ZERO_CONTENT_RANGE_HEADER:
555 return True
556 return False