Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/blob.py: 30%
657 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 07:13 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 07:13 +0000
1# Copyright 2014 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15# pylint: disable=too-many-lines
17"""Create / interact with Google Cloud Storage blobs.
18"""
20import base64
21import copy
22import hashlib
23from io import BytesIO
24from io import TextIOWrapper
25import logging
26import mimetypes
27import os
28import re
29from email.parser import HeaderParser
30from urllib.parse import parse_qsl
31from urllib.parse import quote
32from urllib.parse import urlencode
33from urllib.parse import urlsplit
34from urllib.parse import urlunsplit
35import warnings
37from google import resumable_media
38from google.resumable_media.requests import ChunkedDownload
39from google.resumable_media.requests import Download
40from google.resumable_media.requests import RawDownload
41from google.resumable_media.requests import RawChunkedDownload
42from google.resumable_media.requests import MultipartUpload
43from google.resumable_media.requests import ResumableUpload
45from google.api_core.iam import Policy
46from google.cloud import exceptions
47from google.cloud._helpers import _bytes_to_unicode
48from google.cloud._helpers import _datetime_to_rfc3339
49from google.cloud._helpers import _rfc3339_nanos_to_datetime
50from google.cloud._helpers import _to_bytes
51from google.cloud.exceptions import NotFound
52from google.cloud.storage._helpers import _add_etag_match_headers
53from google.cloud.storage._helpers import _add_generation_match_parameters
54from google.cloud.storage._helpers import _PropertyMixin
55from google.cloud.storage._helpers import _scalar_property
56from google.cloud.storage._helpers import _bucket_bound_hostname_url
57from google.cloud.storage._helpers import _raise_if_more_than_one_set
58from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry
59from google.cloud.storage._helpers import _get_default_headers
60from google.cloud.storage._signing import generate_signed_url_v2
61from google.cloud.storage._signing import generate_signed_url_v4
62from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
63from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
64from google.cloud.storage._helpers import _API_VERSION
65from google.cloud.storage.acl import ACL
66from google.cloud.storage.acl import ObjectACL
67from google.cloud.storage.constants import _DEFAULT_TIMEOUT
68from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS
69from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS
70from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS
71from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
72from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS
73from google.cloud.storage.constants import STANDARD_STORAGE_CLASS
74from google.cloud.storage.retry import ConditionalRetryPolicy
75from google.cloud.storage.retry import DEFAULT_RETRY
76from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON
77from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED
78from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
79from google.cloud.storage.fileio import BlobReader
80from google.cloud.storage.fileio import BlobWriter
83_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
84_DEFAULT_CONTENT_TYPE = "application/octet-stream"
85_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
86_BASE_UPLOAD_TEMPLATE = (
87 "{hostname}/upload/storage/{api_version}{bucket_path}/o?uploadType="
88)
89_MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "multipart"
90_RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "resumable"
91# NOTE: "acl" is also writeable but we defer ACL management to
92# the classes in the google.cloud.storage.acl module.
93_CONTENT_TYPE_FIELD = "contentType"
94_WRITABLE_FIELDS = (
95 "cacheControl",
96 "contentDisposition",
97 "contentEncoding",
98 "contentLanguage",
99 _CONTENT_TYPE_FIELD,
100 "crc32c",
101 "customTime",
102 "md5Hash",
103 "metadata",
104 "name",
105 "storageClass",
106)
107_READ_LESS_THAN_SIZE = (
108 "Size {:d} was specified but the file-like object only had " "{:d} bytes remaining."
109)
110_CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE = (
111 "A checksum of type `{}` was requested, but checksumming is not available "
112 "for downloads when chunk_size is set."
113)
114_COMPOSE_IF_GENERATION_LIST_DEPRECATED = (
115 "'if_generation_match: type list' is deprecated and supported for "
116 "backwards-compatability reasons only. Use 'if_source_generation_match' "
117 "instead' to match source objects' generations."
118)
119_COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR = (
120 "Use 'if_generation_match' to match the generation of the destination "
121 "object by passing in a generation number, instead of a list. "
122 "Use 'if_source_generation_match' to match source objects generations."
123)
124_COMPOSE_IF_METAGENERATION_LIST_DEPRECATED = (
125 "'if_metageneration_match: type list' is deprecated and supported for "
126 "backwards-compatability reasons only. Note that the metageneration to "
127 "be matched is that of the destination blob. Please pass in a single "
128 "value (type long)."
129)
130_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR = (
131 "'if_source_generation_match' length must be the same as 'sources' length"
132)
133_DOWNLOAD_AS_STRING_DEPRECATED = (
134 "Blob.download_as_string() is deprecated and will be removed in future. "
135 "Use Blob.download_as_bytes() instead."
136)
139_DEFAULT_CHUNKSIZE = 104857600 # 1024 * 1024 B * 100 = 100 MB
140_MAX_MULTIPART_SIZE = 8388608 # 8 MB
142_logger = logging.getLogger(__name__)
145class Blob(_PropertyMixin):
146 """A wrapper around Cloud Storage's concept of an ``Object``.
148 :type name: str
149 :param name: The name of the blob. This corresponds to the unique path of
150 the object in the bucket. If bytes, will be converted to a
151 unicode object. Blob / object names can contain any sequence
152 of valid unicode characters, of length 1-1024 bytes when
153 UTF-8 encoded.
155 :type bucket: :class:`google.cloud.storage.bucket.Bucket`
156 :param bucket: The bucket to which this blob belongs.
158 :type chunk_size: int
159 :param chunk_size:
160 (Optional) The size of a chunk of data whenever iterating (in bytes).
161 This must be a multiple of 256 KB per the API specification. If not
162 specified, the chunk_size of the blob itself is used. If that is not
163 specified, a default value of 40 MB is used.
165 :type encryption_key: bytes
166 :param encryption_key:
167 (Optional) 32 byte encryption key for customer-supplied encryption.
168 See https://cloud.google.com/storage/docs/encryption#customer-supplied.
170 :type kms_key_name: str
171 :param kms_key_name:
172 (Optional) Resource name of Cloud KMS key used to encrypt the blob's
173 contents.
175 :type generation: long
176 :param generation:
177 (Optional) If present, selects a specific revision of this object.
178 """
180 _chunk_size = None # Default value for each instance.
181 _CHUNK_SIZE_MULTIPLE = 256 * 1024
182 """Number (256 KB, in bytes) that must divide the chunk size."""
184 STORAGE_CLASSES = (
185 STANDARD_STORAGE_CLASS,
186 NEARLINE_STORAGE_CLASS,
187 COLDLINE_STORAGE_CLASS,
188 ARCHIVE_STORAGE_CLASS,
189 MULTI_REGIONAL_LEGACY_STORAGE_CLASS,
190 REGIONAL_LEGACY_STORAGE_CLASS,
191 )
192 """Allowed values for :attr:`storage_class`.
194 See
195 https://cloud.google.com/storage/docs/json_api/v1/objects#storageClass
196 https://cloud.google.com/storage/docs/per-object-storage-class
198 .. note::
199 This list does not include 'DURABLE_REDUCED_AVAILABILITY', which
200 is only documented for buckets (and deprecated).
201 """
203 def __init__(
204 self,
205 name,
206 bucket,
207 chunk_size=None,
208 encryption_key=None,
209 kms_key_name=None,
210 generation=None,
211 ):
212 """
213 property :attr:`name`
214 Get the blob's name.
215 """
216 name = _bytes_to_unicode(name)
217 super(Blob, self).__init__(name=name)
219 self.chunk_size = chunk_size # Check that setter accepts value.
220 self._bucket = bucket
221 self._acl = ObjectACL(self)
222 _raise_if_more_than_one_set(
223 encryption_key=encryption_key, kms_key_name=kms_key_name
224 )
226 self._encryption_key = encryption_key
228 if kms_key_name is not None:
229 self._properties["kmsKeyName"] = kms_key_name
231 if generation is not None:
232 self._properties["generation"] = generation
234 @property
235 def bucket(self):
236 """Bucket which contains the object.
238 :rtype: :class:`~google.cloud.storage.bucket.Bucket`
239 :returns: The object's bucket.
240 """
241 return self._bucket
243 @property
244 def chunk_size(self):
245 """Get the blob's default chunk size.
247 :rtype: int or ``NoneType``
248 :returns: The current blob's chunk size, if it is set.
249 """
250 return self._chunk_size
252 @chunk_size.setter
253 def chunk_size(self, value):
254 """Set the blob's default chunk size.
256 :type value: int
257 :param value: (Optional) The current blob's chunk size, if it is set.
259 :raises: :class:`ValueError` if ``value`` is not ``None`` and is not a
260 multiple of 256 KB.
261 """
262 if value is not None and value > 0 and value % self._CHUNK_SIZE_MULTIPLE != 0:
263 raise ValueError(
264 "Chunk size must be a multiple of %d." % (self._CHUNK_SIZE_MULTIPLE,)
265 )
266 self._chunk_size = value
268 @property
269 def encryption_key(self):
270 """Retrieve the customer-supplied encryption key for the object.
272 :rtype: bytes or ``NoneType``
273 :returns:
274 The encryption key or ``None`` if no customer-supplied encryption key was used,
275 or the blob's resource has not been loaded from the server.
276 """
277 return self._encryption_key
279 @encryption_key.setter
280 def encryption_key(self, value):
281 """Set the blob's encryption key.
283 See https://cloud.google.com/storage/docs/encryption#customer-supplied
285 To perform a key rotation for an encrypted blob, use :meth:`rewrite`.
286 See https://cloud.google.com/storage/docs/encryption/using-customer-supplied-keys?hl=ca#rotating
288 :type value: bytes
289 :param value: 32 byte encryption key for customer-supplied encryption.
290 """
291 self._encryption_key = value
293 @staticmethod
294 def path_helper(bucket_path, blob_name):
295 """Relative URL path for a blob.
297 :type bucket_path: str
298 :param bucket_path: The URL path for a bucket.
300 :type blob_name: str
301 :param blob_name: The name of the blob.
303 :rtype: str
304 :returns: The relative URL path for ``blob_name``.
305 """
306 return bucket_path + "/o/" + _quote(blob_name)
308 @property
309 def acl(self):
310 """Create our ACL on demand."""
311 return self._acl
313 def __repr__(self):
314 if self.bucket:
315 bucket_name = self.bucket.name
316 else:
317 bucket_name = None
319 return f"<Blob: {bucket_name}, {self.name}, {self.generation}>"
321 @property
322 def path(self):
323 """Getter property for the URL path to this Blob.
325 :rtype: str
326 :returns: The URL path to this Blob.
327 """
328 if not self.name:
329 raise ValueError("Cannot determine path without a blob name.")
331 return self.path_helper(self.bucket.path, self.name)
333 @property
334 def client(self):
335 """The client bound to this blob."""
336 return self.bucket.client
338 @property
339 def user_project(self):
340 """Project ID billed for API requests made via this blob.
342 Derived from bucket's value.
344 :rtype: str
345 """
346 return self.bucket.user_project
348 def _encryption_headers(self):
349 """Return any encryption headers needed to fetch the object.
351 :rtype: List(Tuple(str, str))
352 :returns: a list of tuples to be passed as headers.
353 """
354 return _get_encryption_headers(self._encryption_key)
356 @property
357 def _query_params(self):
358 """Default query parameters."""
359 params = {}
360 if self.generation is not None:
361 params["generation"] = self.generation
362 if self.user_project is not None:
363 params["userProject"] = self.user_project
364 return params
366 @property
367 def public_url(self):
368 """The public URL for this blob.
370 Use :meth:`make_public` to enable anonymous access via the returned
371 URL.
373 :rtype: `string`
374 :returns: The public URL for this blob.
375 """
376 return "{storage_base_url}/{bucket_name}/{quoted_name}".format(
377 storage_base_url=_API_ACCESS_ENDPOINT,
378 bucket_name=self.bucket.name,
379 quoted_name=_quote(self.name, safe=b"/~"),
380 )
382 @classmethod
383 def from_string(cls, uri, client=None):
384 """Get a constructor for blob object by URI.
386 .. code-block:: python
388 from google.cloud import storage
389 from google.cloud.storage.blob import Blob
390 client = storage.Client()
391 blob = Blob.from_string("gs://bucket/object", client=client)
393 :type uri: str
394 :param uri: The blob uri pass to get blob object.
396 :type client: :class:`~google.cloud.storage.client.Client`
397 :param client:
398 (Optional) The client to use. Application code should
399 *always* pass ``client``.
401 :rtype: :class:`google.cloud.storage.blob.Blob`
402 :returns: The blob object created.
403 """
404 from google.cloud.storage.bucket import Bucket
406 scheme, netloc, path, query, frag = urlsplit(uri)
407 if scheme != "gs":
408 raise ValueError("URI scheme must be gs")
410 bucket = Bucket(client, name=netloc)
411 return cls(path[1:], bucket)
413 def generate_signed_url(
414 self,
415 expiration=None,
416 api_access_endpoint=_API_ACCESS_ENDPOINT,
417 method="GET",
418 content_md5=None,
419 content_type=None,
420 response_disposition=None,
421 response_type=None,
422 generation=None,
423 headers=None,
424 query_parameters=None,
425 client=None,
426 credentials=None,
427 version=None,
428 service_account_email=None,
429 access_token=None,
430 virtual_hosted_style=False,
431 bucket_bound_hostname=None,
432 scheme="http",
433 ):
434 """Generates a signed URL for this blob.
436 .. note::
438 If you are on Google Compute Engine, you can't generate a signed
439 URL using GCE service account.
440 If you'd like to be able to generate a signed URL from GCE,
441 you can use a standard service account from a JSON file rather
442 than a GCE service account.
444 If you have a blob that you want to allow access to for a set
445 amount of time, you can use this method to generate a URL that
446 is only valid within a certain time period.
448 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-generate-signed-url-v4#storage_generate_signed_url_v4-python).
450 This is particularly useful if you don't want publicly
451 accessible blobs, but don't want to require users to explicitly
452 log in.
454 If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,
455 ``https`` works only if using a ``CDN``.
457 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
458 :param expiration:
459 Point in time when the signed URL should expire. If a ``datetime``
460 instance is passed without an explicit ``tzinfo`` set, it will be
461 assumed to be ``UTC``.
463 :type api_access_endpoint: str
464 :param api_access_endpoint: (Optional) URI base.
466 :type method: str
467 :param method: The HTTP verb that will be used when requesting the URL.
469 :type content_md5: str
470 :param content_md5:
471 (Optional) The MD5 hash of the object referenced by ``resource``.
473 :type content_type: str
474 :param content_type:
475 (Optional) The content type of the object referenced by
476 ``resource``.
478 :type response_disposition: str
479 :param response_disposition:
480 (Optional) Content disposition of responses to requests for the
481 signed URL. For example, to enable the signed URL to initiate a
482 file of ``blog.png``, use the value ``'attachment;
483 filename=blob.png'``.
485 :type response_type: str
486 :param response_type:
487 (Optional) Content type of responses to requests for the signed
488 URL. Ignored if content_type is set on object/blob metadata.
490 :type generation: str
491 :param generation:
492 (Optional) A value that indicates which generation of the resource
493 to fetch.
495 :type headers: dict
496 :param headers:
497 (Optional) Additional HTTP headers to be included as part of the
498 signed URLs. See:
499 https://cloud.google.com/storage/docs/xml-api/reference-headers
500 Requests using the signed URL *must* pass the specified header
501 (name and value) with each request for the URL.
503 :type query_parameters: dict
504 :param query_parameters:
505 (Optional) Additional query parameters to be included as part of the
506 signed URLs. See:
507 https://cloud.google.com/storage/docs/xml-api/reference-headers#query
509 :type client: :class:`~google.cloud.storage.client.Client`
510 :param client:
511 (Optional) The client to use. If not passed, falls back to the
512 ``client`` stored on the blob's bucket.
514 :type credentials: :class:`google.auth.credentials.Credentials`
515 :param credentials:
516 (Optional) The authorization credentials to attach to requests.
517 These credentials identify this application to the service. If
518 none are specified, the client will attempt to ascertain the
519 credentials from the environment.
521 :type version: str
522 :param version:
523 (Optional) The version of signed credential to create. Must be one
524 of 'v2' | 'v4'.
526 :type service_account_email: str
527 :param service_account_email:
528 (Optional) E-mail address of the service account.
530 :type access_token: str
531 :param access_token: (Optional) Access token for a service account.
533 :type virtual_hosted_style: bool
534 :param virtual_hosted_style:
535 (Optional) If true, then construct the URL relative the bucket's
536 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
538 :type bucket_bound_hostname: str
539 :param bucket_bound_hostname:
540 (Optional) If passed, then construct the URL relative to the
541 bucket-bound hostname. Value can be a bare or with scheme, e.g.,
542 'example.com' or 'http://example.com'. See:
543 https://cloud.google.com/storage/docs/request-endpoints#cname
545 :type scheme: str
546 :param scheme:
547 (Optional) If ``bucket_bound_hostname`` is passed as a bare
548 hostname, use this value as the scheme. ``https`` will work only
549 when using a CDN. Defaults to ``"http"``.
551 :raises: :exc:`ValueError` when version is invalid.
552 :raises: :exc:`TypeError` when expiration is not a valid type.
553 :raises: :exc:`AttributeError` if credentials is not an instance
554 of :class:`google.auth.credentials.Signing`.
556 :rtype: str
557 :returns: A signed URL you can use to access the resource
558 until expiration.
559 """
560 if version is None:
561 version = "v2"
562 elif version not in ("v2", "v4"):
563 raise ValueError("'version' must be either 'v2' or 'v4'")
565 quoted_name = _quote(self.name, safe=b"/~")
567 # If you are on Google Compute Engine, you can't generate a signed URL
568 # using GCE service account.
569 # See https://github.com/googleapis/google-auth-library-python/issues/50
570 if virtual_hosted_style:
571 api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com"
572 elif bucket_bound_hostname:
573 api_access_endpoint = _bucket_bound_hostname_url(
574 bucket_bound_hostname, scheme
575 )
576 else:
577 resource = f"/{self.bucket.name}/{quoted_name}"
579 if virtual_hosted_style or bucket_bound_hostname:
580 resource = f"/{quoted_name}"
582 if credentials is None:
583 client = self._require_client(client)
584 credentials = client._credentials
586 if version == "v2":
587 helper = generate_signed_url_v2
588 else:
589 helper = generate_signed_url_v4
591 if self._encryption_key is not None:
592 encryption_headers = _get_encryption_headers(self._encryption_key)
593 if headers is None:
594 headers = {}
595 if version == "v2":
596 # See: https://cloud.google.com/storage/docs/access-control/signed-urls-v2#about-canonical-extension-headers
597 v2_copy_only = "X-Goog-Encryption-Algorithm"
598 headers[v2_copy_only] = encryption_headers[v2_copy_only]
599 else:
600 headers.update(encryption_headers)
602 return helper(
603 credentials,
604 resource=resource,
605 expiration=expiration,
606 api_access_endpoint=api_access_endpoint,
607 method=method.upper(),
608 content_md5=content_md5,
609 content_type=content_type,
610 response_type=response_type,
611 response_disposition=response_disposition,
612 generation=generation,
613 headers=headers,
614 query_parameters=query_parameters,
615 service_account_email=service_account_email,
616 access_token=access_token,
617 )
619 def exists(
620 self,
621 client=None,
622 if_etag_match=None,
623 if_etag_not_match=None,
624 if_generation_match=None,
625 if_generation_not_match=None,
626 if_metageneration_match=None,
627 if_metageneration_not_match=None,
628 timeout=_DEFAULT_TIMEOUT,
629 retry=DEFAULT_RETRY,
630 ):
631 """Determines whether or not this blob exists.
633 If :attr:`user_project` is set on the bucket, bills the API request
634 to that project.
636 :type client: :class:`~google.cloud.storage.client.Client`
637 :param client:
638 (Optional) The client to use. If not passed, falls back to the
639 ``client`` stored on the blob's bucket.
641 :type if_etag_match: Union[str, Set[str]]
642 :param if_etag_match:
643 (Optional) See :ref:`using-if-etag-match`
645 :type if_etag_not_match: Union[str, Set[str]]
646 :param if_etag_not_match:
647 (Optional) See :ref:`using-if-etag-not-match`
649 :type if_generation_match: long
650 :param if_generation_match:
651 (Optional) See :ref:`using-if-generation-match`
653 :type if_generation_not_match: long
654 :param if_generation_not_match:
655 (Optional) See :ref:`using-if-generation-not-match`
657 :type if_metageneration_match: long
658 :param if_metageneration_match:
659 (Optional) See :ref:`using-if-metageneration-match`
661 :type if_metageneration_not_match: long
662 :param if_metageneration_not_match:
663 (Optional) See :ref:`using-if-metageneration-not-match`
665 :type timeout: float or tuple
666 :param timeout:
667 (Optional) The amount of time, in seconds, to wait
668 for the server response. See: :ref:`configuring_timeouts`
670 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
671 :param retry:
672 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
674 :rtype: bool
675 :returns: True if the blob exists in Cloud Storage.
676 """
677 client = self._require_client(client)
678 # We only need the status code (200 or not) so we seek to
679 # minimize the returned payload.
680 query_params = self._query_params
681 query_params["fields"] = "name"
683 _add_generation_match_parameters(
684 query_params,
685 if_generation_match=if_generation_match,
686 if_generation_not_match=if_generation_not_match,
687 if_metageneration_match=if_metageneration_match,
688 if_metageneration_not_match=if_metageneration_not_match,
689 )
691 headers = {}
692 _add_etag_match_headers(
693 headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
694 )
696 try:
697 # We intentionally pass `_target_object=None` since fields=name
698 # would limit the local properties.
699 client._get_resource(
700 self.path,
701 query_params=query_params,
702 headers=headers,
703 timeout=timeout,
704 retry=retry,
705 _target_object=None,
706 )
707 except NotFound:
708 # NOTE: This will not fail immediately in a batch. However, when
709 # Batch.finish() is called, the resulting `NotFound` will be
710 # raised.
711 return False
712 return True
714 def delete(
715 self,
716 client=None,
717 if_generation_match=None,
718 if_generation_not_match=None,
719 if_metageneration_match=None,
720 if_metageneration_not_match=None,
721 timeout=_DEFAULT_TIMEOUT,
722 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
723 ):
724 """Deletes a blob from Cloud Storage.
726 If :attr:`user_project` is set on the bucket, bills the API request
727 to that project.
729 :type client: :class:`~google.cloud.storage.client.Client`
730 :param client:
731 (Optional) The client to use. If not passed, falls back to the
732 ``client`` stored on the blob's bucket.
734 :type if_generation_match: long
735 :param if_generation_match:
736 (Optional) See :ref:`using-if-generation-match`
738 :type if_generation_not_match: long
739 :param if_generation_not_match:
740 (Optional) See :ref:`using-if-generation-not-match`
742 :type if_metageneration_match: long
743 :param if_metageneration_match:
744 (Optional) See :ref:`using-if-metageneration-match`
746 :type if_metageneration_not_match: long
747 :param if_metageneration_not_match:
748 (Optional) See :ref:`using-if-metageneration-not-match`
750 :type timeout: float or tuple
751 :param timeout:
752 (Optional) The amount of time, in seconds, to wait
753 for the server response. See: :ref:`configuring_timeouts`
755 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
756 :param retry:
757 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
759 :raises: :class:`google.cloud.exceptions.NotFound`
760 (propagated from
761 :meth:`google.cloud.storage.bucket.Bucket.delete_blob`).
762 """
763 self.bucket.delete_blob(
764 self.name,
765 client=client,
766 generation=self.generation,
767 timeout=timeout,
768 if_generation_match=if_generation_match,
769 if_generation_not_match=if_generation_not_match,
770 if_metageneration_match=if_metageneration_match,
771 if_metageneration_not_match=if_metageneration_not_match,
772 retry=retry,
773 )
775 def _get_transport(self, client):
776 """Return the client's transport.
778 :type client: :class:`~google.cloud.storage.client.Client`
779 :param client:
780 (Optional) The client to use. If not passed, falls back to the
781 ``client`` stored on the blob's bucket.
783 :rtype transport:
784 :class:`~google.auth.transport.requests.AuthorizedSession`
785 :returns: The transport (with credentials) that will
786 make authenticated requests.
787 """
788 client = self._require_client(client)
789 return client._http
791 def _get_download_url(
792 self,
793 client,
794 if_generation_match=None,
795 if_generation_not_match=None,
796 if_metageneration_match=None,
797 if_metageneration_not_match=None,
798 ):
799 """Get the download URL for the current blob.
801 If the ``media_link`` has been loaded, it will be used, otherwise
802 the URL will be constructed from the current blob's path (and possibly
803 generation) to avoid a round trip.
805 :type client: :class:`~google.cloud.storage.client.Client`
806 :param client: The client to use.
808 :type if_generation_match: long
809 :param if_generation_match:
810 (Optional) See :ref:`using-if-generation-match`
812 :type if_generation_not_match: long
813 :param if_generation_not_match:
814 (Optional) See :ref:`using-if-generation-not-match`
816 :type if_metageneration_match: long
817 :param if_metageneration_match:
818 (Optional) See :ref:`using-if-metageneration-match`
820 :type if_metageneration_not_match: long
821 :param if_metageneration_not_match:
822 (Optional) See :ref:`using-if-metageneration-not-match`
824 :rtype: str
825 :returns: The download URL for the current blob.
826 """
827 name_value_pairs = []
828 if self.media_link is None:
829 hostname = _get_host_name(client._connection)
830 base_url = _DOWNLOAD_URL_TEMPLATE.format(
831 hostname=hostname, path=self.path, api_version=_API_VERSION
832 )
833 if self.generation is not None:
834 name_value_pairs.append(("generation", f"{self.generation:d}"))
835 else:
836 base_url = self.media_link
838 if self.user_project is not None:
839 name_value_pairs.append(("userProject", self.user_project))
841 _add_generation_match_parameters(
842 name_value_pairs,
843 if_generation_match=if_generation_match,
844 if_generation_not_match=if_generation_not_match,
845 if_metageneration_match=if_metageneration_match,
846 if_metageneration_not_match=if_metageneration_not_match,
847 )
848 return _add_query_parameters(base_url, name_value_pairs)
850 def _extract_headers_from_download(self, response):
851 """Extract headers from a non-chunked request's http object.
853 This avoids the need to make a second request for commonly used
854 headers.
856 :type response:
857 :class requests.models.Response
858 :param response: The server response from downloading a non-chunked file
859 """
860 self._properties["contentEncoding"] = response.headers.get(
861 "Content-Encoding", None
862 )
863 self._properties[_CONTENT_TYPE_FIELD] = response.headers.get(
864 "Content-Type", None
865 )
866 self._properties["cacheControl"] = response.headers.get("Cache-Control", None)
867 self._properties["storageClass"] = response.headers.get(
868 "X-Goog-Storage-Class", None
869 )
870 self._properties["contentLanguage"] = response.headers.get(
871 "Content-Language", None
872 )
873 self._properties["etag"] = response.headers.get("ETag", None)
874 self._properties["generation"] = response.headers.get("X-goog-generation", None)
875 self._properties["metageneration"] = response.headers.get(
876 "X-goog-metageneration", None
877 )
878 # 'X-Goog-Hash': 'crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==',
879 x_goog_hash = response.headers.get("X-Goog-Hash", "")
881 if x_goog_hash:
882 digests = {}
883 for encoded_digest in x_goog_hash.split(","):
884 match = re.match(r"(crc32c|md5)=([\w\d/\+/]+={0,3})", encoded_digest)
885 if match:
886 method, digest = match.groups()
887 digests[method] = digest
889 self._properties["crc32c"] = digests.get("crc32c", None)
890 self._properties["md5Hash"] = digests.get("md5", None)
892 def _do_download(
893 self,
894 transport,
895 file_obj,
896 download_url,
897 headers,
898 start=None,
899 end=None,
900 raw_download=False,
901 timeout=_DEFAULT_TIMEOUT,
902 checksum="md5",
903 retry=None,
904 ):
905 """Perform a download without any error handling.
907 This is intended to be called by :meth:`download_to_file` so it can
908 be wrapped with error handling / remapping.
910 :type transport:
911 :class:`~google.auth.transport.requests.AuthorizedSession`
912 :param transport:
913 The transport (with credentials) that will make authenticated
914 requests.
916 :type file_obj: file
917 :param file_obj: A file handle to which to write the blob's data.
919 :type download_url: str
920 :param download_url: The URL where the media can be accessed.
922 :type headers: dict
923 :param headers: Headers to be sent with the request(s).
925 :type start: int
926 :param start: (Optional) The first byte in a range to be downloaded.
928 :type end: int
929 :param end: (Optional) The last byte in a range to be downloaded.
931 :type raw_download: bool
932 :param raw_download:
933 (Optional) If true, download the object without any expansion.
935 :type timeout: float or tuple
936 :param timeout:
937 (Optional) The amount of time, in seconds, to wait
938 for the server response. See: :ref:`configuring_timeouts`
940 :type checksum: str
941 :param checksum:
942 (Optional) The type of checksum to compute to verify the integrity
943 of the object. The response headers must contain a checksum of the
944 requested type. If the headers lack an appropriate checksum (for
945 instance in the case of transcoded or ranged downloads where the
946 remote service does not know the correct checksum, including
947 downloads where chunk_size is set) an INFO-level log will be
948 emitted. Supported values are "md5", "crc32c" and None. The default
949 is "md5".
951 :type retry: google.api_core.retry.Retry
952 :param retry: (Optional) How to retry the RPC. A None value will disable
953 retries. A google.api_core.retry.Retry value will enable retries,
954 and the object will configure backoff and timeout options. Custom
955 predicates (customizable error codes) are not supported for media
956 operations such as this one.
958 This private method does not accept ConditionalRetryPolicy values
959 because the information necessary to evaluate the policy is instead
960 evaluated in client.download_blob_to_file().
962 See the retry.py source code and docstrings in this package
963 (google.cloud.storage.retry) for information on retry types and how
964 to configure them.
965 """
967 retry_strategy = _api_core_retry_to_resumable_media_retry(retry)
969 if self.chunk_size is None:
970 if raw_download:
971 klass = RawDownload
972 else:
973 klass = Download
975 download = klass(
976 download_url,
977 stream=file_obj,
978 headers=headers,
979 start=start,
980 end=end,
981 checksum=checksum,
982 )
983 download._retry_strategy = retry_strategy
984 response = download.consume(transport, timeout=timeout)
985 self._extract_headers_from_download(response)
986 else:
988 if checksum:
989 msg = _CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE.format(checksum)
990 _logger.info(msg)
992 if raw_download:
993 klass = RawChunkedDownload
994 else:
995 klass = ChunkedDownload
997 download = klass(
998 download_url,
999 self.chunk_size,
1000 file_obj,
1001 headers=headers,
1002 start=start if start else 0,
1003 end=end,
1004 )
1006 download._retry_strategy = retry_strategy
1007 while not download.finished:
1008 download.consume_next_chunk(transport, timeout=timeout)
1010 def download_to_file(
1011 self,
1012 file_obj,
1013 client=None,
1014 start=None,
1015 end=None,
1016 raw_download=False,
1017 if_etag_match=None,
1018 if_etag_not_match=None,
1019 if_generation_match=None,
1020 if_generation_not_match=None,
1021 if_metageneration_match=None,
1022 if_metageneration_not_match=None,
1023 timeout=_DEFAULT_TIMEOUT,
1024 checksum="md5",
1025 retry=DEFAULT_RETRY,
1026 ):
1027 """Download the contents of this blob into a file-like object.
1029 .. note::
1031 If the server-set property, :attr:`media_link`, is not yet
1032 initialized, makes an additional API request to load it.
1034 If the :attr:`chunk_size` of a current blob is `None`, will download data
1035 in single download request otherwise it will download the :attr:`chunk_size`
1036 of data in each request.
1038 For more fine-grained control over the download process, check out
1039 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html).
1040 For example, this library allows downloading **parts** of a blob rather than the whole thing.
1042 If :attr:`user_project` is set on the bucket, bills the API request
1043 to that project.
1045 :type file_obj: file
1046 :param file_obj: A file handle to which to write the blob's data.
1048 :type client: :class:`~google.cloud.storage.client.Client`
1049 :param client:
1050 (Optional) The client to use. If not passed, falls back to the
1051 ``client`` stored on the blob's bucket.
1053 :type start: int
1054 :param start: (Optional) The first byte in a range to be downloaded.
1056 :type end: int
1057 :param end: (Optional) The last byte in a range to be downloaded.
1059 :type raw_download: bool
1060 :param raw_download:
1061 (Optional) If true, download the object without any expansion.
1063 :type if_etag_match: Union[str, Set[str]]
1064 :param if_etag_match:
1065 (Optional) See :ref:`using-if-etag-match`
1067 :type if_etag_not_match: Union[str, Set[str]]
1068 :param if_etag_not_match:
1069 (Optional) See :ref:`using-if-etag-not-match`
1071 :type if_generation_match: long
1072 :param if_generation_match:
1073 (Optional) See :ref:`using-if-generation-match`
1075 :type if_generation_not_match: long
1076 :param if_generation_not_match:
1077 (Optional) See :ref:`using-if-generation-not-match`
1079 :type if_metageneration_match: long
1080 :param if_metageneration_match:
1081 (Optional) See :ref:`using-if-metageneration-match`
1083 :type if_metageneration_not_match: long
1084 :param if_metageneration_not_match:
1085 (Optional) See :ref:`using-if-metageneration-not-match`
1087 :type timeout: float or tuple
1088 :param timeout:
1089 (Optional) The amount of time, in seconds, to wait
1090 for the server response. See: :ref:`configuring_timeouts`
1092 :type checksum: str
1093 :param checksum:
1094 (Optional) The type of checksum to compute to verify the integrity
1095 of the object. The response headers must contain a checksum of the
1096 requested type. If the headers lack an appropriate checksum (for
1097 instance in the case of transcoded or ranged downloads where the
1098 remote service does not know the correct checksum, including
1099 downloads where chunk_size is set) an INFO-level log will be
1100 emitted. Supported values are "md5", "crc32c" and None. The default
1101 is "md5".
1103 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1104 :param retry: (Optional) How to retry the RPC. A None value will disable
1105 retries. A google.api_core.retry.Retry value will enable retries,
1106 and the object will define retriable response codes and errors and
1107 configure backoff and timeout options.
1109 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1110 Retry object and activates it only if certain conditions are met.
1111 This class exists to provide safe defaults for RPC calls that are
1112 not technically safe to retry normally (due to potential data
1113 duplication or other side-effects) but become safe to retry if a
1114 condition such as if_metageneration_match is set.
1116 See the retry.py source code and docstrings in this package
1117 (google.cloud.storage.retry) for information on retry types and how
1118 to configure them.
1120 Media operations (downloads and uploads) do not support non-default
1121 predicates in a Retry object. The default will always be used. Other
1122 configuration changes for Retry objects such as delays and deadlines
1123 are respected.
1125 :raises: :class:`google.cloud.exceptions.NotFound`
1126 """
1127 client = self._require_client(client)
1129 client.download_blob_to_file(
1130 self,
1131 file_obj=file_obj,
1132 start=start,
1133 end=end,
1134 raw_download=raw_download,
1135 if_etag_match=if_etag_match,
1136 if_etag_not_match=if_etag_not_match,
1137 if_generation_match=if_generation_match,
1138 if_generation_not_match=if_generation_not_match,
1139 if_metageneration_match=if_metageneration_match,
1140 if_metageneration_not_match=if_metageneration_not_match,
1141 timeout=timeout,
1142 checksum=checksum,
1143 retry=retry,
1144 )
1146 def download_to_filename(
1147 self,
1148 filename,
1149 client=None,
1150 start=None,
1151 end=None,
1152 raw_download=False,
1153 if_etag_match=None,
1154 if_etag_not_match=None,
1155 if_generation_match=None,
1156 if_generation_not_match=None,
1157 if_metageneration_match=None,
1158 if_metageneration_not_match=None,
1159 timeout=_DEFAULT_TIMEOUT,
1160 checksum="md5",
1161 retry=DEFAULT_RETRY,
1162 ):
1163 """Download the contents of this blob into a named file.
1165 If :attr:`user_project` is set on the bucket, bills the API request
1166 to that project.
1168 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-download-encrypted-file#storage_download_encrypted_file-python)
1169 to download a file with a [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied).
1171 :type filename: str
1172 :param filename: A filename to be passed to ``open``.
1174 :type client: :class:`~google.cloud.storage.client.Client`
1175 :param client:
1176 (Optional) The client to use. If not passed, falls back to the
1177 ``client`` stored on the blob's bucket.
1179 :type start: int
1180 :param start: (Optional) The first byte in a range to be downloaded.
1182 :type end: int
1183 :param end: (Optional) The last byte in a range to be downloaded.
1185 :type raw_download: bool
1186 :param raw_download:
1187 (Optional) If true, download the object without any expansion.
1189 :type if_etag_match: Union[str, Set[str]]
1190 :param if_etag_match:
1191 (Optional) See :ref:`using-if-etag-match`
1193 :type if_etag_not_match: Union[str, Set[str]]
1194 :param if_etag_not_match:
1195 (Optional) See :ref:`using-if-etag-not-match`
1197 :type if_generation_match: long
1198 :param if_generation_match:
1199 (Optional) See :ref:`using-if-generation-match`
1201 :type if_generation_not_match: long
1202 :param if_generation_not_match:
1203 (Optional) See :ref:`using-if-generation-not-match`
1205 :type if_metageneration_match: long
1206 :param if_metageneration_match:
1207 (Optional) See :ref:`using-if-metageneration-match`
1209 :type if_metageneration_not_match: long
1210 :param if_metageneration_not_match:
1211 (Optional) See :ref:`using-if-metageneration-not-match`
1213 :type timeout: float or tuple
1214 :param timeout:
1215 (Optional) The amount of time, in seconds, to wait
1216 for the server response. See: :ref:`configuring_timeouts`
1218 :type checksum: str
1219 :param checksum:
1220 (Optional) The type of checksum to compute to verify the integrity
1221 of the object. The response headers must contain a checksum of the
1222 requested type. If the headers lack an appropriate checksum (for
1223 instance in the case of transcoded or ranged downloads where the
1224 remote service does not know the correct checksum, including
1225 downloads where chunk_size is set) an INFO-level log will be
1226 emitted. Supported values are "md5", "crc32c" and None. The default
1227 is "md5".
1229 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1230 :param retry: (Optional) How to retry the RPC. A None value will disable
1231 retries. A google.api_core.retry.Retry value will enable retries,
1232 and the object will define retriable response codes and errors and
1233 configure backoff and timeout options.
1235 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1236 Retry object and activates it only if certain conditions are met.
1237 This class exists to provide safe defaults for RPC calls that are
1238 not technically safe to retry normally (due to potential data
1239 duplication or other side-effects) but become safe to retry if a
1240 condition such as if_metageneration_match is set.
1242 See the retry.py source code and docstrings in this package
1243 (google.cloud.storage.retry) for information on retry types and how
1244 to configure them.
1246 Media operations (downloads and uploads) do not support non-default
1247 predicates in a Retry object. The default will always be used. Other
1248 configuration changes for Retry objects such as delays and deadlines
1249 are respected.
1251 :raises: :class:`google.cloud.exceptions.NotFound`
1252 """
1253 client = self._require_client(client)
1254 try:
1255 with open(filename, "wb") as file_obj:
1256 client.download_blob_to_file(
1257 self,
1258 file_obj,
1259 start=start,
1260 end=end,
1261 raw_download=raw_download,
1262 if_etag_match=if_etag_match,
1263 if_etag_not_match=if_etag_not_match,
1264 if_generation_match=if_generation_match,
1265 if_generation_not_match=if_generation_not_match,
1266 if_metageneration_match=if_metageneration_match,
1267 if_metageneration_not_match=if_metageneration_not_match,
1268 timeout=timeout,
1269 checksum=checksum,
1270 retry=retry,
1271 )
1272 except resumable_media.DataCorruption:
1273 # Delete the corrupt downloaded file.
1274 os.remove(filename)
1275 raise
1277 updated = self.updated
1278 if updated is not None:
1279 mtime = updated.timestamp()
1280 os.utime(file_obj.name, (mtime, mtime))
1282 def download_as_bytes(
1283 self,
1284 client=None,
1285 start=None,
1286 end=None,
1287 raw_download=False,
1288 if_etag_match=None,
1289 if_etag_not_match=None,
1290 if_generation_match=None,
1291 if_generation_not_match=None,
1292 if_metageneration_match=None,
1293 if_metageneration_not_match=None,
1294 timeout=_DEFAULT_TIMEOUT,
1295 checksum="md5",
1296 retry=DEFAULT_RETRY,
1297 ):
1298 """Download the contents of this blob as a bytes object.
1300 If :attr:`user_project` is set on the bucket, bills the API request
1301 to that project.
1303 :type client: :class:`~google.cloud.storage.client.Client`
1304 :param client:
1305 (Optional) The client to use. If not passed, falls back to the
1306 ``client`` stored on the blob's bucket.
1308 :type start: int
1309 :param start: (Optional) The first byte in a range to be downloaded.
1311 :type end: int
1312 :param end: (Optional) The last byte in a range to be downloaded.
1314 :type raw_download: bool
1315 :param raw_download:
1316 (Optional) If true, download the object without any expansion.
1318 :type if_etag_match: Union[str, Set[str]]
1319 :param if_etag_match:
1320 (Optional) See :ref:`using-if-etag-match`
1322 :type if_etag_not_match: Union[str, Set[str]]
1323 :param if_etag_not_match:
1324 (Optional) See :ref:`using-if-etag-not-match`
1326 :type if_generation_match: long
1327 :param if_generation_match:
1328 (Optional) See :ref:`using-if-generation-match`
1330 :type if_generation_not_match: long
1331 :param if_generation_not_match:
1332 (Optional) See :ref:`using-if-generation-not-match`
1334 :type if_metageneration_match: long
1335 :param if_metageneration_match:
1336 (Optional) See :ref:`using-if-metageneration-match`
1338 :type if_metageneration_not_match: long
1339 :param if_metageneration_not_match:
1340 (Optional) See :ref:`using-if-metageneration-not-match`
1342 :type timeout: float or tuple
1343 :param timeout:
1344 (Optional) The amount of time, in seconds, to wait
1345 for the server response. See: :ref:`configuring_timeouts`
1347 :type checksum: str
1348 :param checksum:
1349 (Optional) The type of checksum to compute to verify the integrity
1350 of the object. The response headers must contain a checksum of the
1351 requested type. If the headers lack an appropriate checksum (for
1352 instance in the case of transcoded or ranged downloads where the
1353 remote service does not know the correct checksum, including
1354 downloads where chunk_size is set) an INFO-level log will be
1355 emitted. Supported values are "md5", "crc32c" and None. The default
1356 is "md5".
1358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1359 :param retry: (Optional) How to retry the RPC. A None value will disable
1360 retries. A google.api_core.retry.Retry value will enable retries,
1361 and the object will define retriable response codes and errors and
1362 configure backoff and timeout options.
1364 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1365 Retry object and activates it only if certain conditions are met.
1366 This class exists to provide safe defaults for RPC calls that are
1367 not technically safe to retry normally (due to potential data
1368 duplication or other side-effects) but become safe to retry if a
1369 condition such as if_metageneration_match is set.
1371 See the retry.py source code and docstrings in this package
1372 (google.cloud.storage.retry) for information on retry types and how
1373 to configure them.
1375 Media operations (downloads and uploads) do not support non-default
1376 predicates in a Retry object. The default will always be used. Other
1377 configuration changes for Retry objects such as delays and deadlines
1378 are respected.
1380 :rtype: bytes
1381 :returns: The data stored in this blob.
1383 :raises: :class:`google.cloud.exceptions.NotFound`
1384 """
1385 client = self._require_client(client)
1386 string_buffer = BytesIO()
1387 client.download_blob_to_file(
1388 self,
1389 string_buffer,
1390 start=start,
1391 end=end,
1392 raw_download=raw_download,
1393 if_etag_match=if_etag_match,
1394 if_etag_not_match=if_etag_not_match,
1395 if_generation_match=if_generation_match,
1396 if_generation_not_match=if_generation_not_match,
1397 if_metageneration_match=if_metageneration_match,
1398 if_metageneration_not_match=if_metageneration_not_match,
1399 timeout=timeout,
1400 checksum=checksum,
1401 retry=retry,
1402 )
1403 return string_buffer.getvalue()
1405 def download_as_string(
1406 self,
1407 client=None,
1408 start=None,
1409 end=None,
1410 raw_download=False,
1411 if_etag_match=None,
1412 if_etag_not_match=None,
1413 if_generation_match=None,
1414 if_generation_not_match=None,
1415 if_metageneration_match=None,
1416 if_metageneration_not_match=None,
1417 timeout=_DEFAULT_TIMEOUT,
1418 retry=DEFAULT_RETRY,
1419 ):
1420 """(Deprecated) Download the contents of this blob as a bytes object.
1422 If :attr:`user_project` is set on the bucket, bills the API request
1423 to that project.
1425 .. note::
1426 Deprecated alias for :meth:`download_as_bytes`.
1428 :type client: :class:`~google.cloud.storage.client.Client`
1429 :param client:
1430 (Optional) The client to use. If not passed, falls back to the
1431 ``client`` stored on the blob's bucket.
1433 :type start: int
1434 :param start: (Optional) The first byte in a range to be downloaded.
1436 :type end: int
1437 :param end: (Optional) The last byte in a range to be downloaded.
1439 :type raw_download: bool
1440 :param raw_download:
1441 (Optional) If true, download the object without any expansion.
1443 :type if_etag_match: Union[str, Set[str]]
1444 :param if_etag_match:
1445 (Optional) See :ref:`using-if-etag-match`
1447 :type if_etag_not_match: Union[str, Set[str]]
1448 :param if_etag_not_match:
1449 (Optional) See :ref:`using-if-etag-not-match`
1451 :type if_generation_match: long
1452 :param if_generation_match:
1453 (Optional) See :ref:`using-if-generation-match`
1455 :type if_generation_not_match: long
1456 :param if_generation_not_match:
1457 (Optional) See :ref:`using-if-generation-not-match`
1459 :type if_metageneration_match: long
1460 :param if_metageneration_match:
1461 (Optional) See :ref:`using-if-metageneration-match`
1463 :type if_metageneration_not_match: long
1464 :param if_metageneration_not_match:
1465 (Optional) See :ref:`using-if-metageneration-not-match`
1467 :type timeout: float or tuple
1468 :param timeout:
1469 (Optional) The amount of time, in seconds, to wait
1470 for the server response. See: :ref:`configuring_timeouts`
1472 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1473 :param retry: (Optional) How to retry the RPC. A None value will disable
1474 retries. A google.api_core.retry.Retry value will enable retries,
1475 and the object will define retriable response codes and errors and
1476 configure backoff and timeout options.
1478 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1479 Retry object and activates it only if certain conditions are met.
1480 This class exists to provide safe defaults for RPC calls that are
1481 not technically safe to retry normally (due to potential data
1482 duplication or other side-effects) but become safe to retry if a
1483 condition such as if_metageneration_match is set.
1485 See the retry.py source code and docstrings in this package
1486 (google.cloud.storage.retry) for information on retry types and how
1487 to configure them.
1489 Media operations (downloads and uploads) do not support non-default
1490 predicates in a Retry object. The default will always be used. Other
1491 configuration changes for Retry objects such as delays and deadlines
1492 are respected.
1494 :rtype: bytes
1495 :returns: The data stored in this blob.
1497 :raises: :class:`google.cloud.exceptions.NotFound`
1498 """
1499 warnings.warn(
1500 _DOWNLOAD_AS_STRING_DEPRECATED, PendingDeprecationWarning, stacklevel=2
1501 )
1502 return self.download_as_bytes(
1503 client=client,
1504 start=start,
1505 end=end,
1506 raw_download=raw_download,
1507 if_etag_match=if_etag_match,
1508 if_etag_not_match=if_etag_not_match,
1509 if_generation_match=if_generation_match,
1510 if_generation_not_match=if_generation_not_match,
1511 if_metageneration_match=if_metageneration_match,
1512 if_metageneration_not_match=if_metageneration_not_match,
1513 timeout=timeout,
1514 retry=retry,
1515 )
1517 def download_as_text(
1518 self,
1519 client=None,
1520 start=None,
1521 end=None,
1522 raw_download=False,
1523 encoding=None,
1524 if_etag_match=None,
1525 if_etag_not_match=None,
1526 if_generation_match=None,
1527 if_generation_not_match=None,
1528 if_metageneration_match=None,
1529 if_metageneration_not_match=None,
1530 timeout=_DEFAULT_TIMEOUT,
1531 retry=DEFAULT_RETRY,
1532 ):
1533 """Download the contents of this blob as text (*not* bytes).
1535 If :attr:`user_project` is set on the bucket, bills the API request
1536 to that project.
1538 :type client: :class:`~google.cloud.storage.client.Client`
1539 :param client:
1540 (Optional) The client to use. If not passed, falls back to the
1541 ``client`` stored on the blob's bucket.
1543 :type start: int
1544 :param start: (Optional) The first byte in a range to be downloaded.
1546 :type end: int
1547 :param end: (Optional) The last byte in a range to be downloaded.
1549 :type raw_download: bool
1550 :param raw_download:
1551 (Optional) If true, download the object without any expansion.
1553 :type encoding: str
1554 :param encoding: (Optional) encoding to be used to decode the
1555 downloaded bytes. Defaults to the ``charset`` param of
1556 attr:`content_type`, or else to "utf-8".
1558 :type if_etag_match: Union[str, Set[str]]
1559 :param if_etag_match:
1560 (Optional) See :ref:`using-if-etag-match`
1562 :type if_etag_not_match: Union[str, Set[str]]
1563 :param if_etag_not_match:
1564 (Optional) See :ref:`using-if-etag-not-match`
1566 :type if_generation_match: long
1567 :param if_generation_match:
1568 (Optional) See :ref:`using-if-generation-match`
1570 :type if_generation_not_match: long
1571 :param if_generation_not_match:
1572 (Optional) See :ref:`using-if-generation-not-match`
1574 :type if_metageneration_match: long
1575 :param if_metageneration_match:
1576 (Optional) See :ref:`using-if-metageneration-match`
1578 :type if_metageneration_not_match: long
1579 :param if_metageneration_not_match:
1580 (Optional) See :ref:`using-if-metageneration-not-match`
1582 :type timeout: float or tuple
1583 :param timeout:
1584 (Optional) The amount of time, in seconds, to wait
1585 for the server response. See: :ref:`configuring_timeouts`
1587 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1588 :param retry: (Optional) How to retry the RPC. A None value will disable
1589 retries. A google.api_core.retry.Retry value will enable retries,
1590 and the object will define retriable response codes and errors and
1591 configure backoff and timeout options.
1593 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1594 Retry object and activates it only if certain conditions are met.
1595 This class exists to provide safe defaults for RPC calls that are
1596 not technically safe to retry normally (due to potential data
1597 duplication or other side-effects) but become safe to retry if a
1598 condition such as if_metageneration_match is set.
1600 See the retry.py source code and docstrings in this package
1601 (google.cloud.storage.retry) for information on retry types and how
1602 to configure them.
1604 Media operations (downloads and uploads) do not support non-default
1605 predicates in a Retry object. The default will always be used. Other
1606 configuration changes for Retry objects such as delays and deadlines
1607 are respected.
1609 :rtype: text
1610 :returns: The data stored in this blob, decoded to text.
1611 """
1612 data = self.download_as_bytes(
1613 client=client,
1614 start=start,
1615 end=end,
1616 raw_download=raw_download,
1617 if_etag_match=if_etag_match,
1618 if_etag_not_match=if_etag_not_match,
1619 if_generation_match=if_generation_match,
1620 if_generation_not_match=if_generation_not_match,
1621 if_metageneration_match=if_metageneration_match,
1622 if_metageneration_not_match=if_metageneration_not_match,
1623 timeout=timeout,
1624 retry=retry,
1625 )
1627 if encoding is not None:
1628 return data.decode(encoding)
1630 if self.content_type is not None:
1631 msg = HeaderParser().parsestr("Content-Type: " + self.content_type)
1632 params = dict(msg.get_params()[1:])
1633 if "charset" in params:
1634 return data.decode(params["charset"])
1636 return data.decode("utf-8")
1638 def _get_content_type(self, content_type, filename=None):
1639 """Determine the content type from the current object.
1641 The return value will be determined in order of precedence:
1643 - The value passed in to this method (if not :data:`None`)
1644 - The value stored on the current blob
1645 - The default value ('application/octet-stream')
1647 :type content_type: str
1648 :param content_type: (Optional) Type of content.
1650 :type filename: str
1651 :param filename:
1652 (Optional) The name of the file where the content is stored.
1654 :rtype: str
1655 :returns: Type of content gathered from the object.
1656 """
1657 if content_type is None:
1658 content_type = self.content_type
1660 if content_type is None and filename is not None:
1661 content_type, _ = mimetypes.guess_type(filename)
1663 if content_type is None:
1664 content_type = _DEFAULT_CONTENT_TYPE
1666 return content_type
1668 def _get_writable_metadata(self):
1669 """Get the object / blob metadata which is writable.
1671 This is intended to be used when creating a new object / blob.
1673 See the [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects)
1674 for more information, the fields marked as writable are:
1676 * ``acl``
1677 * ``cacheControl``
1678 * ``contentDisposition``
1679 * ``contentEncoding``
1680 * ``contentLanguage``
1681 * ``contentType``
1682 * ``crc32c``
1683 * ``customTime``
1684 * ``md5Hash``
1685 * ``metadata``
1686 * ``name``
1687 * ``storageClass``
1689 For now, we don't support ``acl``, access control lists should be
1690 managed directly through :class:`ObjectACL` methods.
1691 """
1692 # NOTE: This assumes `self.name` is unicode.
1693 object_metadata = {"name": self.name}
1694 for key in self._changes:
1695 if key in _WRITABLE_FIELDS:
1696 object_metadata[key] = self._properties[key]
1698 return object_metadata
1700 def _get_upload_arguments(self, client, content_type):
1701 """Get required arguments for performing an upload.
1703 The content type returned will be determined in order of precedence:
1705 - The value passed in to this method (if not :data:`None`)
1706 - The value stored on the current blob
1707 - The default value ('application/octet-stream')
1709 :type content_type: str
1710 :param content_type: Type of content being uploaded (or :data:`None`).
1712 :rtype: tuple
1713 :returns: A triple of
1715 * A header dictionary
1716 * An object metadata dictionary
1717 * The ``content_type`` as a string (according to precedence)
1718 """
1719 content_type = self._get_content_type(content_type)
1720 headers = {
1721 **_get_default_headers(client._connection.user_agent, content_type),
1722 **_get_encryption_headers(self._encryption_key),
1723 }
1724 object_metadata = self._get_writable_metadata()
1725 return headers, object_metadata, content_type
1727 def _do_multipart_upload(
1728 self,
1729 client,
1730 stream,
1731 content_type,
1732 size,
1733 num_retries,
1734 predefined_acl,
1735 if_generation_match,
1736 if_generation_not_match,
1737 if_metageneration_match,
1738 if_metageneration_not_match,
1739 timeout=_DEFAULT_TIMEOUT,
1740 checksum=None,
1741 retry=None,
1742 ):
1743 """Perform a multipart upload.
1745 The content type of the upload will be determined in order
1746 of precedence:
1748 - The value passed in to this method (if not :data:`None`)
1749 - The value stored on the current blob
1750 - The default value ('application/octet-stream')
1752 :type client: :class:`~google.cloud.storage.client.Client`
1753 :param client:
1754 (Optional) The client to use. If not passed, falls back to the
1755 ``client`` stored on the blob's bucket.
1757 :type stream: IO[bytes]
1758 :param stream: A bytes IO object open for reading.
1760 :type content_type: str
1761 :param content_type: Type of content being uploaded (or :data:`None`).
1763 :type size: int
1764 :param size:
1765 The number of bytes to be uploaded (which will be read from
1766 ``stream``). If not provided, the upload will be concluded once
1767 ``stream`` is exhausted (or :data:`None`).
1769 :type num_retries: int
1770 :param num_retries:
1771 Number of upload retries. By default, only uploads with
1772 if_generation_match set will be retried, as uploads without the
1773 argument are not guaranteed to be idempotent. Setting num_retries
1774 will override this default behavior and guarantee retries even when
1775 if_generation_match is not set. (Deprecated: This argument
1776 will be removed in a future release.)
1778 :type predefined_acl: str
1779 :param predefined_acl: (Optional) Predefined access control list
1781 :type if_generation_match: long
1782 :param if_generation_match:
1783 (Optional) See :ref:`using-if-generation-match`
1785 :type if_generation_not_match: long
1786 :param if_generation_not_match:
1787 (Optional) See :ref:`using-if-generation-not-match`
1789 :type if_metageneration_match: long
1790 :param if_metageneration_match:
1791 (Optional) See :ref:`using-if-metageneration-match`
1793 :type if_metageneration_not_match: long
1794 :param if_metageneration_not_match:
1795 (Optional) See :ref:`using-if-metageneration-not-match`
1797 :type timeout: float or tuple
1798 :param timeout:
1799 (Optional) The amount of time, in seconds, to wait
1800 for the server response. See: :ref:`configuring_timeouts`
1802 :type checksum: str
1803 :param checksum:
1804 (Optional) The type of checksum to compute to verify
1805 the integrity of the object. The request metadata will be amended
1806 to include the computed value. Using this option will override a
1807 manually-set checksum value. Supported values are "md5",
1808 "crc32c" and None. The default is None.
1810 :type retry: google.api_core.retry.Retry
1811 :param retry: (Optional) How to retry the RPC. A None value will disable
1812 retries. A google.api_core.retry.Retry value will enable retries,
1813 and the object will configure backoff and timeout options. Custom
1814 predicates (customizable error codes) are not supported for media
1815 operations such as this one.
1817 This private method does not accept ConditionalRetryPolicy values
1818 because the information necessary to evaluate the policy is instead
1819 evaluated in blob._do_upload().
1821 See the retry.py source code and docstrings in this package
1822 (google.cloud.storage.retry) for information on retry types and how
1823 to configure them.
1825 :rtype: :class:`~requests.Response`
1826 :returns: The "200 OK" response object returned after the multipart
1827 upload request.
1828 :raises: :exc:`ValueError` if ``size`` is not :data:`None` but the
1829 ``stream`` has fewer than ``size`` bytes remaining.
1830 """
1831 if size is None:
1832 data = stream.read()
1833 else:
1834 data = stream.read(size)
1835 if len(data) < size:
1836 msg = _READ_LESS_THAN_SIZE.format(size, len(data))
1837 raise ValueError(msg)
1839 client = self._require_client(client)
1840 transport = self._get_transport(client)
1841 if "metadata" in self._properties and "metadata" not in self._changes:
1842 self._changes.add("metadata")
1843 info = self._get_upload_arguments(client, content_type)
1844 headers, object_metadata, content_type = info
1846 hostname = _get_host_name(client._connection)
1847 base_url = _MULTIPART_URL_TEMPLATE.format(
1848 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
1849 )
1850 name_value_pairs = []
1852 if self.user_project is not None:
1853 name_value_pairs.append(("userProject", self.user_project))
1855 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object
1856 # at rest, object resource metadata will store the version of the Key Management
1857 # Service cryptographic material. If a Blob instance with KMS Key metadata set is
1858 # used to upload a new version of the object then the existing kmsKeyName version
1859 # value can't be used in the upload request and the client instead ignores it.
1860 if (
1861 self.kms_key_name is not None
1862 and "cryptoKeyVersions" not in self.kms_key_name
1863 ):
1864 name_value_pairs.append(("kmsKeyName", self.kms_key_name))
1866 if predefined_acl is not None:
1867 name_value_pairs.append(("predefinedAcl", predefined_acl))
1869 if if_generation_match is not None:
1870 name_value_pairs.append(("ifGenerationMatch", if_generation_match))
1872 if if_generation_not_match is not None:
1873 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match))
1875 if if_metageneration_match is not None:
1876 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match))
1878 if if_metageneration_not_match is not None:
1879 name_value_pairs.append(
1880 ("ifMetaGenerationNotMatch", if_metageneration_not_match)
1881 )
1883 upload_url = _add_query_parameters(base_url, name_value_pairs)
1884 upload = MultipartUpload(upload_url, headers=headers, checksum=checksum)
1886 upload._retry_strategy = _api_core_retry_to_resumable_media_retry(
1887 retry, num_retries
1888 )
1890 response = upload.transmit(
1891 transport, data, object_metadata, content_type, timeout=timeout
1892 )
1894 return response
1896 def _initiate_resumable_upload(
1897 self,
1898 client,
1899 stream,
1900 content_type,
1901 size,
1902 num_retries,
1903 predefined_acl=None,
1904 extra_headers=None,
1905 chunk_size=None,
1906 if_generation_match=None,
1907 if_generation_not_match=None,
1908 if_metageneration_match=None,
1909 if_metageneration_not_match=None,
1910 timeout=_DEFAULT_TIMEOUT,
1911 checksum=None,
1912 retry=None,
1913 ):
1914 """Initiate a resumable upload.
1916 The content type of the upload will be determined in order
1917 of precedence:
1919 - The value passed in to this method (if not :data:`None`)
1920 - The value stored on the current blob
1921 - The default value ('application/octet-stream')
1923 :type client: :class:`~google.cloud.storage.client.Client`
1924 :param client:
1925 (Optional) The client to use. If not passed, falls back to the
1926 ``client`` stored on the blob's bucket.
1928 :type stream: IO[bytes]
1929 :param stream: A bytes IO object open for reading.
1931 :type content_type: str
1932 :param content_type: Type of content being uploaded (or :data:`None`).
1934 :type size: int
1935 :param size:
1936 The number of bytes to be uploaded (which will be read from
1937 ``stream``). If not provided, the upload will be concluded once
1938 ``stream`` is exhausted (or :data:`None`).
1940 :type predefined_acl: str
1941 :param predefined_acl: (Optional) Predefined access control list
1943 :type num_retries: int
1944 :param num_retries:
1945 Number of upload retries. By default, only uploads with
1946 if_generation_match set will be retried, as uploads without the
1947 argument are not guaranteed to be idempotent. Setting num_retries
1948 will override this default behavior and guarantee retries even when
1949 if_generation_match is not set. (Deprecated: This argument
1950 will be removed in a future release.)
1952 :type extra_headers: dict
1953 :param extra_headers:
1954 (Optional) Extra headers to add to standard headers.
1956 :type chunk_size: int
1957 :param chunk_size:
1958 (Optional) Chunk size to use when creating a
1959 :class:`~google.resumable_media.requests.ResumableUpload`.
1960 If not passed, will fall back to the chunk size on the
1961 current blob, if the chunk size of a current blob is also
1962 `None`, will set the default value.
1963 The default value of ``chunk_size`` is 100 MB.
1965 :type if_generation_match: long
1966 :param if_generation_match:
1967 (Optional) See :ref:`using-if-generation-match`
1969 :type if_generation_not_match: long
1970 :param if_generation_not_match:
1971 (Optional) See :ref:`using-if-generation-not-match`
1973 :type if_metageneration_match: long
1974 :param if_metageneration_match:
1975 (Optional) See :ref:`using-if-metageneration-match`
1977 :type if_metageneration_not_match: long
1978 :param if_metageneration_not_match:
1979 (Optional) See :ref:`using-if-metageneration-not-match`
1981 :type timeout: float or tuple
1982 :param timeout:
1983 (Optional) The amount of time, in seconds, to wait
1984 for the server response. See: :ref:`configuring_timeouts`
1986 :type checksum: str
1987 :param checksum:
1988 (Optional) The type of checksum to compute to verify
1989 the integrity of the object. After the upload is complete, the
1990 server-computed checksum of the resulting object will be checked
1991 and google.resumable_media.common.DataCorruption will be raised on
1992 a mismatch. On a validation failure, the client will attempt to
1993 delete the uploaded object automatically. Supported values
1994 are "md5", "crc32c" and None. The default is None.
1996 :type retry: google.api_core.retry.Retry
1997 :param retry: (Optional) How to retry the RPC. A None value will disable
1998 retries. A google.api_core.retry.Retry value will enable retries,
1999 and the object will configure backoff and timeout options. Custom
2000 predicates (customizable error codes) are not supported for media
2001 operations such as this one.
2003 This private method does not accept ConditionalRetryPolicy values
2004 because the information necessary to evaluate the policy is instead
2005 evaluated in blob._do_upload().
2007 See the retry.py source code and docstrings in this package
2008 (google.cloud.storage.retry) for information on retry types and how
2009 to configure them.
2011 :rtype: tuple
2012 :returns:
2013 Pair of
2015 * The :class:`~google.resumable_media.requests.ResumableUpload`
2016 that was created
2017 * The ``transport`` used to initiate the upload.
2018 """
2019 client = self._require_client(client)
2020 if chunk_size is None:
2021 chunk_size = self.chunk_size
2022 if chunk_size is None:
2023 chunk_size = _DEFAULT_CHUNKSIZE
2025 transport = self._get_transport(client)
2026 if "metadata" in self._properties and "metadata" not in self._changes:
2027 self._changes.add("metadata")
2028 info = self._get_upload_arguments(client, content_type)
2029 headers, object_metadata, content_type = info
2030 if extra_headers is not None:
2031 headers.update(extra_headers)
2033 hostname = _get_host_name(client._connection)
2034 base_url = _RESUMABLE_URL_TEMPLATE.format(
2035 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
2036 )
2037 name_value_pairs = []
2039 if self.user_project is not None:
2040 name_value_pairs.append(("userProject", self.user_project))
2042 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object
2043 # at rest, object resource metadata will store the version of the Key Management
2044 # Service cryptographic material. If a Blob instance with KMS Key metadata set is
2045 # used to upload a new version of the object then the existing kmsKeyName version
2046 # value can't be used in the upload request and the client instead ignores it.
2047 if (
2048 self.kms_key_name is not None
2049 and "cryptoKeyVersions" not in self.kms_key_name
2050 ):
2051 name_value_pairs.append(("kmsKeyName", self.kms_key_name))
2053 if predefined_acl is not None:
2054 name_value_pairs.append(("predefinedAcl", predefined_acl))
2056 if if_generation_match is not None:
2057 name_value_pairs.append(("ifGenerationMatch", if_generation_match))
2059 if if_generation_not_match is not None:
2060 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match))
2062 if if_metageneration_match is not None:
2063 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match))
2065 if if_metageneration_not_match is not None:
2066 name_value_pairs.append(
2067 ("ifMetaGenerationNotMatch", if_metageneration_not_match)
2068 )
2070 upload_url = _add_query_parameters(base_url, name_value_pairs)
2071 upload = ResumableUpload(
2072 upload_url, chunk_size, headers=headers, checksum=checksum
2073 )
2075 upload._retry_strategy = _api_core_retry_to_resumable_media_retry(
2076 retry, num_retries
2077 )
2079 upload.initiate(
2080 transport,
2081 stream,
2082 object_metadata,
2083 content_type,
2084 total_bytes=size,
2085 stream_final=False,
2086 timeout=timeout,
2087 )
2089 return upload, transport
2091 def _do_resumable_upload(
2092 self,
2093 client,
2094 stream,
2095 content_type,
2096 size,
2097 num_retries,
2098 predefined_acl,
2099 if_generation_match,
2100 if_generation_not_match,
2101 if_metageneration_match,
2102 if_metageneration_not_match,
2103 timeout=_DEFAULT_TIMEOUT,
2104 checksum=None,
2105 retry=None,
2106 ):
2107 """Perform a resumable upload.
2109 Assumes ``chunk_size`` is not :data:`None` on the current blob.
2110 The default value of ``chunk_size`` is 100 MB.
2112 The content type of the upload will be determined in order
2113 of precedence:
2115 - The value passed in to this method (if not :data:`None`)
2116 - The value stored on the current blob
2117 - The default value ('application/octet-stream')
2119 :type client: :class:`~google.cloud.storage.client.Client`
2120 :param client:
2121 (Optional) The client to use. If not passed, falls back to the
2122 ``client`` stored on the blob's bucket.
2124 :type stream: IO[bytes]
2125 :param stream: A bytes IO object open for reading.
2127 :type content_type: str
2128 :param content_type: Type of content being uploaded (or :data:`None`).
2130 :type size: int
2131 :param size:
2132 The number of bytes to be uploaded (which will be read from
2133 ``stream``). If not provided, the upload will be concluded once
2134 ``stream`` is exhausted (or :data:`None`).
2136 :type num_retries: int
2137 :param num_retries:
2138 Number of upload retries. By default, only uploads with
2139 if_generation_match set will be retried, as uploads without the
2140 argument are not guaranteed to be idempotent. Setting num_retries
2141 will override this default behavior and guarantee retries even when
2142 if_generation_match is not set. (Deprecated: This argument
2143 will be removed in a future release.)
2145 :type predefined_acl: str
2146 :param predefined_acl: (Optional) Predefined access control list
2148 :type if_generation_match: long
2149 :param if_generation_match:
2150 (Optional) See :ref:`using-if-generation-match`
2152 :type if_generation_not_match: long
2153 :param if_generation_not_match:
2154 (Optional) See :ref:`using-if-generation-not-match`
2156 :type if_metageneration_match: long
2157 :param if_metageneration_match:
2158 (Optional) See :ref:`using-if-metageneration-match`
2160 :type if_metageneration_not_match: long
2161 :param if_metageneration_not_match:
2162 (Optional) See :ref:`using-if-metageneration-not-match`
2164 :type timeout: float or tuple
2165 :param timeout:
2166 (Optional) The amount of time, in seconds, to wait
2167 for the server response. See: :ref:`configuring_timeouts`
2169 :type checksum: str
2170 :param checksum:
2171 (Optional) The type of checksum to compute to verify
2172 the integrity of the object. After the upload is complete, the
2173 server-computed checksum of the resulting object will be checked
2174 and google.resumable_media.common.DataCorruption will be raised on
2175 a mismatch. On a validation failure, the client will attempt to
2176 delete the uploaded object automatically. Supported values
2177 are "md5", "crc32c" and None. The default is None.
2179 :type retry: google.api_core.retry.Retry
2180 :param retry: (Optional) How to retry the RPC. A None value will disable
2181 retries. A google.api_core.retry.Retry value will enable retries,
2182 and the object will configure backoff and timeout options. Custom
2183 predicates (customizable error codes) are not supported for media
2184 operations such as this one.
2186 This private method does not accept ConditionalRetryPolicy values
2187 because the information necessary to evaluate the policy is instead
2188 evaluated in blob._do_upload().
2190 See the retry.py source code and docstrings in this package
2191 (google.cloud.storage.retry) for information on retry types and how
2192 to configure them.
2194 :rtype: :class:`~requests.Response`
2195 :returns: The "200 OK" response object returned after the final chunk
2196 is uploaded.
2197 """
2198 upload, transport = self._initiate_resumable_upload(
2199 client,
2200 stream,
2201 content_type,
2202 size,
2203 num_retries,
2204 predefined_acl=predefined_acl,
2205 if_generation_match=if_generation_match,
2206 if_generation_not_match=if_generation_not_match,
2207 if_metageneration_match=if_metageneration_match,
2208 if_metageneration_not_match=if_metageneration_not_match,
2209 timeout=timeout,
2210 checksum=checksum,
2211 retry=retry,
2212 )
2213 while not upload.finished:
2214 try:
2215 response = upload.transmit_next_chunk(transport, timeout=timeout)
2216 except resumable_media.DataCorruption:
2217 # Attempt to delete the corrupted object.
2218 self.delete()
2219 raise
2220 return response
2222 def _do_upload(
2223 self,
2224 client,
2225 stream,
2226 content_type,
2227 size,
2228 num_retries,
2229 predefined_acl,
2230 if_generation_match,
2231 if_generation_not_match,
2232 if_metageneration_match,
2233 if_metageneration_not_match,
2234 timeout=_DEFAULT_TIMEOUT,
2235 checksum=None,
2236 retry=None,
2237 ):
2238 """Determine an upload strategy and then perform the upload.
2240 If the size of the data to be uploaded exceeds 8 MB a resumable media
2241 request will be used, otherwise the content and the metadata will be
2242 uploaded in a single multipart upload request.
2244 The content type of the upload will be determined in order
2245 of precedence:
2247 - The value passed in to this method (if not :data:`None`)
2248 - The value stored on the current blob
2249 - The default value ('application/octet-stream')
2251 :type client: :class:`~google.cloud.storage.client.Client`
2252 :param client:
2253 (Optional) The client to use. If not passed, falls back to the
2254 ``client`` stored on the blob's bucket.
2256 :type stream: IO[bytes]
2257 :param stream: A bytes IO object open for reading.
2259 :type content_type: str
2260 :param content_type: Type of content being uploaded (or :data:`None`).
2262 :type size: int
2263 :param size:
2264 The number of bytes to be uploaded (which will be read from
2265 ``stream``). If not provided, the upload will be concluded once
2266 ``stream`` is exhausted (or :data:`None`).
2268 :type num_retries: int
2269 :param num_retries:
2270 Number of upload retries. By default, only uploads with
2271 if_generation_match set will be retried, as uploads without the
2272 argument are not guaranteed to be idempotent. Setting num_retries
2273 will override this default behavior and guarantee retries even when
2274 if_generation_match is not set. (Deprecated: This argument
2275 will be removed in a future release.)
2277 :type predefined_acl: str
2278 :param predefined_acl: (Optional) Predefined access control list
2280 :type if_generation_match: long
2281 :param if_generation_match:
2282 (Optional) See :ref:`using-if-generation-match`
2284 :type if_generation_not_match: long
2285 :param if_generation_not_match:
2286 (Optional) See :ref:`using-if-generation-not-match`
2288 :type if_metageneration_match: long
2289 :param if_metageneration_match:
2290 (Optional) See :ref:`using-if-metageneration-match`
2292 :type if_metageneration_not_match: long
2293 :param if_metageneration_not_match:
2294 (Optional) See :ref:`using-if-metageneration-not-match`
2296 :type timeout: float or tuple
2297 :param timeout:
2298 (Optional) The amount of time, in seconds, to wait
2299 for the server response. See: :ref:`configuring_timeouts`
2301 :type checksum: str
2302 :param checksum:
2303 (Optional) The type of checksum to compute to verify
2304 the integrity of the object. If the upload is completed in a single
2305 request, the checksum will be entirely precomputed and the remote
2306 server will handle verification and error handling. If the upload
2307 is too large and must be transmitted in multiple requests, the
2308 checksum will be incrementally computed and the client will handle
2309 verification and error handling, raising
2310 google.resumable_media.common.DataCorruption on a mismatch and
2311 attempting to delete the corrupted file. Supported values are
2312 "md5", "crc32c" and None. The default is None.
2314 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2315 :param retry: (Optional) How to retry the RPC. A None value will disable
2316 retries. A google.api_core.retry.Retry value will enable retries,
2317 and the object will define retriable response codes and errors and
2318 configure backoff and timeout options.
2320 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2321 Retry object and activates it only if certain conditions are met.
2322 This class exists to provide safe defaults for RPC calls that are
2323 not technically safe to retry normally (due to potential data
2324 duplication or other side-effects) but become safe to retry if a
2325 condition such as if_generation_match is set.
2327 See the retry.py source code and docstrings in this package
2328 (google.cloud.storage.retry) for information on retry types and how
2329 to configure them.
2331 Media operations (downloads and uploads) do not support non-default
2332 predicates in a Retry object. The default will always be used. Other
2333 configuration changes for Retry objects such as delays and deadlines
2334 are respected.
2336 :rtype: dict
2337 :returns: The parsed JSON from the "200 OK" response. This will be the
2338 **only** response in the multipart case and it will be the
2339 **final** response in the resumable case.
2340 """
2342 # Handle ConditionalRetryPolicy.
2343 if isinstance(retry, ConditionalRetryPolicy):
2344 # Conditional retries are designed for non-media calls, which change
2345 # arguments into query_params dictionaries. Media operations work
2346 # differently, so here we make a "fake" query_params to feed to the
2347 # ConditionalRetryPolicy.
2348 query_params = {
2349 "ifGenerationMatch": if_generation_match,
2350 "ifMetagenerationMatch": if_metageneration_match,
2351 }
2352 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)
2354 if size is not None and size <= _MAX_MULTIPART_SIZE:
2355 response = self._do_multipart_upload(
2356 client,
2357 stream,
2358 content_type,
2359 size,
2360 num_retries,
2361 predefined_acl,
2362 if_generation_match,
2363 if_generation_not_match,
2364 if_metageneration_match,
2365 if_metageneration_not_match,
2366 timeout=timeout,
2367 checksum=checksum,
2368 retry=retry,
2369 )
2370 else:
2371 response = self._do_resumable_upload(
2372 client,
2373 stream,
2374 content_type,
2375 size,
2376 num_retries,
2377 predefined_acl,
2378 if_generation_match,
2379 if_generation_not_match,
2380 if_metageneration_match,
2381 if_metageneration_not_match,
2382 timeout=timeout,
2383 checksum=checksum,
2384 retry=retry,
2385 )
2387 return response.json()
2389 def upload_from_file(
2390 self,
2391 file_obj,
2392 rewind=False,
2393 size=None,
2394 content_type=None,
2395 num_retries=None,
2396 client=None,
2397 predefined_acl=None,
2398 if_generation_match=None,
2399 if_generation_not_match=None,
2400 if_metageneration_match=None,
2401 if_metageneration_not_match=None,
2402 timeout=_DEFAULT_TIMEOUT,
2403 checksum=None,
2404 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
2405 ):
2406 """Upload the contents of this blob from a file-like object.
2408 The content type of the upload will be determined in order
2409 of precedence:
2411 - The value passed in to this method (if not :data:`None`)
2412 - The value stored on the current blob
2413 - The default value ('application/octet-stream')
2415 .. note::
2416 The effect of uploading to an existing blob depends on the
2417 "versioning" and "lifecycle" policies defined on the blob's
2418 bucket. In the absence of those policies, upload will
2419 overwrite any existing contents.
2421 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)
2422 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)
2423 API documents for details.
2425 If the size of the data to be uploaded exceeds 8 MB a resumable media
2426 request will be used, otherwise the content and the metadata will be
2427 uploaded in a single multipart upload request.
2429 For more fine-grained over the upload process, check out
2430 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html).
2432 If :attr:`user_project` is set on the bucket, bills the API request
2433 to that project.
2435 :type file_obj: file
2436 :param file_obj: A file handle opened in binary mode for reading.
2438 :type rewind: bool
2439 :param rewind:
2440 If True, seek to the beginning of the file handle before writing
2441 the file to Cloud Storage.
2443 :type size: int
2444 :param size:
2445 The number of bytes to be uploaded (which will be read from
2446 ``file_obj``). If not provided, the upload will be concluded once
2447 ``file_obj`` is exhausted.
2449 :type content_type: str
2450 :param content_type: (Optional) Type of content being uploaded.
2452 :type num_retries: int
2453 :param num_retries:
2454 Number of upload retries. By default, only uploads with
2455 if_generation_match set will be retried, as uploads without the
2456 argument are not guaranteed to be idempotent. Setting num_retries
2457 will override this default behavior and guarantee retries even when
2458 if_generation_match is not set. (Deprecated: This argument
2459 will be removed in a future release.)
2461 :type client: :class:`~google.cloud.storage.client.Client`
2462 :param client:
2463 (Optional) The client to use. If not passed, falls back to the
2464 ``client`` stored on the blob's bucket.
2466 :type predefined_acl: str
2467 :param predefined_acl: (Optional) Predefined access control list
2469 :type if_generation_match: long
2470 :param if_generation_match:
2471 (Optional) See :ref:`using-if-generation-match`
2473 :type if_generation_not_match: long
2474 :param if_generation_not_match:
2475 (Optional) See :ref:`using-if-generation-not-match`
2477 :type if_metageneration_match: long
2478 :param if_metageneration_match:
2479 (Optional) See :ref:`using-if-metageneration-match`
2481 :type if_metageneration_not_match: long
2482 :param if_metageneration_not_match:
2483 (Optional) See :ref:`using-if-metageneration-not-match`
2485 :type timeout: float or tuple
2486 :param timeout:
2487 (Optional) The amount of time, in seconds, to wait
2488 for the server response. See: :ref:`configuring_timeouts`
2490 :type checksum: str
2491 :param checksum:
2492 (Optional) The type of checksum to compute to verify
2493 the integrity of the object. If the upload is completed in a single
2494 request, the checksum will be entirely precomputed and the remote
2495 server will handle verification and error handling. If the upload
2496 is too large and must be transmitted in multiple requests, the
2497 checksum will be incrementally computed and the client will handle
2498 verification and error handling, raising
2499 google.resumable_media.common.DataCorruption on a mismatch and
2500 attempting to delete the corrupted file. Supported values are
2501 "md5", "crc32c" and None. The default is None.
2503 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2504 :param retry: (Optional) How to retry the RPC. A None value will disable
2505 retries. A google.api_core.retry.Retry value will enable retries,
2506 and the object will define retriable response codes and errors and
2507 configure backoff and timeout options.
2509 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2510 Retry object and activates it only if certain conditions are met.
2511 This class exists to provide safe defaults for RPC calls that are
2512 not technically safe to retry normally (due to potential data
2513 duplication or other side-effects) but become safe to retry if a
2514 condition such as if_generation_match is set.
2516 See the retry.py source code and docstrings in this package
2517 (google.cloud.storage.retry) for information on retry types and how
2518 to configure them.
2520 Media operations (downloads and uploads) do not support non-default
2521 predicates in a Retry object. The default will always be used. Other
2522 configuration changes for Retry objects such as delays and deadlines
2523 are respected.
2525 :raises: :class:`~google.cloud.exceptions.GoogleCloudError`
2526 if the upload response returns an error status.
2527 """
2528 if num_retries is not None:
2529 warnings.warn(_NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2)
2530 # num_retries and retry are mutually exclusive. If num_retries is
2531 # set and retry is exactly the default, then nullify retry for
2532 # backwards compatibility.
2533 if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED:
2534 retry = None
2536 _maybe_rewind(file_obj, rewind=rewind)
2537 predefined_acl = ACL.validate_predefined(predefined_acl)
2539 try:
2540 created_json = self._do_upload(
2541 client,
2542 file_obj,
2543 content_type,
2544 size,
2545 num_retries,
2546 predefined_acl,
2547 if_generation_match,
2548 if_generation_not_match,
2549 if_metageneration_match,
2550 if_metageneration_not_match,
2551 timeout=timeout,
2552 checksum=checksum,
2553 retry=retry,
2554 )
2555 self._set_properties(created_json)
2556 except resumable_media.InvalidResponse as exc:
2557 _raise_from_invalid_response(exc)
2559 def upload_from_filename(
2560 self,
2561 filename,
2562 content_type=None,
2563 num_retries=None,
2564 client=None,
2565 predefined_acl=None,
2566 if_generation_match=None,
2567 if_generation_not_match=None,
2568 if_metageneration_match=None,
2569 if_metageneration_not_match=None,
2570 timeout=_DEFAULT_TIMEOUT,
2571 checksum=None,
2572 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
2573 ):
2574 """Upload this blob's contents from the content of a named file.
2576 The content type of the upload will be determined in order
2577 of precedence:
2579 - The value passed in to this method (if not :data:`None`)
2580 - The value stored on the current blob
2581 - The value given by ``mimetypes.guess_type``
2582 - The default value ('application/octet-stream')
2584 .. note::
2585 The effect of uploading to an existing blob depends on the
2586 "versioning" and "lifecycle" policies defined on the blob's
2587 bucket. In the absence of those policies, upload will
2588 overwrite any existing contents.
2590 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)
2591 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)
2592 API documents for details.
2594 If :attr:`user_project` is set on the bucket, bills the API request
2595 to that project.
2597 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-upload-encrypted-file#storage_upload_encrypted_file-python)
2598 to upload a file with a
2599 [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied).
2601 :type filename: str
2602 :param filename: The path to the file.
2604 :type content_type: str
2605 :param content_type: (Optional) Type of content being uploaded.
2607 :type client: :class:`~google.cloud.storage.client.Client`
2608 :param client:
2609 (Optional) The client to use. If not passed, falls back to the
2610 ``client`` stored on the blob's bucket.
2612 :type num_retries: int
2613 :param num_retries:
2614 Number of upload retries. By default, only uploads with
2615 if_generation_match set will be retried, as uploads without the
2616 argument are not guaranteed to be idempotent. Setting num_retries
2617 will override this default behavior and guarantee retries even when
2618 if_generation_match is not set. (Deprecated: This argument
2619 will be removed in a future release.)
2621 :type predefined_acl: str
2622 :param predefined_acl: (Optional) Predefined access control list
2624 :type if_generation_match: long
2625 :param if_generation_match:
2626 (Optional) See :ref:`using-if-generation-match`
2628 :type if_generation_not_match: long
2629 :param if_generation_not_match:
2630 (Optional) See :ref:`using-if-generation-not-match`
2632 :type if_metageneration_match: long
2633 :param if_metageneration_match:
2634 (Optional) See :ref:`using-if-metageneration-match`
2636 :type if_metageneration_not_match: long
2637 :param if_metageneration_not_match:
2638 (Optional) See :ref:`using-if-metageneration-not-match`
2640 :type timeout: float or tuple
2641 :param timeout:
2642 (Optional) The amount of time, in seconds, to wait
2643 for the server response. See: :ref:`configuring_timeouts`
2645 :type checksum: str
2646 :param checksum:
2647 (Optional) The type of checksum to compute to verify
2648 the integrity of the object. If the upload is completed in a single
2649 request, the checksum will be entirely precomputed and the remote
2650 server will handle verification and error handling. If the upload
2651 is too large and must be transmitted in multiple requests, the
2652 checksum will be incrementally computed and the client will handle
2653 verification and error handling, raising
2654 google.resumable_media.common.DataCorruption on a mismatch and
2655 attempting to delete the corrupted file. Supported values are
2656 "md5", "crc32c" and None. The default is None.
2658 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2659 :param retry: (Optional) How to retry the RPC. A None value will disable
2660 retries. A google.api_core.retry.Retry value will enable retries,
2661 and the object will define retriable response codes and errors and
2662 configure backoff and timeout options.
2664 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2665 Retry object and activates it only if certain conditions are met.
2666 This class exists to provide safe defaults for RPC calls that are
2667 not technically safe to retry normally (due to potential data
2668 duplication or other side-effects) but become safe to retry if a
2669 condition such as if_generation_match is set.
2671 See the retry.py source code and docstrings in this package
2672 (google.cloud.storage.retry) for information on retry types and how
2673 to configure them.
2675 Media operations (downloads and uploads) do not support non-default
2676 predicates in a Retry object. The default will always be used. Other
2677 configuration changes for Retry objects such as delays and deadlines
2678 are respected.
2679 """
2680 content_type = self._get_content_type(content_type, filename=filename)
2682 with open(filename, "rb") as file_obj:
2683 total_bytes = os.fstat(file_obj.fileno()).st_size
2684 self.upload_from_file(
2685 file_obj,
2686 content_type=content_type,
2687 num_retries=num_retries,
2688 client=client,
2689 size=total_bytes,
2690 predefined_acl=predefined_acl,
2691 if_generation_match=if_generation_match,
2692 if_generation_not_match=if_generation_not_match,
2693 if_metageneration_match=if_metageneration_match,
2694 if_metageneration_not_match=if_metageneration_not_match,
2695 timeout=timeout,
2696 checksum=checksum,
2697 retry=retry,
2698 )
2700 def upload_from_string(
2701 self,
2702 data,
2703 content_type="text/plain",
2704 num_retries=None,
2705 client=None,
2706 predefined_acl=None,
2707 if_generation_match=None,
2708 if_generation_not_match=None,
2709 if_metageneration_match=None,
2710 if_metageneration_not_match=None,
2711 timeout=_DEFAULT_TIMEOUT,
2712 checksum=None,
2713 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
2714 ):
2715 """Upload contents of this blob from the provided string.
2717 .. note::
2718 The effect of uploading to an existing blob depends on the
2719 "versioning" and "lifecycle" policies defined on the blob's
2720 bucket. In the absence of those policies, upload will
2721 overwrite any existing contents.
2723 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)
2724 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)
2725 API documents for details.
2727 If :attr:`user_project` is set on the bucket, bills the API request
2728 to that project.
2730 :type data: bytes or str
2731 :param data:
2732 The data to store in this blob. If the value is text, it will be
2733 encoded as UTF-8.
2735 :type content_type: str
2736 :param content_type:
2737 (Optional) Type of content being uploaded. Defaults to
2738 ``'text/plain'``.
2740 :type num_retries: int
2741 :param num_retries:
2742 Number of upload retries. By default, only uploads with
2743 if_generation_match set will be retried, as uploads without the
2744 argument are not guaranteed to be idempotent. Setting num_retries
2745 will override this default behavior and guarantee retries even when
2746 if_generation_match is not set. (Deprecated: This argument
2747 will be removed in a future release.)
2749 :type client: :class:`~google.cloud.storage.client.Client`
2750 :param client:
2751 (Optional) The client to use. If not passed, falls back to the
2752 ``client`` stored on the blob's bucket.
2754 :type predefined_acl: str
2755 :param predefined_acl: (Optional) Predefined access control list
2757 :type if_generation_match: long
2758 :param if_generation_match:
2759 (Optional) See :ref:`using-if-generation-match`
2761 :type if_generation_not_match: long
2762 :param if_generation_not_match:
2763 (Optional) See :ref:`using-if-generation-not-match`
2765 :type if_metageneration_match: long
2766 :param if_metageneration_match:
2767 (Optional) See :ref:`using-if-metageneration-match`
2769 :type if_metageneration_not_match: long
2770 :param if_metageneration_not_match:
2771 (Optional) See :ref:`using-if-metageneration-not-match`
2773 :type timeout: float or tuple
2774 :param timeout:
2775 (Optional) The amount of time, in seconds, to wait
2776 for the server response. See: :ref:`configuring_timeouts`
2778 :type checksum: str
2779 :param checksum:
2780 (Optional) The type of checksum to compute to verify
2781 the integrity of the object. If the upload is completed in a single
2782 request, the checksum will be entirely precomputed and the remote
2783 server will handle verification and error handling. If the upload
2784 is too large and must be transmitted in multiple requests, the
2785 checksum will be incrementally computed and the client will handle
2786 verification and error handling, raising
2787 google.resumable_media.common.DataCorruption on a mismatch and
2788 attempting to delete the corrupted file. Supported values are
2789 "md5", "crc32c" and None. The default is None.
2791 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2792 :param retry: (Optional) How to retry the RPC. A None value will disable
2793 retries. A google.api_core.retry.Retry value will enable retries,
2794 and the object will define retriable response codes and errors and
2795 configure backoff and timeout options.
2797 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2798 Retry object and activates it only if certain conditions are met.
2799 This class exists to provide safe defaults for RPC calls that are
2800 not technically safe to retry normally (due to potential data
2801 duplication or other side-effects) but become safe to retry if a
2802 condition such as if_generation_match is set.
2804 See the retry.py source code and docstrings in this package
2805 (google.cloud.storage.retry) for information on retry types and how
2806 to configure them.
2808 Media operations (downloads and uploads) do not support non-default
2809 predicates in a Retry object. The default will always be used. Other
2810 configuration changes for Retry objects such as delays and deadlines
2811 are respected.
2812 """
2813 data = _to_bytes(data, encoding="utf-8")
2814 string_buffer = BytesIO(data)
2815 self.upload_from_file(
2816 file_obj=string_buffer,
2817 size=len(data),
2818 content_type=content_type,
2819 num_retries=num_retries,
2820 client=client,
2821 predefined_acl=predefined_acl,
2822 if_generation_match=if_generation_match,
2823 if_generation_not_match=if_generation_not_match,
2824 if_metageneration_match=if_metageneration_match,
2825 if_metageneration_not_match=if_metageneration_not_match,
2826 timeout=timeout,
2827 checksum=checksum,
2828 retry=retry,
2829 )
2831 def create_resumable_upload_session(
2832 self,
2833 content_type=None,
2834 size=None,
2835 origin=None,
2836 client=None,
2837 timeout=_DEFAULT_TIMEOUT,
2838 checksum=None,
2839 predefined_acl=None,
2840 if_generation_match=None,
2841 if_generation_not_match=None,
2842 if_metageneration_match=None,
2843 if_metageneration_not_match=None,
2844 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
2845 ):
2846 """Create a resumable upload session.
2848 Resumable upload sessions allow you to start an upload session from
2849 one client and complete the session in another. This method is called
2850 by the initiator to set the metadata and limits. The initiator then
2851 passes the session URL to the client that will upload the binary data.
2852 The client performs a PUT request on the session URL to complete the
2853 upload. This process allows untrusted clients to upload to an
2854 access-controlled bucket.
2856 For more details, see the
2857 documentation on [`signed URLs`](https://cloud.google.com/storage/docs/access-control/signed-urls#signing-resumable).
2859 The content type of the upload will be determined in order
2860 of precedence:
2862 - The value passed in to this method (if not :data:`None`)
2863 - The value stored on the current blob
2864 - The default value ('application/octet-stream')
2866 .. note::
2867 The effect of uploading to an existing blob depends on the
2868 "versioning" and "lifecycle" policies defined on the blob's
2869 bucket. In the absence of those policies, upload will
2870 overwrite any existing contents.
2872 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)
2873 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)
2874 API documents for details.
2876 If :attr:`encryption_key` is set, the blob will be encrypted with
2877 a [`customer-supplied`](https://cloud.google.com/storage/docs/encryption#customer-supplied)
2878 encryption key.
2880 If :attr:`user_project` is set on the bucket, bills the API request
2881 to that project.
2883 :type size: int
2884 :param size:
2885 (Optional) The maximum number of bytes that can be uploaded using
2886 this session. If the size is not known when creating the session,
2887 this should be left blank.
2889 :type content_type: str
2890 :param content_type: (Optional) Type of content being uploaded.
2892 :type origin: str
2893 :param origin:
2894 (Optional) If set, the upload can only be completed by a user-agent
2895 that uploads from the given origin. This can be useful when passing
2896 the session to a web client.
2898 :type client: :class:`~google.cloud.storage.client.Client`
2899 :param client:
2900 (Optional) The client to use. If not passed, falls back to the
2901 ``client`` stored on the blob's bucket.
2903 :type timeout: float or tuple
2904 :param timeout:
2905 (Optional) The amount of time, in seconds, to wait
2906 for the server response. See: :ref:`configuring_timeouts`
2908 :type checksum: str
2909 :param checksum:
2910 (Optional) The type of checksum to compute to verify
2911 the integrity of the object. After the upload is complete, the
2912 server-computed checksum of the resulting object will be checked
2913 and google.resumable_media.common.DataCorruption will be raised on
2914 a mismatch. On a validation failure, the client will attempt to
2915 delete the uploaded object automatically. Supported values
2916 are "md5", "crc32c" and None. The default is None.
2918 :type predefined_acl: str
2919 :param predefined_acl: (Optional) Predefined access control list
2921 :type if_generation_match: long
2922 :param if_generation_match:
2923 (Optional) See :ref:`using-if-generation-match`
2925 :type if_generation_not_match: long
2926 :param if_generation_not_match:
2927 (Optional) See :ref:`using-if-generation-not-match`
2929 :type if_metageneration_match: long
2930 :param if_metageneration_match:
2931 (Optional) See :ref:`using-if-metageneration-match`
2933 :type if_metageneration_not_match: long
2934 :param if_metageneration_not_match:
2935 (Optional) See :ref:`using-if-metageneration-not-match`
2937 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2938 :param retry: (Optional) How to retry the RPC. A None value will disable
2939 retries. A google.api_core.retry.Retry value will enable retries,
2940 and the object will define retriable response codes and errors and
2941 configure backoff and timeout options.
2942 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2943 Retry object and activates it only if certain conditions are met.
2944 This class exists to provide safe defaults for RPC calls that are
2945 not technically safe to retry normally (due to potential data
2946 duplication or other side-effects) but become safe to retry if a
2947 condition such as if_generation_match is set.
2948 See the retry.py source code and docstrings in this package
2949 (google.cloud.storage.retry) for information on retry types and how
2950 to configure them.
2951 Media operations (downloads and uploads) do not support non-default
2952 predicates in a Retry object. The default will always be used. Other
2953 configuration changes for Retry objects such as delays and deadlines
2954 are respected.
2956 :rtype: str
2957 :returns: The resumable upload session URL. The upload can be
2958 completed by making an HTTP PUT request with the
2959 file's contents.
2961 :raises: :class:`google.cloud.exceptions.GoogleCloudError`
2962 if the session creation response returns an error status.
2963 """
2965 # Handle ConditionalRetryPolicy.
2966 if isinstance(retry, ConditionalRetryPolicy):
2967 # Conditional retries are designed for non-media calls, which change
2968 # arguments into query_params dictionaries. Media operations work
2969 # differently, so here we make a "fake" query_params to feed to the
2970 # ConditionalRetryPolicy.
2971 query_params = {
2972 "ifGenerationMatch": if_generation_match,
2973 "ifMetagenerationMatch": if_metageneration_match,
2974 }
2975 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)
2977 extra_headers = {}
2978 if origin is not None:
2979 # This header is specifically for client-side uploads, it
2980 # determines the origins allowed for CORS.
2981 extra_headers["Origin"] = origin
2983 try:
2984 fake_stream = BytesIO(b"")
2985 # Send a fake the chunk size which we **know** will be acceptable
2986 # to the `ResumableUpload` constructor. The chunk size only
2987 # matters when **sending** bytes to an upload.
2988 upload, _ = self._initiate_resumable_upload(
2989 client,
2990 fake_stream,
2991 content_type,
2992 size,
2993 None,
2994 predefined_acl=predefined_acl,
2995 if_generation_match=if_generation_match,
2996 if_generation_not_match=if_generation_not_match,
2997 if_metageneration_match=if_metageneration_match,
2998 if_metageneration_not_match=if_metageneration_not_match,
2999 extra_headers=extra_headers,
3000 chunk_size=self._CHUNK_SIZE_MULTIPLE,
3001 timeout=timeout,
3002 checksum=checksum,
3003 retry=retry,
3004 )
3006 return upload.resumable_url
3007 except resumable_media.InvalidResponse as exc:
3008 _raise_from_invalid_response(exc)
3010 def get_iam_policy(
3011 self,
3012 client=None,
3013 requested_policy_version=None,
3014 timeout=_DEFAULT_TIMEOUT,
3015 retry=DEFAULT_RETRY,
3016 ):
3017 """Retrieve the IAM policy for the object.
3019 .. note::
3021 Blob- / object-level IAM support does not yet exist and methods
3022 currently call an internal ACL backend not providing any utility
3023 beyond the blob's :attr:`acl` at this time. The API may be enhanced
3024 in the future and is currently undocumented. Use :attr:`acl` for
3025 managing object access control.
3027 If :attr:`user_project` is set on the bucket, bills the API request
3028 to that project.
3030 :type client: :class:`~google.cloud.storage.client.Client`
3031 :param client:
3032 (Optional) The client to use. If not passed, falls back to the
3033 ``client`` stored on the current object's bucket.
3035 :type requested_policy_version: int or ``NoneType``
3036 :param requested_policy_version:
3037 (Optional) The version of IAM policies to request. If a policy
3038 with a condition is requested without setting this, the server will
3039 return an error. This must be set to a value of 3 to retrieve IAM
3040 policies containing conditions. This is to prevent client code that
3041 isn't aware of IAM conditions from interpreting and modifying
3042 policies incorrectly. The service might return a policy with
3043 version lower than the one that was requested, based on the feature
3044 syntax in the policy fetched.
3046 :type timeout: float or tuple
3047 :param timeout:
3048 (Optional) The amount of time, in seconds, to wait
3049 for the server response. See: :ref:`configuring_timeouts`
3051 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3052 :param retry:
3053 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3055 :rtype: :class:`google.api_core.iam.Policy`
3056 :returns: the policy instance, based on the resource returned from
3057 the ``getIamPolicy`` API request.
3058 """
3059 client = self._require_client(client)
3061 query_params = {}
3063 if self.user_project is not None:
3064 query_params["userProject"] = self.user_project
3066 if requested_policy_version is not None:
3067 query_params["optionsRequestedPolicyVersion"] = requested_policy_version
3069 info = client._get_resource(
3070 f"{self.path}/iam",
3071 query_params=query_params,
3072 timeout=timeout,
3073 retry=retry,
3074 _target_object=None,
3075 )
3076 return Policy.from_api_repr(info)
3078 def set_iam_policy(
3079 self,
3080 policy,
3081 client=None,
3082 timeout=_DEFAULT_TIMEOUT,
3083 retry=DEFAULT_RETRY_IF_ETAG_IN_JSON,
3084 ):
3085 """Update the IAM policy for the bucket.
3087 .. note::
3089 Blob- / object-level IAM support does not yet exist and methods
3090 currently call an internal ACL backend not providing any utility
3091 beyond the blob's :attr:`acl` at this time. The API may be enhanced
3092 in the future and is currently undocumented. Use :attr:`acl` for
3093 managing object access control.
3095 If :attr:`user_project` is set on the bucket, bills the API request
3096 to that project.
3098 :type policy: :class:`google.api_core.iam.Policy`
3099 :param policy: policy instance used to update bucket's IAM policy.
3101 :type client: :class:`~google.cloud.storage.client.Client`
3102 :param client:
3103 (Optional) The client to use. If not passed, falls back to the
3104 ``client`` stored on the current bucket.
3106 :type timeout: float or tuple
3107 :param timeout:
3108 (Optional) The amount of time, in seconds, to wait
3109 for the server response. See: :ref:`configuring_timeouts`
3111 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3112 :param retry:
3113 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3115 :rtype: :class:`google.api_core.iam.Policy`
3116 :returns: the policy instance, based on the resource returned from
3117 the ``setIamPolicy`` API request.
3118 """
3119 client = self._require_client(client)
3121 query_params = {}
3123 if self.user_project is not None:
3124 query_params["userProject"] = self.user_project
3126 path = f"{self.path}/iam"
3127 resource = policy.to_api_repr()
3128 resource["resourceId"] = self.path
3129 info = client._put_resource(
3130 path,
3131 resource,
3132 query_params=query_params,
3133 timeout=timeout,
3134 retry=retry,
3135 _target_object=None,
3136 )
3137 return Policy.from_api_repr(info)
3139 def test_iam_permissions(
3140 self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
3141 ):
3142 """API call: test permissions
3144 .. note::
3146 Blob- / object-level IAM support does not yet exist and methods
3147 currently call an internal ACL backend not providing any utility
3148 beyond the blob's :attr:`acl` at this time. The API may be enhanced
3149 in the future and is currently undocumented. Use :attr:`acl` for
3150 managing object access control.
3152 If :attr:`user_project` is set on the bucket, bills the API request
3153 to that project.
3155 :type permissions: list of string
3156 :param permissions: the permissions to check
3158 :type client: :class:`~google.cloud.storage.client.Client`
3159 :param client:
3160 (Optional) The client to use. If not passed, falls back to the
3161 ``client`` stored on the current bucket.
3163 :type timeout: float or tuple
3164 :param timeout:
3165 (Optional) The amount of time, in seconds, to wait
3166 for the server response. See: :ref:`configuring_timeouts`
3168 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3169 :param retry:
3170 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3172 :rtype: list of string
3173 :returns: the permissions returned by the ``testIamPermissions`` API
3174 request.
3175 """
3176 client = self._require_client(client)
3177 query_params = {"permissions": permissions}
3179 if self.user_project is not None:
3180 query_params["userProject"] = self.user_project
3182 path = f"{self.path}/iam/testPermissions"
3183 resp = client._get_resource(
3184 path,
3185 query_params=query_params,
3186 timeout=timeout,
3187 retry=retry,
3188 _target_object=None,
3189 )
3191 return resp.get("permissions", [])
3193 def make_public(
3194 self,
3195 client=None,
3196 timeout=_DEFAULT_TIMEOUT,
3197 if_generation_match=None,
3198 if_generation_not_match=None,
3199 if_metageneration_match=None,
3200 if_metageneration_not_match=None,
3201 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,
3202 ):
3203 """Update blob's ACL, granting read access to anonymous users.
3205 :type client: :class:`~google.cloud.storage.client.Client` or
3206 ``NoneType``
3207 :param client: (Optional) The client to use. If not passed, falls back
3208 to the ``client`` stored on the blob's bucket.
3210 :type timeout: float or tuple
3211 :param timeout:
3212 (Optional) The amount of time, in seconds, to wait
3213 for the server response. See: :ref:`configuring_timeouts`
3215 :type if_generation_match: long
3216 :param if_generation_match:
3217 (Optional) See :ref:`using-if-generation-match`
3219 :type if_generation_not_match: long
3220 :param if_generation_not_match:
3221 (Optional) See :ref:`using-if-generation-not-match`
3223 :type if_metageneration_match: long
3224 :param if_metageneration_match:
3225 (Optional) See :ref:`using-if-metageneration-match`
3227 :type if_metageneration_not_match: long
3228 :param if_metageneration_not_match:
3229 (Optional) See :ref:`using-if-metageneration-not-match`
3231 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3232 :param retry:
3233 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3234 """
3235 self.acl.all().grant_read()
3236 self.acl.save(
3237 client=client,
3238 timeout=timeout,
3239 if_generation_match=if_generation_match,
3240 if_generation_not_match=if_generation_not_match,
3241 if_metageneration_match=if_metageneration_match,
3242 if_metageneration_not_match=if_metageneration_not_match,
3243 retry=retry,
3244 )
3246 def make_private(
3247 self,
3248 client=None,
3249 timeout=_DEFAULT_TIMEOUT,
3250 if_generation_match=None,
3251 if_generation_not_match=None,
3252 if_metageneration_match=None,
3253 if_metageneration_not_match=None,
3254 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,
3255 ):
3256 """Update blob's ACL, revoking read access for anonymous users.
3258 :type client: :class:`~google.cloud.storage.client.Client` or
3259 ``NoneType``
3260 :param client: (Optional) The client to use. If not passed, falls back
3261 to the ``client`` stored on the blob's bucket.
3263 :type timeout: float or tuple
3264 :param timeout:
3265 (Optional) The amount of time, in seconds, to wait
3266 for the server response. See: :ref:`configuring_timeouts`
3268 :type if_generation_match: long
3269 :param if_generation_match:
3270 (Optional) See :ref:`using-if-generation-match`
3272 :type if_generation_not_match: long
3273 :param if_generation_not_match:
3274 (Optional) See :ref:`using-if-generation-not-match`
3276 :type if_metageneration_match: long
3277 :param if_metageneration_match:
3278 (Optional) See :ref:`using-if-metageneration-match`
3280 :type if_metageneration_not_match: long
3281 :param if_metageneration_not_match:
3282 (Optional) See :ref:`using-if-metageneration-not-match`
3284 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3285 :param retry:
3286 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3287 """
3288 self.acl.all().revoke_read()
3289 self.acl.save(
3290 client=client,
3291 timeout=timeout,
3292 if_generation_match=if_generation_match,
3293 if_generation_not_match=if_generation_not_match,
3294 if_metageneration_match=if_metageneration_match,
3295 if_metageneration_not_match=if_metageneration_not_match,
3296 retry=retry,
3297 )
3299 def compose(
3300 self,
3301 sources,
3302 client=None,
3303 timeout=_DEFAULT_TIMEOUT,
3304 if_generation_match=None,
3305 if_metageneration_match=None,
3306 if_source_generation_match=None,
3307 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
3308 ):
3309 """Concatenate source blobs into this one.
3311 If :attr:`user_project` is set on the bucket, bills the API request
3312 to that project.
3314 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/objects/compose)
3315 and a [code sample](https://cloud.google.com/storage/docs/samples/storage-compose-file#storage_compose_file-python).
3317 :type sources: list of :class:`Blob`
3318 :param sources: Blobs whose contents will be composed into this blob.
3320 :type client: :class:`~google.cloud.storage.client.Client`
3321 :param client:
3322 (Optional) The client to use. If not passed, falls back to the
3323 ``client`` stored on the blob's bucket.
3325 :type timeout: float or tuple
3326 :param timeout:
3327 (Optional) The amount of time, in seconds, to wait
3328 for the server response. See: :ref:`configuring_timeouts`
3330 :type if_generation_match: long
3331 :param if_generation_match:
3332 (Optional) Makes the operation conditional on whether the
3333 destination object's current generation matches the given value.
3334 Setting to 0 makes the operation succeed only if there are no live
3335 versions of the object.
3336 Note: In a previous version, this argument worked identically to the
3337 ``if_source_generation_match`` argument. For
3338 backwards-compatibility reasons, if a list is passed in,
3339 this argument will behave like ``if_source_generation_match``
3340 and also issue a DeprecationWarning.
3342 :type if_metageneration_match: long
3343 :param if_metageneration_match:
3344 (Optional) Makes the operation conditional on whether the
3345 destination object's current metageneration matches the given
3346 value.
3348 If a list of long is passed in, no match operation will be
3349 performed. (Deprecated: type(list of long) is supported for
3350 backwards-compatability reasons only.)
3352 :type if_source_generation_match: list of long
3353 :param if_source_generation_match:
3354 (Optional) Makes the operation conditional on whether the current
3355 generation of each source blob matches the corresponding generation.
3356 The list must match ``sources`` item-to-item.
3358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3359 :param retry:
3360 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3361 """
3362 sources_len = len(sources)
3363 client = self._require_client(client)
3364 query_params = {}
3366 if isinstance(if_generation_match, list):
3367 warnings.warn(
3368 _COMPOSE_IF_GENERATION_LIST_DEPRECATED,
3369 DeprecationWarning,
3370 stacklevel=2,
3371 )
3373 if if_source_generation_match is not None:
3374 raise ValueError(
3375 _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR
3376 )
3378 if_source_generation_match = if_generation_match
3379 if_generation_match = None
3381 if isinstance(if_metageneration_match, list):
3382 warnings.warn(
3383 _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED,
3384 DeprecationWarning,
3385 stacklevel=2,
3386 )
3388 if_metageneration_match = None
3390 if if_source_generation_match is None:
3391 if_source_generation_match = [None] * sources_len
3392 if len(if_source_generation_match) != sources_len:
3393 raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR)
3395 source_objects = []
3396 for source, source_generation in zip(sources, if_source_generation_match):
3397 source_object = {"name": source.name, "generation": source.generation}
3399 preconditions = {}
3400 if source_generation is not None:
3401 preconditions["ifGenerationMatch"] = source_generation
3403 if preconditions:
3404 source_object["objectPreconditions"] = preconditions
3406 source_objects.append(source_object)
3408 request = {
3409 "sourceObjects": source_objects,
3410 "destination": self._properties.copy(),
3411 }
3413 if self.user_project is not None:
3414 query_params["userProject"] = self.user_project
3416 _add_generation_match_parameters(
3417 query_params,
3418 if_generation_match=if_generation_match,
3419 if_metageneration_match=if_metageneration_match,
3420 )
3422 api_response = client._post_resource(
3423 f"{self.path}/compose",
3424 request,
3425 query_params=query_params,
3426 timeout=timeout,
3427 retry=retry,
3428 _target_object=self,
3429 )
3430 self._set_properties(api_response)
3432 def rewrite(
3433 self,
3434 source,
3435 token=None,
3436 client=None,
3437 if_generation_match=None,
3438 if_generation_not_match=None,
3439 if_metageneration_match=None,
3440 if_metageneration_not_match=None,
3441 if_source_generation_match=None,
3442 if_source_generation_not_match=None,
3443 if_source_metageneration_match=None,
3444 if_source_metageneration_not_match=None,
3445 timeout=_DEFAULT_TIMEOUT,
3446 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
3447 ):
3448 """Rewrite source blob into this one.
3450 If :attr:`user_project` is set on the bucket, bills the API request
3451 to that project.
3453 .. note::
3455 ``rewrite`` is not supported in a ``Batch`` context.
3457 :type source: :class:`Blob`
3458 :param source: blob whose contents will be rewritten into this blob.
3460 :type token: str
3461 :param token:
3462 (Optional) Token returned from an earlier, not-completed call to
3463 rewrite the same source blob. If passed, result will include
3464 updated status, total bytes written.
3466 :type client: :class:`~google.cloud.storage.client.Client`
3467 :param client:
3468 (Optional) The client to use. If not passed, falls back to the
3469 ``client`` stored on the blob's bucket.
3471 :type if_generation_match: long
3472 :param if_generation_match:
3473 (Optional) See :ref:`using-if-generation-match`
3474 Note that the generation to be matched is that of the
3475 ``destination`` blob.
3477 :type if_generation_not_match: long
3478 :param if_generation_not_match:
3479 (Optional) See :ref:`using-if-generation-not-match`
3480 Note that the generation to be matched is that of the
3481 ``destination`` blob.
3483 :type if_metageneration_match: long
3484 :param if_metageneration_match:
3485 (Optional) See :ref:`using-if-metageneration-match`
3486 Note that the metageneration to be matched is that of the
3487 ``destination`` blob.
3489 :type if_metageneration_not_match: long
3490 :param if_metageneration_not_match:
3491 (Optional) See :ref:`using-if-metageneration-not-match`
3492 Note that the metageneration to be matched is that of the
3493 ``destination`` blob.
3495 :type if_source_generation_match: long
3496 :param if_source_generation_match:
3497 (Optional) Makes the operation conditional on whether the source
3498 object's generation matches the given value.
3500 :type if_source_generation_not_match: long
3501 :param if_source_generation_not_match:
3502 (Optional) Makes the operation conditional on whether the source
3503 object's generation does not match the given value.
3505 :type if_source_metageneration_match: long
3506 :param if_source_metageneration_match:
3507 (Optional) Makes the operation conditional on whether the source
3508 object's current metageneration matches the given value.
3510 :type if_source_metageneration_not_match: long
3511 :param if_source_metageneration_not_match:
3512 (Optional) Makes the operation conditional on whether the source
3513 object's current metageneration does not match the given value.
3515 :type timeout: float or tuple
3516 :param timeout:
3517 (Optional) The amount of time, in seconds, to wait
3518 for the server response. See: :ref:`configuring_timeouts`
3520 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3521 :param retry:
3522 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3524 :rtype: tuple
3525 :returns: ``(token, bytes_rewritten, total_bytes)``, where ``token``
3526 is a rewrite token (``None`` if the rewrite is complete),
3527 ``bytes_rewritten`` is the number of bytes rewritten so far,
3528 and ``total_bytes`` is the total number of bytes to be
3529 rewritten.
3530 """
3531 client = self._require_client(client)
3532 headers = _get_encryption_headers(self._encryption_key)
3533 headers.update(_get_encryption_headers(source._encryption_key, source=True))
3535 query_params = self._query_params
3536 if "generation" in query_params:
3537 del query_params["generation"]
3539 if token:
3540 query_params["rewriteToken"] = token
3542 if source.generation:
3543 query_params["sourceGeneration"] = source.generation
3545 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object
3546 # at rest, object resource metadata will store the version of the Key Management
3547 # Service cryptographic material. If a Blob instance with KMS Key metadata set is
3548 # used to rewrite the object, then the existing kmsKeyName version
3549 # value can't be used in the rewrite request and the client instead ignores it.
3550 if (
3551 self.kms_key_name is not None
3552 and "cryptoKeyVersions" not in self.kms_key_name
3553 ):
3554 query_params["destinationKmsKeyName"] = self.kms_key_name
3556 _add_generation_match_parameters(
3557 query_params,
3558 if_generation_match=if_generation_match,
3559 if_generation_not_match=if_generation_not_match,
3560 if_metageneration_match=if_metageneration_match,
3561 if_metageneration_not_match=if_metageneration_not_match,
3562 if_source_generation_match=if_source_generation_match,
3563 if_source_generation_not_match=if_source_generation_not_match,
3564 if_source_metageneration_match=if_source_metageneration_match,
3565 if_source_metageneration_not_match=if_source_metageneration_not_match,
3566 )
3568 path = f"{source.path}/rewriteTo{self.path}"
3569 api_response = client._post_resource(
3570 path,
3571 self._properties,
3572 query_params=query_params,
3573 headers=headers,
3574 timeout=timeout,
3575 retry=retry,
3576 _target_object=self,
3577 )
3578 rewritten = int(api_response["totalBytesRewritten"])
3579 size = int(api_response["objectSize"])
3581 # The resource key is set if and only if the API response is
3582 # completely done. Additionally, there is no rewrite token to return
3583 # in this case.
3584 if api_response["done"]:
3585 self._set_properties(api_response["resource"])
3586 return None, rewritten, size
3588 return api_response["rewriteToken"], rewritten, size
3590 def update_storage_class(
3591 self,
3592 new_class,
3593 client=None,
3594 if_generation_match=None,
3595 if_generation_not_match=None,
3596 if_metageneration_match=None,
3597 if_metageneration_not_match=None,
3598 if_source_generation_match=None,
3599 if_source_generation_not_match=None,
3600 if_source_metageneration_match=None,
3601 if_source_metageneration_not_match=None,
3602 timeout=_DEFAULT_TIMEOUT,
3603 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
3604 ):
3605 """Update blob's storage class via a rewrite-in-place. This helper will
3606 wait for the rewrite to complete before returning, so it may take some
3607 time for large files.
3609 See
3610 https://cloud.google.com/storage/docs/per-object-storage-class
3612 If :attr:`user_project` is set on the bucket, bills the API request
3613 to that project.
3615 :type new_class: str
3616 :param new_class:
3617 new storage class for the object. One of:
3618 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
3619 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
3620 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
3621 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
3622 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
3623 or
3624 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`.
3626 :type client: :class:`~google.cloud.storage.client.Client`
3627 :param client:
3628 (Optional) The client to use. If not passed, falls back to the
3629 ``client`` stored on the blob's bucket.
3631 :type if_generation_match: long
3632 :param if_generation_match:
3633 (Optional) See :ref:`using-if-generation-match`
3634 Note that the generation to be matched is that of the
3635 ``destination`` blob.
3637 :type if_generation_not_match: long
3638 :param if_generation_not_match:
3639 (Optional) See :ref:`using-if-generation-not-match`
3640 Note that the generation to be matched is that of the
3641 ``destination`` blob.
3643 :type if_metageneration_match: long
3644 :param if_metageneration_match:
3645 (Optional) See :ref:`using-if-metageneration-match`
3646 Note that the metageneration to be matched is that of the
3647 ``destination`` blob.
3649 :type if_metageneration_not_match: long
3650 :param if_metageneration_not_match:
3651 (Optional) See :ref:`using-if-metageneration-not-match`
3652 Note that the metageneration to be matched is that of the
3653 ``destination`` blob.
3655 :type if_source_generation_match: long
3656 :param if_source_generation_match:
3657 (Optional) Makes the operation conditional on whether the source
3658 object's generation matches the given value.
3660 :type if_source_generation_not_match: long
3661 :param if_source_generation_not_match:
3662 (Optional) Makes the operation conditional on whether the source
3663 object's generation does not match the given value.
3665 :type if_source_metageneration_match: long
3666 :param if_source_metageneration_match:
3667 (Optional) Makes the operation conditional on whether the source
3668 object's current metageneration matches the given value.
3670 :type if_source_metageneration_not_match: long
3671 :param if_source_metageneration_not_match:
3672 (Optional) Makes the operation conditional on whether the source
3673 object's current metageneration does not match the given value.
3675 :type timeout: float or tuple
3676 :param timeout:
3677 (Optional) The amount of time, in seconds, to wait
3678 for the server response. See: :ref:`configuring_timeouts`
3680 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
3681 :param retry:
3682 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
3683 """
3684 # Update current blob's storage class prior to rewrite
3685 self._patch_property("storageClass", new_class)
3687 # Execute consecutive rewrite operations until operation is done
3688 token, _, _ = self.rewrite(
3689 self,
3690 if_generation_match=if_generation_match,
3691 if_generation_not_match=if_generation_not_match,
3692 if_metageneration_match=if_metageneration_match,
3693 if_metageneration_not_match=if_metageneration_not_match,
3694 if_source_generation_match=if_source_generation_match,
3695 if_source_generation_not_match=if_source_generation_not_match,
3696 if_source_metageneration_match=if_source_metageneration_match,
3697 if_source_metageneration_not_match=if_source_metageneration_not_match,
3698 timeout=timeout,
3699 retry=retry,
3700 )
3701 while token is not None:
3702 token, _, _ = self.rewrite(
3703 self,
3704 token=token,
3705 if_generation_match=if_generation_match,
3706 if_generation_not_match=if_generation_not_match,
3707 if_metageneration_match=if_metageneration_match,
3708 if_metageneration_not_match=if_metageneration_not_match,
3709 if_source_generation_match=if_source_generation_match,
3710 if_source_generation_not_match=if_source_generation_not_match,
3711 if_source_metageneration_match=if_source_metageneration_match,
3712 if_source_metageneration_not_match=if_source_metageneration_not_match,
3713 timeout=timeout,
3714 retry=retry,
3715 )
3717 def open(
3718 self,
3719 mode="r",
3720 chunk_size=None,
3721 ignore_flush=None,
3722 encoding=None,
3723 errors=None,
3724 newline=None,
3725 **kwargs,
3726 ):
3727 r"""Create a file handler for file-like I/O to or from this blob.
3729 This method can be used as a context manager, just like Python's
3730 built-in 'open()' function.
3732 While reading, as with other read methods, if blob.generation is not set
3733 the most recent blob generation will be used. Because the file-like IO
3734 reader downloads progressively in chunks, this could result in data from
3735 multiple versions being mixed together. If this is a concern, use
3736 either bucket.get_blob(), or blob.reload(), which will download the
3737 latest generation number and set it; or, if the generation is known, set
3738 it manually, for instance with bucket.blob(generation=123456).
3740 Checksumming (hashing) to verify data integrity is disabled for reads
3741 using this feature because reads are implemented using request ranges,
3742 which do not provide checksums to validate. See
3743 https://cloud.google.com/storage/docs/hashes-etags for details.
3745 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_fileio_write_read.py).
3747 Keyword arguments to pass to the underlying API calls.
3748 For both uploads and downloads, the following arguments are
3749 supported:
3751 - ``if_generation_match``
3752 - ``if_generation_not_match``
3753 - ``if_metageneration_match``
3754 - ``if_metageneration_not_match``
3755 - ``timeout``
3756 - ``retry``
3758 For downloads only, the following additional arguments are supported:
3760 - ``raw_download``
3762 For uploads only, the following additional arguments are supported:
3764 - ``content_type``
3765 - ``num_retries``
3766 - ``predefined_acl``
3767 - ``checksum``
3769 .. note::
3771 ``num_retries`` is supported for backwards-compatibility
3772 reasons only; please use ``retry`` with a Retry object or
3773 ConditionalRetryPolicy instead.
3775 :type mode: str
3776 :param mode:
3777 (Optional) A mode string, as per standard Python `open()` semantics.The first
3778 character must be 'r', to open the blob for reading, or 'w' to open
3779 it for writing. The second character, if present, must be 't' for
3780 (unicode) text mode, or 'b' for bytes mode. If the second character
3781 is omitted, text mode is the default.
3783 :type chunk_size: long
3784 :param chunk_size:
3785 (Optional) For reads, the minimum number of bytes to read at a time.
3786 If fewer bytes than the chunk_size are requested, the remainder is
3787 buffered. For writes, the maximum number of bytes to buffer before
3788 sending data to the server, and the size of each request when data
3789 is sent. Writes are implemented as a "resumable upload", so
3790 chunk_size for writes must be exactly a multiple of 256KiB as with
3791 other resumable uploads. The default is 40 MiB.
3793 :type ignore_flush: bool
3794 :param ignore_flush:
3795 (Optional) For non text-mode writes, makes flush() do nothing
3796 instead of raising an error. flush() without closing is not
3797 supported by the remote service and therefore calling it normally
3798 results in io.UnsupportedOperation. However, that behavior is
3799 incompatible with some consumers and wrappers of file objects in
3800 Python, such as zipfile.ZipFile or io.TextIOWrapper. Setting
3801 ignore_flush will cause flush() to successfully do nothing, for
3802 compatibility with those contexts. The correct way to actually flush
3803 data to the remote server is to close() (using a context manager,
3804 such as in the example, will cause this to happen automatically).
3806 :type encoding: str
3807 :param encoding:
3808 (Optional) For text mode only, the name of the encoding that the stream will
3809 be decoded or encoded with. If omitted, it defaults to
3810 locale.getpreferredencoding(False).
3812 :type errors: str
3813 :param errors:
3814 (Optional) For text mode only, an optional string that specifies how encoding
3815 and decoding errors are to be handled. Pass 'strict' to raise a
3816 ValueError exception if there is an encoding error (the default of
3817 None has the same effect), or pass 'ignore' to ignore errors. (Note
3818 that ignoring encoding errors can lead to data loss.) Other more
3819 rarely-used options are also available; see the Python 'io' module
3820 documentation for 'io.TextIOWrapper' for a complete list.
3822 :type newline: str
3823 :param newline:
3824 (Optional) For text mode only, controls how line endings are handled. It can
3825 be None, '', '\n', '\r', and '\r\n'. If None, reads use "universal
3826 newline mode" and writes use the system default. See the Python
3827 'io' module documentation for 'io.TextIOWrapper' for details.
3829 :returns: A 'BlobReader' or 'BlobWriter' from
3830 'google.cloud.storage.fileio', or an 'io.TextIOWrapper' around one
3831 of those classes, depending on the 'mode' argument.
3832 """
3833 if mode == "rb":
3834 if encoding or errors or newline:
3835 raise ValueError(
3836 "encoding, errors and newline arguments are for text mode only"
3837 )
3838 if ignore_flush:
3839 raise ValueError(
3840 "ignore_flush argument is for non-text write mode only"
3841 )
3842 return BlobReader(self, chunk_size=chunk_size, **kwargs)
3843 elif mode == "wb":
3844 if encoding or errors or newline:
3845 raise ValueError(
3846 "encoding, errors and newline arguments are for text mode only"
3847 )
3848 return BlobWriter(
3849 self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs
3850 )
3851 elif mode in ("r", "rt"):
3852 if ignore_flush:
3853 raise ValueError(
3854 "ignore_flush argument is for non-text write mode only"
3855 )
3856 return TextIOWrapper(
3857 BlobReader(self, chunk_size=chunk_size, **kwargs),
3858 encoding=encoding,
3859 errors=errors,
3860 newline=newline,
3861 )
3862 elif mode in ("w", "wt"):
3863 if ignore_flush is False:
3864 raise ValueError(
3865 "ignore_flush is required for text mode writing and "
3866 "cannot be set to False"
3867 )
3868 return TextIOWrapper(
3869 BlobWriter(self, chunk_size=chunk_size, ignore_flush=True, **kwargs),
3870 encoding=encoding,
3871 errors=errors,
3872 newline=newline,
3873 )
3874 else:
3875 raise NotImplementedError(
3876 "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only."
3877 )
3879 cache_control = _scalar_property("cacheControl")
3880 """HTTP 'Cache-Control' header for this object.
3882 See [`RFC 7234`](https://tools.ietf.org/html/rfc7234#section-5.2)
3883 and [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3885 :rtype: str or ``NoneType``
3887 """
3889 content_disposition = _scalar_property("contentDisposition")
3890 """HTTP 'Content-Disposition' header for this object.
3892 See [`RFC 6266`](https://tools.ietf.org/html/rfc7234#section-5.2) and
3893 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3895 :rtype: str or ``NoneType``
3896 """
3898 content_encoding = _scalar_property("contentEncoding")
3899 """HTTP 'Content-Encoding' header for this object.
3901 See [`RFC 7231`](https://tools.ietf.org/html/rfc7231#section-3.1.2.2) and
3902 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3904 :rtype: str or ``NoneType``
3905 """
3907 content_language = _scalar_property("contentLanguage")
3908 """HTTP 'Content-Language' header for this object.
3910 See [`BCP47`](https://tools.ietf.org/html/bcp47) and
3911 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3913 :rtype: str or ``NoneType``
3914 """
3916 content_type = _scalar_property(_CONTENT_TYPE_FIELD)
3917 """HTTP 'Content-Type' header for this object.
3919 See [`RFC 2616`](https://tools.ietf.org/html/rfc2616#section-14.17) and
3920 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3922 :rtype: str or ``NoneType``
3923 """
3925 crc32c = _scalar_property("crc32c")
3926 """CRC32C checksum for this object.
3928 This returns the blob's CRC32C checksum. To retrieve the value, first use a
3929 reload method of the Blob class which loads the blob's properties from the server.
3931 See [`RFC 4960`](https://tools.ietf.org/html/rfc4960#appendix-B) and
3932 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3934 If not set before upload, the server will compute the hash.
3936 :rtype: str or ``NoneType``
3937 """
3939 @property
3940 def component_count(self):
3941 """Number of underlying components that make up this object.
3943 See https://cloud.google.com/storage/docs/json_api/v1/objects
3945 :rtype: int or ``NoneType``
3946 :returns: The component count (in case of a composed object) or
3947 ``None`` if the blob's resource has not been loaded from
3948 the server. This property will not be set on objects
3949 not created via ``compose``.
3950 """
3951 component_count = self._properties.get("componentCount")
3952 if component_count is not None:
3953 return int(component_count)
3955 @property
3956 def etag(self):
3957 """Retrieve the ETag for the object.
3959 See [`RFC 2616 (etags)`](https://tools.ietf.org/html/rfc2616#section-3.11) and
3960 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3962 :rtype: str or ``NoneType``
3963 :returns: The blob etag or ``None`` if the blob's resource has not
3964 been loaded from the server.
3965 """
3966 return self._properties.get("etag")
3968 event_based_hold = _scalar_property("eventBasedHold")
3969 """Is an event-based hold active on the object?
3971 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
3973 If the property is not set locally, returns :data:`None`.
3975 :rtype: bool or ``NoneType``
3976 """
3978 @property
3979 def generation(self):
3980 """Retrieve the generation for the object.
3982 See https://cloud.google.com/storage/docs/json_api/v1/objects
3984 :rtype: int or ``NoneType``
3985 :returns: The generation of the blob or ``None`` if the blob's
3986 resource has not been loaded from the server.
3987 """
3988 generation = self._properties.get("generation")
3989 if generation is not None:
3990 return int(generation)
3992 @property
3993 def id(self):
3994 """Retrieve the ID for the object.
3996 See https://cloud.google.com/storage/docs/json_api/v1/objects
3998 The ID consists of the bucket name, object name, and generation number.
4000 :rtype: str or ``NoneType``
4001 :returns: The ID of the blob or ``None`` if the blob's
4002 resource has not been loaded from the server.
4003 """
4004 return self._properties.get("id")
4006 md5_hash = _scalar_property("md5Hash")
4007 """MD5 hash for this object.
4009 This returns the blob's MD5 hash. To retrieve the value, first use a
4010 reload method of the Blob class which loads the blob's properties from the server.
4012 See [`RFC 1321`](https://tools.ietf.org/html/rfc1321) and
4013 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
4015 If not set before upload, the server will compute the hash.
4017 :rtype: str or ``NoneType``
4018 """
4020 @property
4021 def media_link(self):
4022 """Retrieve the media download URI for the object.
4024 See https://cloud.google.com/storage/docs/json_api/v1/objects
4026 :rtype: str or ``NoneType``
4027 :returns: The media link for the blob or ``None`` if the blob's
4028 resource has not been loaded from the server.
4029 """
4030 return self._properties.get("mediaLink")
4032 @property
4033 def metadata(self):
4034 """Retrieve arbitrary/application specific metadata for the object.
4036 See https://cloud.google.com/storage/docs/json_api/v1/objects
4038 :setter: Update arbitrary/application specific metadata for the
4039 object.
4040 :getter: Retrieve arbitrary/application specific metadata for
4041 the object.
4043 :rtype: dict or ``NoneType``
4044 :returns: The metadata associated with the blob or ``None`` if the
4045 property is not set.
4046 """
4047 return copy.deepcopy(self._properties.get("metadata"))
4049 @metadata.setter
4050 def metadata(self, value):
4051 """Update arbitrary/application specific metadata for the object.
4053 Values are stored to GCS as strings. To delete a key, set its value to
4054 None and call blob.patch().
4056 See https://cloud.google.com/storage/docs/json_api/v1/objects
4058 :type value: dict
4059 :param value: The blob metadata to set.
4060 """
4061 if value is not None:
4062 value = {k: str(v) if v is not None else None for k, v in value.items()}
4063 self._patch_property("metadata", value)
4065 @property
4066 def metageneration(self):
4067 """Retrieve the metageneration for the object.
4069 See https://cloud.google.com/storage/docs/json_api/v1/objects
4071 :rtype: int or ``NoneType``
4072 :returns: The metageneration of the blob or ``None`` if the blob's
4073 resource has not been loaded from the server.
4074 """
4075 metageneration = self._properties.get("metageneration")
4076 if metageneration is not None:
4077 return int(metageneration)
4079 @property
4080 def owner(self):
4081 """Retrieve info about the owner of the object.
4083 See https://cloud.google.com/storage/docs/json_api/v1/objects
4085 :rtype: dict or ``NoneType``
4086 :returns: Mapping of owner's role/ID, or ``None`` if the blob's
4087 resource has not been loaded from the server.
4088 """
4089 return copy.deepcopy(self._properties.get("owner"))
4091 @property
4092 def retention_expiration_time(self):
4093 """Retrieve timestamp at which the object's retention period expires.
4095 See https://cloud.google.com/storage/docs/json_api/v1/objects
4097 :rtype: :class:`datetime.datetime` or ``NoneType``
4098 :returns: Datetime object parsed from RFC3339 valid timestamp, or
4099 ``None`` if the property is not set locally.
4100 """
4101 value = self._properties.get("retentionExpirationTime")
4102 if value is not None:
4103 return _rfc3339_nanos_to_datetime(value)
4105 @property
4106 def self_link(self):
4107 """Retrieve the URI for the object.
4109 See https://cloud.google.com/storage/docs/json_api/v1/objects
4111 :rtype: str or ``NoneType``
4112 :returns: The self link for the blob or ``None`` if the blob's
4113 resource has not been loaded from the server.
4114 """
4115 return self._properties.get("selfLink")
4117 @property
4118 def size(self):
4119 """Size of the object, in bytes.
4121 See https://cloud.google.com/storage/docs/json_api/v1/objects
4123 :rtype: int or ``NoneType``
4124 :returns: The size of the blob or ``None`` if the blob's
4125 resource has not been loaded from the server.
4126 """
4127 size = self._properties.get("size")
4128 if size is not None:
4129 return int(size)
4131 @property
4132 def kms_key_name(self):
4133 """Resource name of Cloud KMS key used to encrypt the blob's contents.
4135 :rtype: str or ``NoneType``
4136 :returns:
4137 The resource name or ``None`` if no Cloud KMS key was used,
4138 or the blob's resource has not been loaded from the server.
4139 """
4140 return self._properties.get("kmsKeyName")
4142 @kms_key_name.setter
4143 def kms_key_name(self, value):
4144 """Set KMS encryption key for object.
4146 :type value: str or ``NoneType``
4147 :param value: new KMS key name (None to clear any existing key).
4148 """
4149 self._patch_property("kmsKeyName", value)
4151 storage_class = _scalar_property("storageClass")
4152 """Retrieve the storage class for the object.
4154 This can only be set at blob / object **creation** time. If you'd
4155 like to change the storage class **after** the blob / object already
4156 exists in a bucket, call :meth:`update_storage_class` (which uses
4157 :meth:`rewrite`).
4159 See https://cloud.google.com/storage/docs/storage-classes
4161 :rtype: str or ``NoneType``
4162 :returns:
4163 If set, one of
4164 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
4165 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
4166 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
4167 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
4168 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
4169 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
4170 :attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`,
4171 else ``None``.
4172 """
4174 temporary_hold = _scalar_property("temporaryHold")
4175 """Is a temporary hold active on the object?
4177 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).
4179 If the property is not set locally, returns :data:`None`.
4181 :rtype: bool or ``NoneType``
4182 """
4184 @property
4185 def time_deleted(self):
4186 """Retrieve the timestamp at which the object was deleted.
4188 See https://cloud.google.com/storage/docs/json_api/v1/objects
4190 :rtype: :class:`datetime.datetime` or ``NoneType``
4191 :returns: Datetime object parsed from RFC3339 valid timestamp, or
4192 ``None`` if the blob's resource has not been loaded from
4193 the server (see :meth:`reload`). If the blob has
4194 not been deleted, this will never be set.
4195 """
4196 value = self._properties.get("timeDeleted")
4197 if value is not None:
4198 return _rfc3339_nanos_to_datetime(value)
4200 @property
4201 def time_created(self):
4202 """Retrieve the timestamp at which the object was created.
4204 See https://cloud.google.com/storage/docs/json_api/v1/objects
4206 :rtype: :class:`datetime.datetime` or ``NoneType``
4207 :returns: Datetime object parsed from RFC3339 valid timestamp, or
4208 ``None`` if the blob's resource has not been loaded from
4209 the server (see :meth:`reload`).
4210 """
4211 value = self._properties.get("timeCreated")
4212 if value is not None:
4213 return _rfc3339_nanos_to_datetime(value)
4215 @property
4216 def updated(self):
4217 """Retrieve the timestamp at which the object was updated.
4219 See https://cloud.google.com/storage/docs/json_api/v1/objects
4221 :rtype: :class:`datetime.datetime` or ``NoneType``
4222 :returns: Datetime object parsed from RFC3339 valid timestamp, or
4223 ``None`` if the blob's resource has not been loaded from
4224 the server (see :meth:`reload`).
4225 """
4226 value = self._properties.get("updated")
4227 if value is not None:
4228 return _rfc3339_nanos_to_datetime(value)
4230 @property
4231 def custom_time(self):
4232 """Retrieve the custom time for the object.
4234 See https://cloud.google.com/storage/docs/json_api/v1/objects
4236 :rtype: :class:`datetime.datetime` or ``NoneType``
4237 :returns: Datetime object parsed from RFC3339 valid timestamp, or
4238 ``None`` if the blob's resource has not been loaded from
4239 the server (see :meth:`reload`).
4240 """
4241 value = self._properties.get("customTime")
4242 if value is not None:
4243 return _rfc3339_nanos_to_datetime(value)
4245 @custom_time.setter
4246 def custom_time(self, value):
4247 """Set the custom time for the object.
4249 Once set on the server side object, this value can't be unset, but may
4250 only changed to a custom datetime in the future.
4252 If :attr:`custom_time` must be unset, either perform a rewrite
4253 operation or upload the data again.
4255 See https://cloud.google.com/storage/docs/json_api/v1/objects
4257 :type value: :class:`datetime.datetime`
4258 :param value: new value
4259 """
4260 if value is not None:
4261 value = _datetime_to_rfc3339(value)
4263 self._patch_property("customTime", value)
4266def _get_host_name(connection):
4267 """Returns the host name from the given connection.
4269 :type connection: :class:`~google.cloud.storage._http.Connection`
4270 :param connection: The connection object.
4272 :rtype: str
4273 :returns: The host name.
4274 """
4275 # TODO: After google-cloud-core 1.6.0 is stable and we upgrade it
4276 # to 1.6.0 in setup.py, we no longer need to check the attribute
4277 # existence. We can simply return connection.get_api_base_url_for_mtls().
4278 return (
4279 connection.API_BASE_URL
4280 if not hasattr(connection, "get_api_base_url_for_mtls")
4281 else connection.get_api_base_url_for_mtls()
4282 )
4285def _get_encryption_headers(key, source=False):
4286 """Builds customer encryption key headers
4288 :type key: bytes
4289 :param key: 32 byte key to build request key and hash.
4291 :type source: bool
4292 :param source: If true, return headers for the "source" blob; otherwise,
4293 return headers for the "destination" blob.
4295 :rtype: dict
4296 :returns: dict of HTTP headers being sent in request.
4297 """
4298 if key is None:
4299 return {}
4301 key = _to_bytes(key)
4302 key_hash = hashlib.sha256(key).digest()
4303 key_hash = base64.b64encode(key_hash)
4304 key = base64.b64encode(key)
4306 if source:
4307 prefix = "X-Goog-Copy-Source-Encryption-"
4308 else:
4309 prefix = "X-Goog-Encryption-"
4311 return {
4312 prefix + "Algorithm": "AES256",
4313 prefix + "Key": _bytes_to_unicode(key),
4314 prefix + "Key-Sha256": _bytes_to_unicode(key_hash),
4315 }
4318def _quote(value, safe=b"~"):
4319 """URL-quote a string.
4321 If the value is unicode, this method first UTF-8 encodes it as bytes and
4322 then quotes the bytes. (In Python 3, ``urllib.parse.quote`` does this
4323 encoding automatically, but in Python 2, non-ASCII characters cannot be
4324 quoted.)
4326 :type value: str or bytes
4327 :param value: The value to be URL-quoted.
4329 :type safe: bytes
4330 :param safe: Bytes *not* to be quoted. By default, includes only ``b'~'``.
4332 :rtype: str
4333 :returns: The encoded value (bytes in Python 2, unicode in Python 3).
4334 """
4335 value = _to_bytes(value, encoding="utf-8")
4336 return quote(value, safe=safe)
4339def _maybe_rewind(stream, rewind=False):
4340 """Rewind the stream if desired.
4342 :type stream: IO[bytes]
4343 :param stream: A bytes IO object open for reading.
4345 :type rewind: bool
4346 :param rewind: Indicates if we should seek to the beginning of the stream.
4347 """
4348 if rewind:
4349 stream.seek(0, os.SEEK_SET)
4352def _raise_from_invalid_response(error):
4353 """Re-wrap and raise an ``InvalidResponse`` exception.
4355 :type error: :exc:`google.resumable_media.InvalidResponse`
4356 :param error: A caught exception from the ``google-resumable-media``
4357 library.
4359 :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
4360 to the failed status code
4361 """
4362 response = error.response
4364 # The 'response.text' gives the actual reason of error, where 'error' gives
4365 # the message of expected status code.
4366 if response.text:
4367 error_message = response.text + ": " + str(error)
4368 else:
4369 error_message = str(error)
4371 message = f"{response.request.method} {response.request.url}: {error_message}"
4373 raise exceptions.from_http_status(response.status_code, message, response=response)
4376def _add_query_parameters(base_url, name_value_pairs):
4377 """Add one query parameter to a base URL.
4379 :type base_url: string
4380 :param base_url: Base URL (may already contain query parameters)
4382 :type name_value_pairs: list of (string, string) tuples.
4383 :param name_value_pairs: Names and values of the query parameters to add
4385 :rtype: string
4386 :returns: URL with additional query strings appended.
4387 """
4388 if len(name_value_pairs) == 0:
4389 return base_url
4391 scheme, netloc, path, query, frag = urlsplit(base_url)
4392 query = parse_qsl(query)
4393 query.extend(name_value_pairs)
4394 return urlunsplit((scheme, netloc, path, urlencode(query), frag))