Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/blob.py: 30%

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

15# pylint: disable=too-many-lines

17"""Create / interact with Google Cloud Storage blobs.

18"""

20import base64

21import copy

22import hashlib

23from io import BytesIO

24from io import TextIOWrapper

25import logging

26import mimetypes

27import os

28import re

29from email.parser import HeaderParser

30from urllib.parse import parse_qsl

31from urllib.parse import quote

32from urllib.parse import urlencode

33from urllib.parse import urlsplit

34from urllib.parse import urlunsplit

35import warnings

37from google import resumable_media

38from google.resumable_media.requests import ChunkedDownload

39from google.resumable_media.requests import Download

40from google.resumable_media.requests import RawDownload

41from google.resumable_media.requests import RawChunkedDownload

42from google.resumable_media.requests import MultipartUpload

43from google.resumable_media.requests import ResumableUpload

45from google.api_core.iam import Policy

46from google.cloud import exceptions

47from google.cloud._helpers import _bytes_to_unicode

48from google.cloud._helpers import _datetime_to_rfc3339

49from google.cloud._helpers import _rfc3339_nanos_to_datetime

50from google.cloud._helpers import _to_bytes

51from google.cloud.exceptions import NotFound

52from google.cloud.storage._helpers import _add_etag_match_headers

53from google.cloud.storage._helpers import _add_generation_match_parameters

54from google.cloud.storage._helpers import _PropertyMixin

55from google.cloud.storage._helpers import _scalar_property

56from google.cloud.storage._helpers import _bucket_bound_hostname_url

57from google.cloud.storage._helpers import _raise_if_more_than_one_set

58from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry

59from google.cloud.storage._helpers import _get_default_headers

60from google.cloud.storage._signing import generate_signed_url_v2

61from google.cloud.storage._signing import generate_signed_url_v4

62from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE

63from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST

64from google.cloud.storage._helpers import _API_VERSION

65from google.cloud.storage.acl import ACL

66from google.cloud.storage.acl import ObjectACL

67from google.cloud.storage.constants import _DEFAULT_TIMEOUT

68from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS

69from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS

70from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS

71from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS

72from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS

73from google.cloud.storage.constants import STANDARD_STORAGE_CLASS

74from google.cloud.storage.retry import ConditionalRetryPolicy

75from google.cloud.storage.retry import DEFAULT_RETRY

76from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON

77from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED

78from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED

79from google.cloud.storage.fileio import BlobReader

80from google.cloud.storage.fileio import BlobWriter

83_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST

84_DEFAULT_CONTENT_TYPE = "application/octet-stream"

85_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"

86_BASE_UPLOAD_TEMPLATE = (

87 "{hostname}/upload/storage/{api_version}{bucket_path}/o?uploadType="

88)

89_MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "multipart"

90_RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "resumable"

91# NOTE: "acl" is also writeable but we defer ACL management to

92# the classes in the google.cloud.storage.acl module.

93_CONTENT_TYPE_FIELD = "contentType"

94_WRITABLE_FIELDS = (

95 "cacheControl",

96 "contentDisposition",

97 "contentEncoding",

98 "contentLanguage",

99 _CONTENT_TYPE_FIELD,

100 "crc32c",

101 "customTime",

102 "md5Hash",

103 "metadata",

104 "name",

105 "storageClass",

106)

107_READ_LESS_THAN_SIZE = (

108 "Size {:d} was specified but the file-like object only had " "{:d} bytes remaining."

109)

110_CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE = (

111 "A checksum of type `{}` was requested, but checksumming is not available "

112 "for downloads when chunk_size is set."

113)

114_COMPOSE_IF_GENERATION_LIST_DEPRECATED = (

115 "'if_generation_match: type list' is deprecated and supported for "

116 "backwards-compatability reasons only. Use 'if_source_generation_match' "

117 "instead' to match source objects' generations."

118)

119_COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR = (

120 "Use 'if_generation_match' to match the generation of the destination "

121 "object by passing in a generation number, instead of a list. "

122 "Use 'if_source_generation_match' to match source objects generations."

123)

124_COMPOSE_IF_METAGENERATION_LIST_DEPRECATED = (

125 "'if_metageneration_match: type list' is deprecated and supported for "

126 "backwards-compatability reasons only. Note that the metageneration to "

127 "be matched is that of the destination blob. Please pass in a single "

128 "value (type long)."

129)

130_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR = (

131 "'if_source_generation_match' length must be the same as 'sources' length"

132)

133_DOWNLOAD_AS_STRING_DEPRECATED = (

134 "Blob.download_as_string() is deprecated and will be removed in future. "

135 "Use Blob.download_as_bytes() instead."

136)

137

138

139_DEFAULT_CHUNKSIZE = 104857600 # 1024 * 1024 B * 100 = 100 MB

140_MAX_MULTIPART_SIZE = 8388608 # 8 MB

141

142_logger = logging.getLogger(__name__)

143

144

145class Blob(_PropertyMixin):

146 """A wrapper around Cloud Storage's concept of an ``Object``.

147

148 :type name: str

149 :param name: The name of the blob. This corresponds to the unique path of

150 the object in the bucket. If bytes, will be converted to a

151 unicode object. Blob / object names can contain any sequence

152 of valid unicode characters, of length 1-1024 bytes when

153 UTF-8 encoded.

154

155 :type bucket: :class:`google.cloud.storage.bucket.Bucket`

156 :param bucket: The bucket to which this blob belongs.

157

158 :type chunk_size: int

159 :param chunk_size:

160 (Optional) The size of a chunk of data whenever iterating (in bytes).

161 This must be a multiple of 256 KB per the API specification. If not

162 specified, the chunk_size of the blob itself is used. If that is not

163 specified, a default value of 40 MB is used.

164

165 :type encryption_key: bytes

166 :param encryption_key:

167 (Optional) 32 byte encryption key for customer-supplied encryption.

168 See https://cloud.google.com/storage/docs/encryption#customer-supplied.

169

170 :type kms_key_name: str

171 :param kms_key_name:

172 (Optional) Resource name of Cloud KMS key used to encrypt the blob's

173 contents.

174

175 :type generation: long

176 :param generation:

177 (Optional) If present, selects a specific revision of this object.

178 """

179

180 _chunk_size = None # Default value for each instance.

181 _CHUNK_SIZE_MULTIPLE = 256 * 1024

182 """Number (256 KB, in bytes) that must divide the chunk size."""

183

184 STORAGE_CLASSES = (

185 STANDARD_STORAGE_CLASS,

186 NEARLINE_STORAGE_CLASS,

187 COLDLINE_STORAGE_CLASS,

188 ARCHIVE_STORAGE_CLASS,

189 MULTI_REGIONAL_LEGACY_STORAGE_CLASS,

190 REGIONAL_LEGACY_STORAGE_CLASS,

191 )

192 """Allowed values for :attr:`storage_class`.

193

194 See

195 https://cloud.google.com/storage/docs/json_api/v1/objects#storageClass

196 https://cloud.google.com/storage/docs/per-object-storage-class

197

198 .. note::

199 This list does not include 'DURABLE_REDUCED_AVAILABILITY', which

200 is only documented for buckets (and deprecated).

201 """

202

203 def __init__(

204 self,

205 name,

206 bucket,

207 chunk_size=None,

208 encryption_key=None,

209 kms_key_name=None,

210 generation=None,

211 ):

212 """

213 property :attr:`name`

214 Get the blob's name.

215 """

216 name = _bytes_to_unicode(name)

217 super(Blob, self).__init__(name=name)

218

219 self.chunk_size = chunk_size # Check that setter accepts value.

220 self._bucket = bucket

221 self._acl = ObjectACL(self)

222 _raise_if_more_than_one_set(

223 encryption_key=encryption_key, kms_key_name=kms_key_name

224 )

225

226 self._encryption_key = encryption_key

227

228 if kms_key_name is not None:

229 self._properties["kmsKeyName"] = kms_key_name

230

231 if generation is not None:

232 self._properties["generation"] = generation

233

234 @property

235 def bucket(self):

236 """Bucket which contains the object.

237

238 :rtype: :class:`~google.cloud.storage.bucket.Bucket`

239 :returns: The object's bucket.

240 """

241 return self._bucket

242

243 @property

244 def chunk_size(self):

245 """Get the blob's default chunk size.

246

247 :rtype: int or ``NoneType``

248 :returns: The current blob's chunk size, if it is set.

249 """

250 return self._chunk_size

251

252 @chunk_size.setter

253 def chunk_size(self, value):

254 """Set the blob's default chunk size.

255

256 :type value: int

257 :param value: (Optional) The current blob's chunk size, if it is set.

258

259 :raises: :class:`ValueError` if ``value`` is not ``None`` and is not a

260 multiple of 256 KB.

261 """

262 if value is not None and value > 0 and value % self._CHUNK_SIZE_MULTIPLE != 0:

263 raise ValueError(

264 "Chunk size must be a multiple of %d." % (self._CHUNK_SIZE_MULTIPLE,)

265 )

266 self._chunk_size = value

267

268 @property

269 def encryption_key(self):

270 """Retrieve the customer-supplied encryption key for the object.

271

272 :rtype: bytes or ``NoneType``

273 :returns:

274 The encryption key or ``None`` if no customer-supplied encryption key was used,

275 or the blob's resource has not been loaded from the server.

276 """

277 return self._encryption_key

278

279 @encryption_key.setter

280 def encryption_key(self, value):

281 """Set the blob's encryption key.

282

283 See https://cloud.google.com/storage/docs/encryption#customer-supplied

284

285 To perform a key rotation for an encrypted blob, use :meth:`rewrite`.

286 See https://cloud.google.com/storage/docs/encryption/using-customer-supplied-keys?hl=ca#rotating

287

288 :type value: bytes

289 :param value: 32 byte encryption key for customer-supplied encryption.

290 """

291 self._encryption_key = value

292

293 @staticmethod

294 def path_helper(bucket_path, blob_name):

295 """Relative URL path for a blob.

296

297 :type bucket_path: str

298 :param bucket_path: The URL path for a bucket.

299

300 :type blob_name: str

301 :param blob_name: The name of the blob.

302

303 :rtype: str

304 :returns: The relative URL path for ``blob_name``.

305 """

306 return bucket_path + "/o/" + _quote(blob_name)

307

308 @property

309 def acl(self):

310 """Create our ACL on demand."""

311 return self._acl

312

313 def __repr__(self):

314 if self.bucket:

315 bucket_name = self.bucket.name

316 else:

317 bucket_name = None

318

319 return f"<Blob: {bucket_name}, {self.name}, {self.generation}>"

320

321 @property

322 def path(self):

323 """Getter property for the URL path to this Blob.

324

325 :rtype: str

326 :returns: The URL path to this Blob.

327 """

328 if not self.name:

329 raise ValueError("Cannot determine path without a blob name.")

330

331 return self.path_helper(self.bucket.path, self.name)

332

333 @property

334 def client(self):

335 """The client bound to this blob."""

336 return self.bucket.client

337

338 @property

339 def user_project(self):

340 """Project ID billed for API requests made via this blob.

341

342 Derived from bucket's value.

343

344 :rtype: str

345 """

346 return self.bucket.user_project

347

348 def _encryption_headers(self):

349 """Return any encryption headers needed to fetch the object.

350

351 :rtype: List(Tuple(str, str))

352 :returns: a list of tuples to be passed as headers.

353 """

354 return _get_encryption_headers(self._encryption_key)

355

356 @property

357 def _query_params(self):

358 """Default query parameters."""

359 params = {}

360 if self.generation is not None:

361 params["generation"] = self.generation

362 if self.user_project is not None:

363 params["userProject"] = self.user_project

364 return params

365

366 @property

367 def public_url(self):

368 """The public URL for this blob.

369

370 Use :meth:`make_public` to enable anonymous access via the returned

371 URL.

372

373 :rtype: `string`

374 :returns: The public URL for this blob.

375 """

376 return "{storage_base_url}/{bucket_name}/{quoted_name}".format(

377 storage_base_url=_API_ACCESS_ENDPOINT,

378 bucket_name=self.bucket.name,

379 quoted_name=_quote(self.name, safe=b"/~"),

380 )

381

382 @classmethod

383 def from_string(cls, uri, client=None):

384 """Get a constructor for blob object by URI.

385

386 .. code-block:: python

387

388 from google.cloud import storage

389 from google.cloud.storage.blob import Blob

390 client = storage.Client()

391 blob = Blob.from_string("gs://bucket/object", client=client)

392

393 :type uri: str

394 :param uri: The blob uri pass to get blob object.

395

396 :type client: :class:`~google.cloud.storage.client.Client`

397 :param client:

398 (Optional) The client to use. Application code should

399 *always* pass ``client``.

400

401 :rtype: :class:`google.cloud.storage.blob.Blob`

402 :returns: The blob object created.

403 """

404 from google.cloud.storage.bucket import Bucket

405

406 scheme, netloc, path, query, frag = urlsplit(uri)

407 if scheme != "gs":

408 raise ValueError("URI scheme must be gs")

409

410 bucket = Bucket(client, name=netloc)

411 return cls(path[1:], bucket)

412

413 def generate_signed_url(

414 self,

415 expiration=None,

416 api_access_endpoint=_API_ACCESS_ENDPOINT,

417 method="GET",

418 content_md5=None,

419 content_type=None,

420 response_disposition=None,

421 response_type=None,

422 generation=None,

423 headers=None,

424 query_parameters=None,

425 client=None,

426 credentials=None,

427 version=None,

428 service_account_email=None,

429 access_token=None,

430 virtual_hosted_style=False,

431 bucket_bound_hostname=None,

432 scheme="http",

433 ):

434 """Generates a signed URL for this blob.

435

436 .. note::

437

438 If you are on Google Compute Engine, you can't generate a signed

439 URL using GCE service account.

440 If you'd like to be able to generate a signed URL from GCE,

441 you can use a standard service account from a JSON file rather

442 than a GCE service account.

443

444 If you have a blob that you want to allow access to for a set

445 amount of time, you can use this method to generate a URL that

446 is only valid within a certain time period.

447

448 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-generate-signed-url-v4#storage_generate_signed_url_v4-python).

449

450 This is particularly useful if you don't want publicly

451 accessible blobs, but don't want to require users to explicitly

452 log in.

453

454 If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,

455 ``https`` works only if using a ``CDN``.

456

457 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]

458 :param expiration:

459 Point in time when the signed URL should expire. If a ``datetime``

460 instance is passed without an explicit ``tzinfo`` set, it will be

461 assumed to be ``UTC``.

462

463 :type api_access_endpoint: str

464 :param api_access_endpoint: (Optional) URI base.

465

466 :type method: str

467 :param method: The HTTP verb that will be used when requesting the URL.

468

469 :type content_md5: str

470 :param content_md5:

471 (Optional) The MD5 hash of the object referenced by ``resource``.

472

473 :type content_type: str

474 :param content_type:

475 (Optional) The content type of the object referenced by

476 ``resource``.

477

478 :type response_disposition: str

479 :param response_disposition:

480 (Optional) Content disposition of responses to requests for the

481 signed URL. For example, to enable the signed URL to initiate a

482 file of ``blog.png``, use the value ``'attachment;

483 filename=blob.png'``.

484

485 :type response_type: str

486 :param response_type:

487 (Optional) Content type of responses to requests for the signed

488 URL. Ignored if content_type is set on object/blob metadata.

489

490 :type generation: str

491 :param generation:

492 (Optional) A value that indicates which generation of the resource

493 to fetch.

494

495 :type headers: dict

496 :param headers:

497 (Optional) Additional HTTP headers to be included as part of the

498 signed URLs. See:

499 https://cloud.google.com/storage/docs/xml-api/reference-headers

500 Requests using the signed URL *must* pass the specified header

501 (name and value) with each request for the URL.

502

503 :type query_parameters: dict

504 :param query_parameters:

505 (Optional) Additional query parameters to be included as part of the

506 signed URLs. See:

507 https://cloud.google.com/storage/docs/xml-api/reference-headers#query

508

509 :type client: :class:`~google.cloud.storage.client.Client`

510 :param client:

511 (Optional) The client to use. If not passed, falls back to the

512 ``client`` stored on the blob's bucket.

513

514 :type credentials: :class:`google.auth.credentials.Credentials`

515 :param credentials:

516 (Optional) The authorization credentials to attach to requests.

517 These credentials identify this application to the service. If

518 none are specified, the client will attempt to ascertain the

519 credentials from the environment.

520

521 :type version: str

522 :param version:

523 (Optional) The version of signed credential to create. Must be one

524 of 'v2' | 'v4'.

525

526 :type service_account_email: str

527 :param service_account_email:

528 (Optional) E-mail address of the service account.

529

530 :type access_token: str

531 :param access_token: (Optional) Access token for a service account.

532

533 :type virtual_hosted_style: bool

534 :param virtual_hosted_style:

535 (Optional) If true, then construct the URL relative the bucket's

536 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.

537

538 :type bucket_bound_hostname: str

539 :param bucket_bound_hostname:

540 (Optional) If passed, then construct the URL relative to the

541 bucket-bound hostname. Value can be a bare or with scheme, e.g.,

542 'example.com' or 'http://example.com'. See:

543 https://cloud.google.com/storage/docs/request-endpoints#cname

544

545 :type scheme: str

546 :param scheme:

547 (Optional) If ``bucket_bound_hostname`` is passed as a bare

548 hostname, use this value as the scheme. ``https`` will work only

549 when using a CDN. Defaults to ``"http"``.

550

551 :raises: :exc:`ValueError` when version is invalid.

552 :raises: :exc:`TypeError` when expiration is not a valid type.

553 :raises: :exc:`AttributeError` if credentials is not an instance

554 of :class:`google.auth.credentials.Signing`.

555

556 :rtype: str

557 :returns: A signed URL you can use to access the resource

558 until expiration.

559 """

560 if version is None:

561 version = "v2"

562 elif version not in ("v2", "v4"):

563 raise ValueError("'version' must be either 'v2' or 'v4'")

564

565 quoted_name = _quote(self.name, safe=b"/~")

566

567 # If you are on Google Compute Engine, you can't generate a signed URL

568 # using GCE service account.

569 # See https://github.com/googleapis/google-auth-library-python/issues/50

570 if virtual_hosted_style:

571 api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com"

572 elif bucket_bound_hostname:

573 api_access_endpoint = _bucket_bound_hostname_url(

574 bucket_bound_hostname, scheme

575 )

576 else:

577 resource = f"/{self.bucket.name}/{quoted_name}"

578

579 if virtual_hosted_style or bucket_bound_hostname:

580 resource = f"/{quoted_name}"

581

582 if credentials is None:

583 client = self._require_client(client)

584 credentials = client._credentials

585

586 if version == "v2":

587 helper = generate_signed_url_v2

588 else:

589 helper = generate_signed_url_v4

590

591 if self._encryption_key is not None:

592 encryption_headers = _get_encryption_headers(self._encryption_key)

593 if headers is None:

594 headers = {}

595 if version == "v2":

596 # See: https://cloud.google.com/storage/docs/access-control/signed-urls-v2#about-canonical-extension-headers

597 v2_copy_only = "X-Goog-Encryption-Algorithm"

598 headers[v2_copy_only] = encryption_headers[v2_copy_only]

599 else:

600 headers.update(encryption_headers)

601

602 return helper(

603 credentials,

604 resource=resource,

605 expiration=expiration,

606 api_access_endpoint=api_access_endpoint,

607 method=method.upper(),

608 content_md5=content_md5,

609 content_type=content_type,

610 response_type=response_type,

611 response_disposition=response_disposition,

612 generation=generation,

613 headers=headers,

614 query_parameters=query_parameters,

615 service_account_email=service_account_email,

616 access_token=access_token,

617 )

618

619 def exists(

620 self,

621 client=None,

622 if_etag_match=None,

623 if_etag_not_match=None,

624 if_generation_match=None,

625 if_generation_not_match=None,

626 if_metageneration_match=None,

627 if_metageneration_not_match=None,

628 timeout=_DEFAULT_TIMEOUT,

629 retry=DEFAULT_RETRY,

630 ):

631 """Determines whether or not this blob exists.

632

633 If :attr:`user_project` is set on the bucket, bills the API request

634 to that project.

635

636 :type client: :class:`~google.cloud.storage.client.Client`

637 :param client:

638 (Optional) The client to use. If not passed, falls back to the

639 ``client`` stored on the blob's bucket.

640

641 :type if_etag_match: Union[str, Set[str]]

642 :param if_etag_match:

643 (Optional) See :ref:`using-if-etag-match`

644

645 :type if_etag_not_match: Union[str, Set[str]]

646 :param if_etag_not_match:

647 (Optional) See :ref:`using-if-etag-not-match`

648

649 :type if_generation_match: long

650 :param if_generation_match:

651 (Optional) See :ref:`using-if-generation-match`

652

653 :type if_generation_not_match: long

654 :param if_generation_not_match:

655 (Optional) See :ref:`using-if-generation-not-match`

656

657 :type if_metageneration_match: long

658 :param if_metageneration_match:

659 (Optional) See :ref:`using-if-metageneration-match`

660

661 :type if_metageneration_not_match: long

662 :param if_metageneration_not_match:

663 (Optional) See :ref:`using-if-metageneration-not-match`

664

665 :type timeout: float or tuple

666 :param timeout:

667 (Optional) The amount of time, in seconds, to wait

668 for the server response. See: :ref:`configuring_timeouts`

669

670 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

671 :param retry:

672 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

673

674 :rtype: bool

675 :returns: True if the blob exists in Cloud Storage.

676 """

677 client = self._require_client(client)

678 # We only need the status code (200 or not) so we seek to

679 # minimize the returned payload.

680 query_params = self._query_params

681 query_params["fields"] = "name"

682

683 _add_generation_match_parameters(

684 query_params,

685 if_generation_match=if_generation_match,

686 if_generation_not_match=if_generation_not_match,

687 if_metageneration_match=if_metageneration_match,

688 if_metageneration_not_match=if_metageneration_not_match,

689 )

690

691 headers = {}

692 _add_etag_match_headers(

693 headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match

694 )

695

696 try:

697 # We intentionally pass `_target_object=None` since fields=name

698 # would limit the local properties.

699 client._get_resource(

700 self.path,

701 query_params=query_params,

702 headers=headers,

703 timeout=timeout,

704 retry=retry,

705 _target_object=None,

706 )

707 except NotFound:

708 # NOTE: This will not fail immediately in a batch. However, when

709 # Batch.finish() is called, the resulting `NotFound` will be

710 # raised.

711 return False

712 return True

713

714 def delete(

715 self,

716 client=None,

717 if_generation_match=None,

718 if_generation_not_match=None,

719 if_metageneration_match=None,

720 if_metageneration_not_match=None,

721 timeout=_DEFAULT_TIMEOUT,

722 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

723 ):

724 """Deletes a blob from Cloud Storage.

725

726 If :attr:`user_project` is set on the bucket, bills the API request

727 to that project.

728

729 :type client: :class:`~google.cloud.storage.client.Client`

730 :param client:

731 (Optional) The client to use. If not passed, falls back to the

732 ``client`` stored on the blob's bucket.

733

734 :type if_generation_match: long

735 :param if_generation_match:

736 (Optional) See :ref:`using-if-generation-match`

737

738 :type if_generation_not_match: long

739 :param if_generation_not_match:

740 (Optional) See :ref:`using-if-generation-not-match`

741

742 :type if_metageneration_match: long

743 :param if_metageneration_match:

744 (Optional) See :ref:`using-if-metageneration-match`

745

746 :type if_metageneration_not_match: long

747 :param if_metageneration_not_match:

748 (Optional) See :ref:`using-if-metageneration-not-match`

749

750 :type timeout: float or tuple

751 :param timeout:

752 (Optional) The amount of time, in seconds, to wait

753 for the server response. See: :ref:`configuring_timeouts`

754

755 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

756 :param retry:

757 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

758

759 :raises: :class:`google.cloud.exceptions.NotFound`

760 (propagated from

761 :meth:`google.cloud.storage.bucket.Bucket.delete_blob`).

762 """

763 self.bucket.delete_blob(

764 self.name,

765 client=client,

766 generation=self.generation,

767 timeout=timeout,

768 if_generation_match=if_generation_match,

769 if_generation_not_match=if_generation_not_match,

770 if_metageneration_match=if_metageneration_match,

771 if_metageneration_not_match=if_metageneration_not_match,

772 retry=retry,

773 )

774

775 def _get_transport(self, client):

776 """Return the client's transport.

777

778 :type client: :class:`~google.cloud.storage.client.Client`

779 :param client:

780 (Optional) The client to use. If not passed, falls back to the

781 ``client`` stored on the blob's bucket.

782

783 :rtype transport:

784 :class:`~google.auth.transport.requests.AuthorizedSession`

785 :returns: The transport (with credentials) that will

786 make authenticated requests.

787 """

788 client = self._require_client(client)

789 return client._http

790

791 def _get_download_url(

792 self,

793 client,

794 if_generation_match=None,

795 if_generation_not_match=None,

796 if_metageneration_match=None,

797 if_metageneration_not_match=None,

798 ):

799 """Get the download URL for the current blob.

800

801 If the ``media_link`` has been loaded, it will be used, otherwise

802 the URL will be constructed from the current blob's path (and possibly

803 generation) to avoid a round trip.

804

805 :type client: :class:`~google.cloud.storage.client.Client`

806 :param client: The client to use.

807

808 :type if_generation_match: long

809 :param if_generation_match:

810 (Optional) See :ref:`using-if-generation-match`

811

812 :type if_generation_not_match: long

813 :param if_generation_not_match:

814 (Optional) See :ref:`using-if-generation-not-match`

815

816 :type if_metageneration_match: long

817 :param if_metageneration_match:

818 (Optional) See :ref:`using-if-metageneration-match`

819

820 :type if_metageneration_not_match: long

821 :param if_metageneration_not_match:

822 (Optional) See :ref:`using-if-metageneration-not-match`

823

824 :rtype: str

825 :returns: The download URL for the current blob.

826 """

827 name_value_pairs = []

828 if self.media_link is None:

829 hostname = _get_host_name(client._connection)

830 base_url = _DOWNLOAD_URL_TEMPLATE.format(

831 hostname=hostname, path=self.path, api_version=_API_VERSION

832 )

833 if self.generation is not None:

834 name_value_pairs.append(("generation", f"{self.generation:d}"))

835 else:

836 base_url = self.media_link

837

838 if self.user_project is not None:

839 name_value_pairs.append(("userProject", self.user_project))

840

841 _add_generation_match_parameters(

842 name_value_pairs,

843 if_generation_match=if_generation_match,

844 if_generation_not_match=if_generation_not_match,

845 if_metageneration_match=if_metageneration_match,

846 if_metageneration_not_match=if_metageneration_not_match,

847 )

848 return _add_query_parameters(base_url, name_value_pairs)

849

850 def _extract_headers_from_download(self, response):

851 """Extract headers from a non-chunked request's http object.

852

853 This avoids the need to make a second request for commonly used

854 headers.

855

856 :type response:

857 :class requests.models.Response

858 :param response: The server response from downloading a non-chunked file

859 """

860 self._properties["contentEncoding"] = response.headers.get(

861 "Content-Encoding", None

862 )

863 self._properties[_CONTENT_TYPE_FIELD] = response.headers.get(

864 "Content-Type", None

865 )

866 self._properties["cacheControl"] = response.headers.get("Cache-Control", None)

867 self._properties["storageClass"] = response.headers.get(

868 "X-Goog-Storage-Class", None

869 )

870 self._properties["contentLanguage"] = response.headers.get(

871 "Content-Language", None

872 )

873 self._properties["etag"] = response.headers.get("ETag", None)

874 self._properties["generation"] = response.headers.get("X-goog-generation", None)

875 self._properties["metageneration"] = response.headers.get(

876 "X-goog-metageneration", None

877 )

878 # 'X-Goog-Hash': 'crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==',

879 x_goog_hash = response.headers.get("X-Goog-Hash", "")

880

881 if x_goog_hash:

882 digests = {}

883 for encoded_digest in x_goog_hash.split(","):

884 match = re.match(r"(crc32c|md5)=([\w\d/\+/]+={0,3})", encoded_digest)

885 if match:

886 method, digest = match.groups()

887 digests[method] = digest

888

889 self._properties["crc32c"] = digests.get("crc32c", None)

890 self._properties["md5Hash"] = digests.get("md5", None)

891

892 def _do_download(

893 self,

894 transport,

895 file_obj,

896 download_url,

897 headers,

898 start=None,

899 end=None,

900 raw_download=False,

901 timeout=_DEFAULT_TIMEOUT,

902 checksum="md5",

903 retry=None,

904 ):

905 """Perform a download without any error handling.

906

907 This is intended to be called by :meth:`download_to_file` so it can

908 be wrapped with error handling / remapping.

909

910 :type transport:

911 :class:`~google.auth.transport.requests.AuthorizedSession`

912 :param transport:

913 The transport (with credentials) that will make authenticated

914 requests.

915

916 :type file_obj: file

917 :param file_obj: A file handle to which to write the blob's data.

918

919 :type download_url: str

920 :param download_url: The URL where the media can be accessed.

921

922 :type headers: dict

923 :param headers: Headers to be sent with the request(s).

924

925 :type start: int

926 :param start: (Optional) The first byte in a range to be downloaded.

927

928 :type end: int

929 :param end: (Optional) The last byte in a range to be downloaded.

930

931 :type raw_download: bool

932 :param raw_download:

933 (Optional) If true, download the object without any expansion.

934

935 :type timeout: float or tuple

936 :param timeout:

937 (Optional) The amount of time, in seconds, to wait

938 for the server response. See: :ref:`configuring_timeouts`

939

940 :type checksum: str

941 :param checksum:

942 (Optional) The type of checksum to compute to verify the integrity

943 of the object. The response headers must contain a checksum of the

944 requested type. If the headers lack an appropriate checksum (for

945 instance in the case of transcoded or ranged downloads where the

946 remote service does not know the correct checksum, including

947 downloads where chunk_size is set) an INFO-level log will be

948 emitted. Supported values are "md5", "crc32c" and None. The default

949 is "md5".

950

951 :type retry: google.api_core.retry.Retry

952 :param retry: (Optional) How to retry the RPC. A None value will disable

953 retries. A google.api_core.retry.Retry value will enable retries,

954 and the object will configure backoff and timeout options. Custom

955 predicates (customizable error codes) are not supported for media

956 operations such as this one.

957

958 This private method does not accept ConditionalRetryPolicy values

959 because the information necessary to evaluate the policy is instead

960 evaluated in client.download_blob_to_file().

961

962 See the retry.py source code and docstrings in this package

963 (google.cloud.storage.retry) for information on retry types and how

964 to configure them.

965 """

966

967 retry_strategy = _api_core_retry_to_resumable_media_retry(retry)

968

969 if self.chunk_size is None:

970 if raw_download:

971 klass = RawDownload

972 else:

973 klass = Download

974

975 download = klass(

976 download_url,

977 stream=file_obj,

978 headers=headers,

979 start=start,

980 end=end,

981 checksum=checksum,

982 )

983 download._retry_strategy = retry_strategy

984 response = download.consume(transport, timeout=timeout)

985 self._extract_headers_from_download(response)

986 else:

987

988 if checksum:

989 msg = _CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE.format(checksum)

990 _logger.info(msg)

991

992 if raw_download:

993 klass = RawChunkedDownload

994 else:

995 klass = ChunkedDownload

996

997 download = klass(

998 download_url,

999 self.chunk_size,

1000 file_obj,

1001 headers=headers,

1002 start=start if start else 0,

1003 end=end,

1004 )

1005

1006 download._retry_strategy = retry_strategy

1007 while not download.finished:

1008 download.consume_next_chunk(transport, timeout=timeout)

1009

1010 def download_to_file(

1011 self,

1012 file_obj,

1013 client=None,

1014 start=None,

1015 end=None,

1016 raw_download=False,

1017 if_etag_match=None,

1018 if_etag_not_match=None,

1019 if_generation_match=None,

1020 if_generation_not_match=None,

1021 if_metageneration_match=None,

1022 if_metageneration_not_match=None,

1023 timeout=_DEFAULT_TIMEOUT,

1024 checksum="md5",

1025 retry=DEFAULT_RETRY,

1026 ):

1027 """Download the contents of this blob into a file-like object.

1028

1029 .. note::

1030

1031 If the server-set property, :attr:`media_link`, is not yet

1032 initialized, makes an additional API request to load it.

1033

1034 If the :attr:`chunk_size` of a current blob is `None`, will download data

1035 in single download request otherwise it will download the :attr:`chunk_size`

1036 of data in each request.

1037

1038 For more fine-grained control over the download process, check out

1039 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html).

1040 For example, this library allows downloading **parts** of a blob rather than the whole thing.

1041

1042 If :attr:`user_project` is set on the bucket, bills the API request

1043 to that project.

1044

1045 :type file_obj: file

1046 :param file_obj: A file handle to which to write the blob's data.

1047

1048 :type client: :class:`~google.cloud.storage.client.Client`

1049 :param client:

1050 (Optional) The client to use. If not passed, falls back to the

1051 ``client`` stored on the blob's bucket.

1052

1053 :type start: int

1054 :param start: (Optional) The first byte in a range to be downloaded.

1055

1056 :type end: int

1057 :param end: (Optional) The last byte in a range to be downloaded.

1058

1059 :type raw_download: bool

1060 :param raw_download:

1061 (Optional) If true, download the object without any expansion.

1062

1063 :type if_etag_match: Union[str, Set[str]]

1064 :param if_etag_match:

1065 (Optional) See :ref:`using-if-etag-match`

1066

1067 :type if_etag_not_match: Union[str, Set[str]]

1068 :param if_etag_not_match:

1069 (Optional) See :ref:`using-if-etag-not-match`

1070

1071 :type if_generation_match: long

1072 :param if_generation_match:

1073 (Optional) See :ref:`using-if-generation-match`

1074

1075 :type if_generation_not_match: long

1076 :param if_generation_not_match:

1077 (Optional) See :ref:`using-if-generation-not-match`

1078

1079 :type if_metageneration_match: long

1080 :param if_metageneration_match:

1081 (Optional) See :ref:`using-if-metageneration-match`

1082

1083 :type if_metageneration_not_match: long

1084 :param if_metageneration_not_match:

1085 (Optional) See :ref:`using-if-metageneration-not-match`

1086

1087 :type timeout: float or tuple

1088 :param timeout:

1089 (Optional) The amount of time, in seconds, to wait

1090 for the server response. See: :ref:`configuring_timeouts`

1091

1092 :type checksum: str

1093 :param checksum:

1094 (Optional) The type of checksum to compute to verify the integrity

1095 of the object. The response headers must contain a checksum of the

1096 requested type. If the headers lack an appropriate checksum (for

1097 instance in the case of transcoded or ranged downloads where the

1098 remote service does not know the correct checksum, including

1099 downloads where chunk_size is set) an INFO-level log will be

1100 emitted. Supported values are "md5", "crc32c" and None. The default

1101 is "md5".

1102

1103 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

1104 :param retry: (Optional) How to retry the RPC. A None value will disable

1105 retries. A google.api_core.retry.Retry value will enable retries,

1106 and the object will define retriable response codes and errors and

1107 configure backoff and timeout options.

1108

1109 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

1110 Retry object and activates it only if certain conditions are met.

1111 This class exists to provide safe defaults for RPC calls that are

1112 not technically safe to retry normally (due to potential data

1113 duplication or other side-effects) but become safe to retry if a

1114 condition such as if_metageneration_match is set.

1115

1116 See the retry.py source code and docstrings in this package

1117 (google.cloud.storage.retry) for information on retry types and how

1118 to configure them.

1119

1120 Media operations (downloads and uploads) do not support non-default

1121 predicates in a Retry object. The default will always be used. Other

1122 configuration changes for Retry objects such as delays and deadlines

1123 are respected.

1124

1125 :raises: :class:`google.cloud.exceptions.NotFound`

1126 """

1127 client = self._require_client(client)

1128

1129 client.download_blob_to_file(

1130 self,

1131 file_obj=file_obj,

1132 start=start,

1133 end=end,

1134 raw_download=raw_download,

1135 if_etag_match=if_etag_match,

1136 if_etag_not_match=if_etag_not_match,

1137 if_generation_match=if_generation_match,

1138 if_generation_not_match=if_generation_not_match,

1139 if_metageneration_match=if_metageneration_match,

1140 if_metageneration_not_match=if_metageneration_not_match,

1141 timeout=timeout,

1142 checksum=checksum,

1143 retry=retry,

1144 )

1145

1146 def download_to_filename(

1147 self,

1148 filename,

1149 client=None,

1150 start=None,

1151 end=None,

1152 raw_download=False,

1153 if_etag_match=None,

1154 if_etag_not_match=None,

1155 if_generation_match=None,

1156 if_generation_not_match=None,

1157 if_metageneration_match=None,

1158 if_metageneration_not_match=None,

1159 timeout=_DEFAULT_TIMEOUT,

1160 checksum="md5",

1161 retry=DEFAULT_RETRY,

1162 ):

1163 """Download the contents of this blob into a named file.

1164

1165 If :attr:`user_project` is set on the bucket, bills the API request

1166 to that project.

1167

1168 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-download-encrypted-file#storage_download_encrypted_file-python)

1169 to download a file with a [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied).

1170

1171 :type filename: str

1172 :param filename: A filename to be passed to ``open``.

1173

1174 :type client: :class:`~google.cloud.storage.client.Client`

1175 :param client:

1176 (Optional) The client to use. If not passed, falls back to the

1177 ``client`` stored on the blob's bucket.

1178

1179 :type start: int

1180 :param start: (Optional) The first byte in a range to be downloaded.

1181

1182 :type end: int

1183 :param end: (Optional) The last byte in a range to be downloaded.

1184

1185 :type raw_download: bool

1186 :param raw_download:

1187 (Optional) If true, download the object without any expansion.

1188

1189 :type if_etag_match: Union[str, Set[str]]

1190 :param if_etag_match:

1191 (Optional) See :ref:`using-if-etag-match`

1192

1193 :type if_etag_not_match: Union[str, Set[str]]

1194 :param if_etag_not_match:

1195 (Optional) See :ref:`using-if-etag-not-match`

1196

1197 :type if_generation_match: long

1198 :param if_generation_match:

1199 (Optional) See :ref:`using-if-generation-match`

1200

1201 :type if_generation_not_match: long

1202 :param if_generation_not_match:

1203 (Optional) See :ref:`using-if-generation-not-match`

1204

1205 :type if_metageneration_match: long

1206 :param if_metageneration_match:

1207 (Optional) See :ref:`using-if-metageneration-match`

1208

1209 :type if_metageneration_not_match: long

1210 :param if_metageneration_not_match:

1211 (Optional) See :ref:`using-if-metageneration-not-match`

1212

1213 :type timeout: float or tuple

1214 :param timeout:

1215 (Optional) The amount of time, in seconds, to wait

1216 for the server response. See: :ref:`configuring_timeouts`

1217

1218 :type checksum: str

1219 :param checksum:

1220 (Optional) The type of checksum to compute to verify the integrity

1221 of the object. The response headers must contain a checksum of the

1222 requested type. If the headers lack an appropriate checksum (for

1223 instance in the case of transcoded or ranged downloads where the

1224 remote service does not know the correct checksum, including

1225 downloads where chunk_size is set) an INFO-level log will be

1226 emitted. Supported values are "md5", "crc32c" and None. The default

1227 is "md5".

1228

1229 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

1230 :param retry: (Optional) How to retry the RPC. A None value will disable

1231 retries. A google.api_core.retry.Retry value will enable retries,

1232 and the object will define retriable response codes and errors and

1233 configure backoff and timeout options.

1234

1235 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

1236 Retry object and activates it only if certain conditions are met.

1237 This class exists to provide safe defaults for RPC calls that are

1238 not technically safe to retry normally (due to potential data

1239 duplication or other side-effects) but become safe to retry if a

1240 condition such as if_metageneration_match is set.

1241

1242 See the retry.py source code and docstrings in this package

1243 (google.cloud.storage.retry) for information on retry types and how

1244 to configure them.

1245

1246 Media operations (downloads and uploads) do not support non-default

1247 predicates in a Retry object. The default will always be used. Other

1248 configuration changes for Retry objects such as delays and deadlines

1249 are respected.

1250

1251 :raises: :class:`google.cloud.exceptions.NotFound`

1252 """

1253 client = self._require_client(client)

1254 try:

1255 with open(filename, "wb") as file_obj:

1256 client.download_blob_to_file(

1257 self,

1258 file_obj,

1259 start=start,

1260 end=end,

1261 raw_download=raw_download,

1262 if_etag_match=if_etag_match,

1263 if_etag_not_match=if_etag_not_match,

1264 if_generation_match=if_generation_match,

1265 if_generation_not_match=if_generation_not_match,

1266 if_metageneration_match=if_metageneration_match,

1267 if_metageneration_not_match=if_metageneration_not_match,

1268 timeout=timeout,

1269 checksum=checksum,

1270 retry=retry,

1271 )

1272 except resumable_media.DataCorruption:

1273 # Delete the corrupt downloaded file.

1274 os.remove(filename)

1275 raise

1276

1277 updated = self.updated

1278 if updated is not None:

1279 mtime = updated.timestamp()

1280 os.utime(file_obj.name, (mtime, mtime))

1281

1282 def download_as_bytes(

1283 self,

1284 client=None,

1285 start=None,

1286 end=None,

1287 raw_download=False,

1288 if_etag_match=None,

1289 if_etag_not_match=None,

1290 if_generation_match=None,

1291 if_generation_not_match=None,

1292 if_metageneration_match=None,

1293 if_metageneration_not_match=None,

1294 timeout=_DEFAULT_TIMEOUT,

1295 checksum="md5",

1296 retry=DEFAULT_RETRY,

1297 ):

1298 """Download the contents of this blob as a bytes object.

1299

1300 If :attr:`user_project` is set on the bucket, bills the API request

1301 to that project.

1302

1303 :type client: :class:`~google.cloud.storage.client.Client`

1304 :param client:

1305 (Optional) The client to use. If not passed, falls back to the

1306 ``client`` stored on the blob's bucket.

1307

1308 :type start: int

1309 :param start: (Optional) The first byte in a range to be downloaded.

1310

1311 :type end: int

1312 :param end: (Optional) The last byte in a range to be downloaded.

1313

1314 :type raw_download: bool

1315 :param raw_download:

1316 (Optional) If true, download the object without any expansion.

1317

1318 :type if_etag_match: Union[str, Set[str]]

1319 :param if_etag_match:

1320 (Optional) See :ref:`using-if-etag-match`

1321

1322 :type if_etag_not_match: Union[str, Set[str]]

1323 :param if_etag_not_match:

1324 (Optional) See :ref:`using-if-etag-not-match`

1325

1326 :type if_generation_match: long

1327 :param if_generation_match:

1328 (Optional) See :ref:`using-if-generation-match`

1329

1330 :type if_generation_not_match: long

1331 :param if_generation_not_match:

1332 (Optional) See :ref:`using-if-generation-not-match`

1333

1334 :type if_metageneration_match: long

1335 :param if_metageneration_match:

1336 (Optional) See :ref:`using-if-metageneration-match`

1337

1338 :type if_metageneration_not_match: long

1339 :param if_metageneration_not_match:

1340 (Optional) See :ref:`using-if-metageneration-not-match`

1341

1342 :type timeout: float or tuple

1343 :param timeout:

1344 (Optional) The amount of time, in seconds, to wait

1345 for the server response. See: :ref:`configuring_timeouts`

1346

1347 :type checksum: str

1348 :param checksum:

1349 (Optional) The type of checksum to compute to verify the integrity

1350 of the object. The response headers must contain a checksum of the

1351 requested type. If the headers lack an appropriate checksum (for

1352 instance in the case of transcoded or ranged downloads where the

1353 remote service does not know the correct checksum, including

1354 downloads where chunk_size is set) an INFO-level log will be

1355 emitted. Supported values are "md5", "crc32c" and None. The default

1356 is "md5".

1357

1358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

1359 :param retry: (Optional) How to retry the RPC. A None value will disable

1360 retries. A google.api_core.retry.Retry value will enable retries,

1361 and the object will define retriable response codes and errors and

1362 configure backoff and timeout options.

1363

1364 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

1365 Retry object and activates it only if certain conditions are met.

1366 This class exists to provide safe defaults for RPC calls that are

1367 not technically safe to retry normally (due to potential data

1368 duplication or other side-effects) but become safe to retry if a

1369 condition such as if_metageneration_match is set.

1370

1371 See the retry.py source code and docstrings in this package

1372 (google.cloud.storage.retry) for information on retry types and how

1373 to configure them.

1374

1375 Media operations (downloads and uploads) do not support non-default

1376 predicates in a Retry object. The default will always be used. Other

1377 configuration changes for Retry objects such as delays and deadlines

1378 are respected.

1379

1380 :rtype: bytes

1381 :returns: The data stored in this blob.

1382

1383 :raises: :class:`google.cloud.exceptions.NotFound`

1384 """

1385 client = self._require_client(client)

1386 string_buffer = BytesIO()

1387 client.download_blob_to_file(

1388 self,

1389 string_buffer,

1390 start=start,

1391 end=end,

1392 raw_download=raw_download,

1393 if_etag_match=if_etag_match,

1394 if_etag_not_match=if_etag_not_match,

1395 if_generation_match=if_generation_match,

1396 if_generation_not_match=if_generation_not_match,

1397 if_metageneration_match=if_metageneration_match,

1398 if_metageneration_not_match=if_metageneration_not_match,

1399 timeout=timeout,

1400 checksum=checksum,

1401 retry=retry,

1402 )

1403 return string_buffer.getvalue()

1404

1405 def download_as_string(

1406 self,

1407 client=None,

1408 start=None,

1409 end=None,

1410 raw_download=False,

1411 if_etag_match=None,

1412 if_etag_not_match=None,

1413 if_generation_match=None,

1414 if_generation_not_match=None,

1415 if_metageneration_match=None,

1416 if_metageneration_not_match=None,

1417 timeout=_DEFAULT_TIMEOUT,

1418 retry=DEFAULT_RETRY,

1419 ):

1420 """(Deprecated) Download the contents of this blob as a bytes object.

1421

1422 If :attr:`user_project` is set on the bucket, bills the API request

1423 to that project.

1424

1425 .. note::

1426 Deprecated alias for :meth:`download_as_bytes`.

1427

1428 :type client: :class:`~google.cloud.storage.client.Client`

1429 :param client:

1430 (Optional) The client to use. If not passed, falls back to the

1431 ``client`` stored on the blob's bucket.

1432

1433 :type start: int

1434 :param start: (Optional) The first byte in a range to be downloaded.

1435

1436 :type end: int

1437 :param end: (Optional) The last byte in a range to be downloaded.

1438

1439 :type raw_download: bool

1440 :param raw_download:

1441 (Optional) If true, download the object without any expansion.

1442

1443 :type if_etag_match: Union[str, Set[str]]

1444 :param if_etag_match:

1445 (Optional) See :ref:`using-if-etag-match`

1446

1447 :type if_etag_not_match: Union[str, Set[str]]

1448 :param if_etag_not_match:

1449 (Optional) See :ref:`using-if-etag-not-match`

1450

1451 :type if_generation_match: long

1452 :param if_generation_match:

1453 (Optional) See :ref:`using-if-generation-match`

1454

1455 :type if_generation_not_match: long

1456 :param if_generation_not_match:

1457 (Optional) See :ref:`using-if-generation-not-match`

1458

1459 :type if_metageneration_match: long

1460 :param if_metageneration_match:

1461 (Optional) See :ref:`using-if-metageneration-match`

1462

1463 :type if_metageneration_not_match: long

1464 :param if_metageneration_not_match:

1465 (Optional) See :ref:`using-if-metageneration-not-match`

1466

1467 :type timeout: float or tuple

1468 :param timeout:

1469 (Optional) The amount of time, in seconds, to wait

1470 for the server response. See: :ref:`configuring_timeouts`

1471

1472 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

1473 :param retry: (Optional) How to retry the RPC. A None value will disable

1474 retries. A google.api_core.retry.Retry value will enable retries,

1475 and the object will define retriable response codes and errors and

1476 configure backoff and timeout options.

1477

1478 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

1479 Retry object and activates it only if certain conditions are met.

1480 This class exists to provide safe defaults for RPC calls that are

1481 not technically safe to retry normally (due to potential data

1482 duplication or other side-effects) but become safe to retry if a

1483 condition such as if_metageneration_match is set.

1484

1485 See the retry.py source code and docstrings in this package

1486 (google.cloud.storage.retry) for information on retry types and how

1487 to configure them.

1488

1489 Media operations (downloads and uploads) do not support non-default

1490 predicates in a Retry object. The default will always be used. Other

1491 configuration changes for Retry objects such as delays and deadlines

1492 are respected.

1493

1494 :rtype: bytes

1495 :returns: The data stored in this blob.

1496

1497 :raises: :class:`google.cloud.exceptions.NotFound`

1498 """

1499 warnings.warn(

1500 _DOWNLOAD_AS_STRING_DEPRECATED, PendingDeprecationWarning, stacklevel=2

1501 )

1502 return self.download_as_bytes(

1503 client=client,

1504 start=start,

1505 end=end,

1506 raw_download=raw_download,

1507 if_etag_match=if_etag_match,

1508 if_etag_not_match=if_etag_not_match,

1509 if_generation_match=if_generation_match,

1510 if_generation_not_match=if_generation_not_match,

1511 if_metageneration_match=if_metageneration_match,

1512 if_metageneration_not_match=if_metageneration_not_match,

1513 timeout=timeout,

1514 retry=retry,

1515 )

1516

1517 def download_as_text(

1518 self,

1519 client=None,

1520 start=None,

1521 end=None,

1522 raw_download=False,

1523 encoding=None,

1524 if_etag_match=None,

1525 if_etag_not_match=None,

1526 if_generation_match=None,

1527 if_generation_not_match=None,

1528 if_metageneration_match=None,

1529 if_metageneration_not_match=None,

1530 timeout=_DEFAULT_TIMEOUT,

1531 retry=DEFAULT_RETRY,

1532 ):

1533 """Download the contents of this blob as text (*not* bytes).

1534

1535 If :attr:`user_project` is set on the bucket, bills the API request

1536 to that project.

1537

1538 :type client: :class:`~google.cloud.storage.client.Client`

1539 :param client:

1540 (Optional) The client to use. If not passed, falls back to the

1541 ``client`` stored on the blob's bucket.

1542

1543 :type start: int

1544 :param start: (Optional) The first byte in a range to be downloaded.

1545

1546 :type end: int

1547 :param end: (Optional) The last byte in a range to be downloaded.

1548

1549 :type raw_download: bool

1550 :param raw_download:

1551 (Optional) If true, download the object without any expansion.

1552

1553 :type encoding: str

1554 :param encoding: (Optional) encoding to be used to decode the

1555 downloaded bytes. Defaults to the ``charset`` param of

1556 attr:`content_type`, or else to "utf-8".

1557

1558 :type if_etag_match: Union[str, Set[str]]

1559 :param if_etag_match:

1560 (Optional) See :ref:`using-if-etag-match`

1561

1562 :type if_etag_not_match: Union[str, Set[str]]

1563 :param if_etag_not_match:

1564 (Optional) See :ref:`using-if-etag-not-match`

1565

1566 :type if_generation_match: long

1567 :param if_generation_match:

1568 (Optional) See :ref:`using-if-generation-match`

1569

1570 :type if_generation_not_match: long

1571 :param if_generation_not_match:

1572 (Optional) See :ref:`using-if-generation-not-match`

1573

1574 :type if_metageneration_match: long

1575 :param if_metageneration_match:

1576 (Optional) See :ref:`using-if-metageneration-match`

1577

1578 :type if_metageneration_not_match: long

1579 :param if_metageneration_not_match:

1580 (Optional) See :ref:`using-if-metageneration-not-match`

1581

1582 :type timeout: float or tuple

1583 :param timeout:

1584 (Optional) The amount of time, in seconds, to wait

1585 for the server response. See: :ref:`configuring_timeouts`

1586

1587 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

1588 :param retry: (Optional) How to retry the RPC. A None value will disable

1589 retries. A google.api_core.retry.Retry value will enable retries,

1590 and the object will define retriable response codes and errors and

1591 configure backoff and timeout options.

1592

1593 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

1594 Retry object and activates it only if certain conditions are met.

1595 This class exists to provide safe defaults for RPC calls that are

1596 not technically safe to retry normally (due to potential data

1597 duplication or other side-effects) but become safe to retry if a

1598 condition such as if_metageneration_match is set.

1599

1600 See the retry.py source code and docstrings in this package

1601 (google.cloud.storage.retry) for information on retry types and how

1602 to configure them.

1603

1604 Media operations (downloads and uploads) do not support non-default

1605 predicates in a Retry object. The default will always be used. Other

1606 configuration changes for Retry objects such as delays and deadlines

1607 are respected.

1608

1609 :rtype: text

1610 :returns: The data stored in this blob, decoded to text.

1611 """

1612 data = self.download_as_bytes(

1613 client=client,

1614 start=start,

1615 end=end,

1616 raw_download=raw_download,

1617 if_etag_match=if_etag_match,

1618 if_etag_not_match=if_etag_not_match,

1619 if_generation_match=if_generation_match,

1620 if_generation_not_match=if_generation_not_match,

1621 if_metageneration_match=if_metageneration_match,

1622 if_metageneration_not_match=if_metageneration_not_match,

1623 timeout=timeout,

1624 retry=retry,

1625 )

1626

1627 if encoding is not None:

1628 return data.decode(encoding)

1629

1630 if self.content_type is not None:

1631 msg = HeaderParser().parsestr("Content-Type: " + self.content_type)

1632 params = dict(msg.get_params()[1:])

1633 if "charset" in params:

1634 return data.decode(params["charset"])

1635

1636 return data.decode("utf-8")

1637

1638 def _get_content_type(self, content_type, filename=None):

1639 """Determine the content type from the current object.

1640

1641 The return value will be determined in order of precedence:

1642

1643 - The value passed in to this method (if not :data:`None`)

1644 - The value stored on the current blob

1645 - The default value ('application/octet-stream')

1646

1647 :type content_type: str

1648 :param content_type: (Optional) Type of content.

1649

1650 :type filename: str

1651 :param filename:

1652 (Optional) The name of the file where the content is stored.

1653

1654 :rtype: str

1655 :returns: Type of content gathered from the object.

1656 """

1657 if content_type is None:

1658 content_type = self.content_type

1659

1660 if content_type is None and filename is not None:

1661 content_type, _ = mimetypes.guess_type(filename)

1662

1663 if content_type is None:

1664 content_type = _DEFAULT_CONTENT_TYPE

1665

1666 return content_type

1667

1668 def _get_writable_metadata(self):

1669 """Get the object / blob metadata which is writable.

1670

1671 This is intended to be used when creating a new object / blob.

1672

1673 See the [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects)

1674 for more information, the fields marked as writable are:

1675

1676 * ``acl``

1677 * ``cacheControl``

1678 * ``contentDisposition``

1679 * ``contentEncoding``

1680 * ``contentLanguage``

1681 * ``contentType``

1682 * ``crc32c``

1683 * ``customTime``

1684 * ``md5Hash``

1685 * ``metadata``

1686 * ``name``

1687 * ``storageClass``

1688

1689 For now, we don't support ``acl``, access control lists should be

1690 managed directly through :class:`ObjectACL` methods.

1691 """

1692 # NOTE: This assumes `self.name` is unicode.

1693 object_metadata = {"name": self.name}

1694 for key in self._changes:

1695 if key in _WRITABLE_FIELDS:

1696 object_metadata[key] = self._properties[key]

1697

1698 return object_metadata

1699

1700 def _get_upload_arguments(self, client, content_type):

1701 """Get required arguments for performing an upload.

1702

1703 The content type returned will be determined in order of precedence:

1704

1705 - The value passed in to this method (if not :data:`None`)

1706 - The value stored on the current blob

1707 - The default value ('application/octet-stream')

1708

1709 :type content_type: str

1710 :param content_type: Type of content being uploaded (or :data:`None`).

1711

1712 :rtype: tuple

1713 :returns: A triple of

1714

1715 * A header dictionary

1716 * An object metadata dictionary

1717 * The ``content_type`` as a string (according to precedence)

1718 """

1719 content_type = self._get_content_type(content_type)

1720 headers = {

1721 **_get_default_headers(client._connection.user_agent, content_type),

1722 **_get_encryption_headers(self._encryption_key),

1723 }

1724 object_metadata = self._get_writable_metadata()

1725 return headers, object_metadata, content_type

1726

1727 def _do_multipart_upload(

1728 self,

1729 client,

1730 stream,

1731 content_type,

1732 size,

1733 num_retries,

1734 predefined_acl,

1735 if_generation_match,

1736 if_generation_not_match,

1737 if_metageneration_match,

1738 if_metageneration_not_match,

1739 timeout=_DEFAULT_TIMEOUT,

1740 checksum=None,

1741 retry=None,

1742 ):

1743 """Perform a multipart upload.

1744

1745 The content type of the upload will be determined in order

1746 of precedence:

1747

1748 - The value passed in to this method (if not :data:`None`)

1749 - The value stored on the current blob

1750 - The default value ('application/octet-stream')

1751

1752 :type client: :class:`~google.cloud.storage.client.Client`

1753 :param client:

1754 (Optional) The client to use. If not passed, falls back to the

1755 ``client`` stored on the blob's bucket.

1756

1757 :type stream: IO[bytes]

1758 :param stream: A bytes IO object open for reading.

1759

1760 :type content_type: str

1761 :param content_type: Type of content being uploaded (or :data:`None`).

1762

1763 :type size: int

1764 :param size:

1765 The number of bytes to be uploaded (which will be read from

1766 ``stream``). If not provided, the upload will be concluded once

1767 ``stream`` is exhausted (or :data:`None`).

1768

1769 :type num_retries: int

1770 :param num_retries:

1771 Number of upload retries. By default, only uploads with

1772 if_generation_match set will be retried, as uploads without the

1773 argument are not guaranteed to be idempotent. Setting num_retries

1774 will override this default behavior and guarantee retries even when

1775 if_generation_match is not set. (Deprecated: This argument

1776 will be removed in a future release.)

1777

1778 :type predefined_acl: str

1779 :param predefined_acl: (Optional) Predefined access control list

1780

1781 :type if_generation_match: long

1782 :param if_generation_match:

1783 (Optional) See :ref:`using-if-generation-match`

1784

1785 :type if_generation_not_match: long

1786 :param if_generation_not_match:

1787 (Optional) See :ref:`using-if-generation-not-match`

1788

1789 :type if_metageneration_match: long

1790 :param if_metageneration_match:

1791 (Optional) See :ref:`using-if-metageneration-match`

1792

1793 :type if_metageneration_not_match: long

1794 :param if_metageneration_not_match:

1795 (Optional) See :ref:`using-if-metageneration-not-match`

1796

1797 :type timeout: float or tuple

1798 :param timeout:

1799 (Optional) The amount of time, in seconds, to wait

1800 for the server response. See: :ref:`configuring_timeouts`

1801

1802 :type checksum: str

1803 :param checksum:

1804 (Optional) The type of checksum to compute to verify

1805 the integrity of the object. The request metadata will be amended

1806 to include the computed value. Using this option will override a

1807 manually-set checksum value. Supported values are "md5",

1808 "crc32c" and None. The default is None.

1809

1810 :type retry: google.api_core.retry.Retry

1811 :param retry: (Optional) How to retry the RPC. A None value will disable

1812 retries. A google.api_core.retry.Retry value will enable retries,

1813 and the object will configure backoff and timeout options. Custom

1814 predicates (customizable error codes) are not supported for media

1815 operations such as this one.

1816

1817 This private method does not accept ConditionalRetryPolicy values

1818 because the information necessary to evaluate the policy is instead

1819 evaluated in blob._do_upload().

1820

1821 See the retry.py source code and docstrings in this package

1822 (google.cloud.storage.retry) for information on retry types and how

1823 to configure them.

1824

1825 :rtype: :class:`~requests.Response`

1826 :returns: The "200 OK" response object returned after the multipart

1827 upload request.

1828 :raises: :exc:`ValueError` if ``size`` is not :data:`None` but the

1829 ``stream`` has fewer than ``size`` bytes remaining.

1830 """

1831 if size is None:

1832 data = stream.read()

1833 else:

1834 data = stream.read(size)

1835 if len(data) < size:

1836 msg = _READ_LESS_THAN_SIZE.format(size, len(data))

1837 raise ValueError(msg)

1838

1839 client = self._require_client(client)

1840 transport = self._get_transport(client)

1841 if "metadata" in self._properties and "metadata" not in self._changes:

1842 self._changes.add("metadata")

1843 info = self._get_upload_arguments(client, content_type)

1844 headers, object_metadata, content_type = info

1845

1846 hostname = _get_host_name(client._connection)

1847 base_url = _MULTIPART_URL_TEMPLATE.format(

1848 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION

1849 )

1850 name_value_pairs = []

1851

1852 if self.user_project is not None:

1853 name_value_pairs.append(("userProject", self.user_project))

1854

1855 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object

1856 # at rest, object resource metadata will store the version of the Key Management

1857 # Service cryptographic material. If a Blob instance with KMS Key metadata set is

1858 # used to upload a new version of the object then the existing kmsKeyName version

1859 # value can't be used in the upload request and the client instead ignores it.

1860 if (

1861 self.kms_key_name is not None

1862 and "cryptoKeyVersions" not in self.kms_key_name

1863 ):

1864 name_value_pairs.append(("kmsKeyName", self.kms_key_name))

1865

1866 if predefined_acl is not None:

1867 name_value_pairs.append(("predefinedAcl", predefined_acl))

1868

1869 if if_generation_match is not None:

1870 name_value_pairs.append(("ifGenerationMatch", if_generation_match))

1871

1872 if if_generation_not_match is not None:

1873 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match))

1874

1875 if if_metageneration_match is not None:

1876 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match))

1877

1878 if if_metageneration_not_match is not None:

1879 name_value_pairs.append(

1880 ("ifMetaGenerationNotMatch", if_metageneration_not_match)

1881 )

1882

1883 upload_url = _add_query_parameters(base_url, name_value_pairs)

1884 upload = MultipartUpload(upload_url, headers=headers, checksum=checksum)

1885

1886 upload._retry_strategy = _api_core_retry_to_resumable_media_retry(

1887 retry, num_retries

1888 )

1889

1890 response = upload.transmit(

1891 transport, data, object_metadata, content_type, timeout=timeout

1892 )

1893

1894 return response

1895

1896 def _initiate_resumable_upload(

1897 self,

1898 client,

1899 stream,

1900 content_type,

1901 size,

1902 num_retries,

1903 predefined_acl=None,

1904 extra_headers=None,

1905 chunk_size=None,

1906 if_generation_match=None,

1907 if_generation_not_match=None,

1908 if_metageneration_match=None,

1909 if_metageneration_not_match=None,

1910 timeout=_DEFAULT_TIMEOUT,

1911 checksum=None,

1912 retry=None,

1913 ):

1914 """Initiate a resumable upload.

1915

1916 The content type of the upload will be determined in order

1917 of precedence:

1918

1919 - The value passed in to this method (if not :data:`None`)

1920 - The value stored on the current blob

1921 - The default value ('application/octet-stream')

1922

1923 :type client: :class:`~google.cloud.storage.client.Client`

1924 :param client:

1925 (Optional) The client to use. If not passed, falls back to the

1926 ``client`` stored on the blob's bucket.

1927

1928 :type stream: IO[bytes]

1929 :param stream: A bytes IO object open for reading.

1930

1931 :type content_type: str

1932 :param content_type: Type of content being uploaded (or :data:`None`).

1933

1934 :type size: int

1935 :param size:

1936 The number of bytes to be uploaded (which will be read from

1937 ``stream``). If not provided, the upload will be concluded once

1938 ``stream`` is exhausted (or :data:`None`).

1939

1940 :type predefined_acl: str

1941 :param predefined_acl: (Optional) Predefined access control list

1942

1943 :type num_retries: int

1944 :param num_retries:

1945 Number of upload retries. By default, only uploads with

1946 if_generation_match set will be retried, as uploads without the

1947 argument are not guaranteed to be idempotent. Setting num_retries

1948 will override this default behavior and guarantee retries even when

1949 if_generation_match is not set. (Deprecated: This argument

1950 will be removed in a future release.)

1951

1952 :type extra_headers: dict

1953 :param extra_headers:

1954 (Optional) Extra headers to add to standard headers.

1955

1956 :type chunk_size: int

1957 :param chunk_size:

1958 (Optional) Chunk size to use when creating a

1959 :class:`~google.resumable_media.requests.ResumableUpload`.

1960 If not passed, will fall back to the chunk size on the

1961 current blob, if the chunk size of a current blob is also

1962 `None`, will set the default value.

1963 The default value of ``chunk_size`` is 100 MB.

1964

1965 :type if_generation_match: long

1966 :param if_generation_match:

1967 (Optional) See :ref:`using-if-generation-match`

1968

1969 :type if_generation_not_match: long

1970 :param if_generation_not_match:

1971 (Optional) See :ref:`using-if-generation-not-match`

1972

1973 :type if_metageneration_match: long

1974 :param if_metageneration_match:

1975 (Optional) See :ref:`using-if-metageneration-match`

1976

1977 :type if_metageneration_not_match: long

1978 :param if_metageneration_not_match:

1979 (Optional) See :ref:`using-if-metageneration-not-match`

1980

1981 :type timeout: float or tuple

1982 :param timeout:

1983 (Optional) The amount of time, in seconds, to wait

1984 for the server response. See: :ref:`configuring_timeouts`

1985

1986 :type checksum: str

1987 :param checksum:

1988 (Optional) The type of checksum to compute to verify

1989 the integrity of the object. After the upload is complete, the

1990 server-computed checksum of the resulting object will be checked

1991 and google.resumable_media.common.DataCorruption will be raised on

1992 a mismatch. On a validation failure, the client will attempt to

1993 delete the uploaded object automatically. Supported values

1994 are "md5", "crc32c" and None. The default is None.

1995

1996 :type retry: google.api_core.retry.Retry

1997 :param retry: (Optional) How to retry the RPC. A None value will disable

1998 retries. A google.api_core.retry.Retry value will enable retries,

1999 and the object will configure backoff and timeout options. Custom

2000 predicates (customizable error codes) are not supported for media

2001 operations such as this one.

2002

2003 This private method does not accept ConditionalRetryPolicy values

2004 because the information necessary to evaluate the policy is instead

2005 evaluated in blob._do_upload().

2006

2007 See the retry.py source code and docstrings in this package

2008 (google.cloud.storage.retry) for information on retry types and how

2009 to configure them.

2010

2011 :rtype: tuple

2012 :returns:

2013 Pair of

2014

2015 * The :class:`~google.resumable_media.requests.ResumableUpload`

2016 that was created

2017 * The ``transport`` used to initiate the upload.

2018 """

2019 client = self._require_client(client)

2020 if chunk_size is None:

2021 chunk_size = self.chunk_size

2022 if chunk_size is None:

2023 chunk_size = _DEFAULT_CHUNKSIZE

2024

2025 transport = self._get_transport(client)

2026 if "metadata" in self._properties and "metadata" not in self._changes:

2027 self._changes.add("metadata")

2028 info = self._get_upload_arguments(client, content_type)

2029 headers, object_metadata, content_type = info

2030 if extra_headers is not None:

2031 headers.update(extra_headers)

2032

2033 hostname = _get_host_name(client._connection)

2034 base_url = _RESUMABLE_URL_TEMPLATE.format(

2035 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION

2036 )

2037 name_value_pairs = []

2038

2039 if self.user_project is not None:

2040 name_value_pairs.append(("userProject", self.user_project))

2041

2042 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object

2043 # at rest, object resource metadata will store the version of the Key Management

2044 # Service cryptographic material. If a Blob instance with KMS Key metadata set is

2045 # used to upload a new version of the object then the existing kmsKeyName version

2046 # value can't be used in the upload request and the client instead ignores it.

2047 if (

2048 self.kms_key_name is not None

2049 and "cryptoKeyVersions" not in self.kms_key_name

2050 ):

2051 name_value_pairs.append(("kmsKeyName", self.kms_key_name))

2052

2053 if predefined_acl is not None:

2054 name_value_pairs.append(("predefinedAcl", predefined_acl))

2055

2056 if if_generation_match is not None:

2057 name_value_pairs.append(("ifGenerationMatch", if_generation_match))

2058

2059 if if_generation_not_match is not None:

2060 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match))

2061

2062 if if_metageneration_match is not None:

2063 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match))

2064

2065 if if_metageneration_not_match is not None:

2066 name_value_pairs.append(

2067 ("ifMetaGenerationNotMatch", if_metageneration_not_match)

2068 )

2069

2070 upload_url = _add_query_parameters(base_url, name_value_pairs)

2071 upload = ResumableUpload(

2072 upload_url, chunk_size, headers=headers, checksum=checksum

2073 )

2074

2075 upload._retry_strategy = _api_core_retry_to_resumable_media_retry(

2076 retry, num_retries

2077 )

2078

2079 upload.initiate(

2080 transport,

2081 stream,

2082 object_metadata,

2083 content_type,

2084 total_bytes=size,

2085 stream_final=False,

2086 timeout=timeout,

2087 )

2088

2089 return upload, transport

2090

2091 def _do_resumable_upload(

2092 self,

2093 client,

2094 stream,

2095 content_type,

2096 size,

2097 num_retries,

2098 predefined_acl,

2099 if_generation_match,

2100 if_generation_not_match,

2101 if_metageneration_match,

2102 if_metageneration_not_match,

2103 timeout=_DEFAULT_TIMEOUT,

2104 checksum=None,

2105 retry=None,

2106 ):

2107 """Perform a resumable upload.

2108

2109 Assumes ``chunk_size`` is not :data:`None` on the current blob.

2110 The default value of ``chunk_size`` is 100 MB.

2111

2112 The content type of the upload will be determined in order

2113 of precedence:

2114

2115 - The value passed in to this method (if not :data:`None`)

2116 - The value stored on the current blob

2117 - The default value ('application/octet-stream')

2118

2119 :type client: :class:`~google.cloud.storage.client.Client`

2120 :param client:

2121 (Optional) The client to use. If not passed, falls back to the

2122 ``client`` stored on the blob's bucket.

2123

2124 :type stream: IO[bytes]

2125 :param stream: A bytes IO object open for reading.

2126

2127 :type content_type: str

2128 :param content_type: Type of content being uploaded (or :data:`None`).

2129

2130 :type size: int

2131 :param size:

2132 The number of bytes to be uploaded (which will be read from

2133 ``stream``). If not provided, the upload will be concluded once

2134 ``stream`` is exhausted (or :data:`None`).

2135

2136 :type num_retries: int

2137 :param num_retries:

2138 Number of upload retries. By default, only uploads with

2139 if_generation_match set will be retried, as uploads without the

2140 argument are not guaranteed to be idempotent. Setting num_retries

2141 will override this default behavior and guarantee retries even when

2142 if_generation_match is not set. (Deprecated: This argument

2143 will be removed in a future release.)

2144

2145 :type predefined_acl: str

2146 :param predefined_acl: (Optional) Predefined access control list

2147

2148 :type if_generation_match: long

2149 :param if_generation_match:

2150 (Optional) See :ref:`using-if-generation-match`

2151

2152 :type if_generation_not_match: long

2153 :param if_generation_not_match:

2154 (Optional) See :ref:`using-if-generation-not-match`

2155

2156 :type if_metageneration_match: long

2157 :param if_metageneration_match:

2158 (Optional) See :ref:`using-if-metageneration-match`

2159

2160 :type if_metageneration_not_match: long

2161 :param if_metageneration_not_match:

2162 (Optional) See :ref:`using-if-metageneration-not-match`

2163

2164 :type timeout: float or tuple

2165 :param timeout:

2166 (Optional) The amount of time, in seconds, to wait

2167 for the server response. See: :ref:`configuring_timeouts`

2168

2169 :type checksum: str

2170 :param checksum:

2171 (Optional) The type of checksum to compute to verify

2172 the integrity of the object. After the upload is complete, the

2173 server-computed checksum of the resulting object will be checked

2174 and google.resumable_media.common.DataCorruption will be raised on

2175 a mismatch. On a validation failure, the client will attempt to

2176 delete the uploaded object automatically. Supported values

2177 are "md5", "crc32c" and None. The default is None.

2178

2179 :type retry: google.api_core.retry.Retry

2180 :param retry: (Optional) How to retry the RPC. A None value will disable

2181 retries. A google.api_core.retry.Retry value will enable retries,

2182 and the object will configure backoff and timeout options. Custom

2183 predicates (customizable error codes) are not supported for media

2184 operations such as this one.

2185

2186 This private method does not accept ConditionalRetryPolicy values

2187 because the information necessary to evaluate the policy is instead

2188 evaluated in blob._do_upload().

2189

2190 See the retry.py source code and docstrings in this package

2191 (google.cloud.storage.retry) for information on retry types and how

2192 to configure them.

2193

2194 :rtype: :class:`~requests.Response`

2195 :returns: The "200 OK" response object returned after the final chunk

2196 is uploaded.

2197 """

2198 upload, transport = self._initiate_resumable_upload(

2199 client,

2200 stream,

2201 content_type,

2202 size,

2203 num_retries,

2204 predefined_acl=predefined_acl,

2205 if_generation_match=if_generation_match,

2206 if_generation_not_match=if_generation_not_match,

2207 if_metageneration_match=if_metageneration_match,

2208 if_metageneration_not_match=if_metageneration_not_match,

2209 timeout=timeout,

2210 checksum=checksum,

2211 retry=retry,

2212 )

2213 while not upload.finished:

2214 try:

2215 response = upload.transmit_next_chunk(transport, timeout=timeout)

2216 except resumable_media.DataCorruption:

2217 # Attempt to delete the corrupted object.

2218 self.delete()

2219 raise

2220 return response

2221

2222 def _do_upload(

2223 self,

2224 client,

2225 stream,

2226 content_type,

2227 size,

2228 num_retries,

2229 predefined_acl,

2230 if_generation_match,

2231 if_generation_not_match,

2232 if_metageneration_match,

2233 if_metageneration_not_match,

2234 timeout=_DEFAULT_TIMEOUT,

2235 checksum=None,

2236 retry=None,

2237 ):

2238 """Determine an upload strategy and then perform the upload.

2239

2240 If the size of the data to be uploaded exceeds 8 MB a resumable media

2241 request will be used, otherwise the content and the metadata will be

2242 uploaded in a single multipart upload request.

2243

2244 The content type of the upload will be determined in order

2245 of precedence:

2246

2247 - The value passed in to this method (if not :data:`None`)

2248 - The value stored on the current blob

2249 - The default value ('application/octet-stream')

2250

2251 :type client: :class:`~google.cloud.storage.client.Client`

2252 :param client:

2253 (Optional) The client to use. If not passed, falls back to the

2254 ``client`` stored on the blob's bucket.

2255

2256 :type stream: IO[bytes]

2257 :param stream: A bytes IO object open for reading.

2258

2259 :type content_type: str

2260 :param content_type: Type of content being uploaded (or :data:`None`).

2261

2262 :type size: int

2263 :param size:

2264 The number of bytes to be uploaded (which will be read from

2265 ``stream``). If not provided, the upload will be concluded once

2266 ``stream`` is exhausted (or :data:`None`).

2267

2268 :type num_retries: int

2269 :param num_retries:

2270 Number of upload retries. By default, only uploads with

2271 if_generation_match set will be retried, as uploads without the

2272 argument are not guaranteed to be idempotent. Setting num_retries

2273 will override this default behavior and guarantee retries even when

2274 if_generation_match is not set. (Deprecated: This argument

2275 will be removed in a future release.)

2276

2277 :type predefined_acl: str

2278 :param predefined_acl: (Optional) Predefined access control list

2279

2280 :type if_generation_match: long

2281 :param if_generation_match:

2282 (Optional) See :ref:`using-if-generation-match`

2283

2284 :type if_generation_not_match: long

2285 :param if_generation_not_match:

2286 (Optional) See :ref:`using-if-generation-not-match`

2287

2288 :type if_metageneration_match: long

2289 :param if_metageneration_match:

2290 (Optional) See :ref:`using-if-metageneration-match`

2291

2292 :type if_metageneration_not_match: long

2293 :param if_metageneration_not_match:

2294 (Optional) See :ref:`using-if-metageneration-not-match`

2295

2296 :type timeout: float or tuple

2297 :param timeout:

2298 (Optional) The amount of time, in seconds, to wait

2299 for the server response. See: :ref:`configuring_timeouts`

2300

2301 :type checksum: str

2302 :param checksum:

2303 (Optional) The type of checksum to compute to verify

2304 the integrity of the object. If the upload is completed in a single

2305 request, the checksum will be entirely precomputed and the remote

2306 server will handle verification and error handling. If the upload

2307 is too large and must be transmitted in multiple requests, the

2308 checksum will be incrementally computed and the client will handle

2309 verification and error handling, raising

2310 google.resumable_media.common.DataCorruption on a mismatch and

2311 attempting to delete the corrupted file. Supported values are

2312 "md5", "crc32c" and None. The default is None.

2313

2314 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

2315 :param retry: (Optional) How to retry the RPC. A None value will disable

2316 retries. A google.api_core.retry.Retry value will enable retries,

2317 and the object will define retriable response codes and errors and

2318 configure backoff and timeout options.

2319

2320 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

2321 Retry object and activates it only if certain conditions are met.

2322 This class exists to provide safe defaults for RPC calls that are

2323 not technically safe to retry normally (due to potential data

2324 duplication or other side-effects) but become safe to retry if a

2325 condition such as if_generation_match is set.

2326

2327 See the retry.py source code and docstrings in this package

2328 (google.cloud.storage.retry) for information on retry types and how

2329 to configure them.

2330

2331 Media operations (downloads and uploads) do not support non-default

2332 predicates in a Retry object. The default will always be used. Other

2333 configuration changes for Retry objects such as delays and deadlines

2334 are respected.

2335

2336 :rtype: dict

2337 :returns: The parsed JSON from the "200 OK" response. This will be the

2338 **only** response in the multipart case and it will be the

2339 **final** response in the resumable case.

2340 """

2341

2342 # Handle ConditionalRetryPolicy.

2343 if isinstance(retry, ConditionalRetryPolicy):

2344 # Conditional retries are designed for non-media calls, which change

2345 # arguments into query_params dictionaries. Media operations work

2346 # differently, so here we make a "fake" query_params to feed to the

2347 # ConditionalRetryPolicy.

2348 query_params = {

2349 "ifGenerationMatch": if_generation_match,

2350 "ifMetagenerationMatch": if_metageneration_match,

2351 }

2352 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)

2353

2354 if size is not None and size <= _MAX_MULTIPART_SIZE:

2355 response = self._do_multipart_upload(

2356 client,

2357 stream,

2358 content_type,

2359 size,

2360 num_retries,

2361 predefined_acl,

2362 if_generation_match,

2363 if_generation_not_match,

2364 if_metageneration_match,

2365 if_metageneration_not_match,

2366 timeout=timeout,

2367 checksum=checksum,

2368 retry=retry,

2369 )

2370 else:

2371 response = self._do_resumable_upload(

2372 client,

2373 stream,

2374 content_type,

2375 size,

2376 num_retries,

2377 predefined_acl,

2378 if_generation_match,

2379 if_generation_not_match,

2380 if_metageneration_match,

2381 if_metageneration_not_match,

2382 timeout=timeout,

2383 checksum=checksum,

2384 retry=retry,

2385 )

2386

2387 return response.json()

2388

2389 def upload_from_file(

2390 self,

2391 file_obj,

2392 rewind=False,

2393 size=None,

2394 content_type=None,

2395 num_retries=None,

2396 client=None,

2397 predefined_acl=None,

2398 if_generation_match=None,

2399 if_generation_not_match=None,

2400 if_metageneration_match=None,

2401 if_metageneration_not_match=None,

2402 timeout=_DEFAULT_TIMEOUT,

2403 checksum=None,

2404 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

2405 ):

2406 """Upload the contents of this blob from a file-like object.

2407

2408 The content type of the upload will be determined in order

2409 of precedence:

2410

2411 - The value passed in to this method (if not :data:`None`)

2412 - The value stored on the current blob

2413 - The default value ('application/octet-stream')

2414

2415 .. note::

2416 The effect of uploading to an existing blob depends on the

2417 "versioning" and "lifecycle" policies defined on the blob's

2418 bucket. In the absence of those policies, upload will

2419 overwrite any existing contents.

2420

2421 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)

2422 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)

2423 API documents for details.

2424

2425 If the size of the data to be uploaded exceeds 8 MB a resumable media

2426 request will be used, otherwise the content and the metadata will be

2427 uploaded in a single multipart upload request.

2428

2429 For more fine-grained over the upload process, check out

2430 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html).

2431

2432 If :attr:`user_project` is set on the bucket, bills the API request

2433 to that project.

2434

2435 :type file_obj: file

2436 :param file_obj: A file handle opened in binary mode for reading.

2437

2438 :type rewind: bool

2439 :param rewind:

2440 If True, seek to the beginning of the file handle before writing

2441 the file to Cloud Storage.

2442

2443 :type size: int

2444 :param size:

2445 The number of bytes to be uploaded (which will be read from

2446 ``file_obj``). If not provided, the upload will be concluded once

2447 ``file_obj`` is exhausted.

2448

2449 :type content_type: str

2450 :param content_type: (Optional) Type of content being uploaded.

2451

2452 :type num_retries: int

2453 :param num_retries:

2454 Number of upload retries. By default, only uploads with

2455 if_generation_match set will be retried, as uploads without the

2456 argument are not guaranteed to be idempotent. Setting num_retries

2457 will override this default behavior and guarantee retries even when

2458 if_generation_match is not set. (Deprecated: This argument

2459 will be removed in a future release.)

2460

2461 :type client: :class:`~google.cloud.storage.client.Client`

2462 :param client:

2463 (Optional) The client to use. If not passed, falls back to the

2464 ``client`` stored on the blob's bucket.

2465

2466 :type predefined_acl: str

2467 :param predefined_acl: (Optional) Predefined access control list

2468

2469 :type if_generation_match: long

2470 :param if_generation_match:

2471 (Optional) See :ref:`using-if-generation-match`

2472

2473 :type if_generation_not_match: long

2474 :param if_generation_not_match:

2475 (Optional) See :ref:`using-if-generation-not-match`

2476

2477 :type if_metageneration_match: long

2478 :param if_metageneration_match:

2479 (Optional) See :ref:`using-if-metageneration-match`

2480

2481 :type if_metageneration_not_match: long

2482 :param if_metageneration_not_match:

2483 (Optional) See :ref:`using-if-metageneration-not-match`

2484

2485 :type timeout: float or tuple

2486 :param timeout:

2487 (Optional) The amount of time, in seconds, to wait

2488 for the server response. See: :ref:`configuring_timeouts`

2489

2490 :type checksum: str

2491 :param checksum:

2492 (Optional) The type of checksum to compute to verify

2493 the integrity of the object. If the upload is completed in a single

2494 request, the checksum will be entirely precomputed and the remote

2495 server will handle verification and error handling. If the upload

2496 is too large and must be transmitted in multiple requests, the

2497 checksum will be incrementally computed and the client will handle

2498 verification and error handling, raising

2499 google.resumable_media.common.DataCorruption on a mismatch and

2500 attempting to delete the corrupted file. Supported values are

2501 "md5", "crc32c" and None. The default is None.

2502

2503 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

2504 :param retry: (Optional) How to retry the RPC. A None value will disable

2505 retries. A google.api_core.retry.Retry value will enable retries,

2506 and the object will define retriable response codes and errors and

2507 configure backoff and timeout options.

2508

2509 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

2510 Retry object and activates it only if certain conditions are met.

2511 This class exists to provide safe defaults for RPC calls that are

2512 not technically safe to retry normally (due to potential data

2513 duplication or other side-effects) but become safe to retry if a

2514 condition such as if_generation_match is set.

2515

2516 See the retry.py source code and docstrings in this package

2517 (google.cloud.storage.retry) for information on retry types and how

2518 to configure them.

2519

2520 Media operations (downloads and uploads) do not support non-default

2521 predicates in a Retry object. The default will always be used. Other

2522 configuration changes for Retry objects such as delays and deadlines

2523 are respected.

2524

2525 :raises: :class:`~google.cloud.exceptions.GoogleCloudError`

2526 if the upload response returns an error status.

2527 """

2528 if num_retries is not None:

2529 warnings.warn(_NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2)

2530 # num_retries and retry are mutually exclusive. If num_retries is

2531 # set and retry is exactly the default, then nullify retry for

2532 # backwards compatibility.

2533 if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED:

2534 retry = None

2535

2536 _maybe_rewind(file_obj, rewind=rewind)

2537 predefined_acl = ACL.validate_predefined(predefined_acl)

2538

2539 try:

2540 created_json = self._do_upload(

2541 client,

2542 file_obj,

2543 content_type,

2544 size,

2545 num_retries,

2546 predefined_acl,

2547 if_generation_match,

2548 if_generation_not_match,

2549 if_metageneration_match,

2550 if_metageneration_not_match,

2551 timeout=timeout,

2552 checksum=checksum,

2553 retry=retry,

2554 )

2555 self._set_properties(created_json)

2556 except resumable_media.InvalidResponse as exc:

2557 _raise_from_invalid_response(exc)

2558

2559 def upload_from_filename(

2560 self,

2561 filename,

2562 content_type=None,

2563 num_retries=None,

2564 client=None,

2565 predefined_acl=None,

2566 if_generation_match=None,

2567 if_generation_not_match=None,

2568 if_metageneration_match=None,

2569 if_metageneration_not_match=None,

2570 timeout=_DEFAULT_TIMEOUT,

2571 checksum=None,

2572 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

2573 ):

2574 """Upload this blob's contents from the content of a named file.

2575

2576 The content type of the upload will be determined in order

2577 of precedence:

2578

2579 - The value passed in to this method (if not :data:`None`)

2580 - The value stored on the current blob

2581 - The value given by ``mimetypes.guess_type``

2582 - The default value ('application/octet-stream')

2583

2584 .. note::

2585 The effect of uploading to an existing blob depends on the

2586 "versioning" and "lifecycle" policies defined on the blob's

2587 bucket. In the absence of those policies, upload will

2588 overwrite any existing contents.

2589

2590 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)

2591 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)

2592 API documents for details.

2593

2594 If :attr:`user_project` is set on the bucket, bills the API request

2595 to that project.

2596

2597 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-upload-encrypted-file#storage_upload_encrypted_file-python)

2598 to upload a file with a

2599 [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied).

2600

2601 :type filename: str

2602 :param filename: The path to the file.

2603

2604 :type content_type: str

2605 :param content_type: (Optional) Type of content being uploaded.

2606

2607 :type client: :class:`~google.cloud.storage.client.Client`

2608 :param client:

2609 (Optional) The client to use. If not passed, falls back to the

2610 ``client`` stored on the blob's bucket.

2611

2612 :type num_retries: int

2613 :param num_retries:

2614 Number of upload retries. By default, only uploads with

2615 if_generation_match set will be retried, as uploads without the

2616 argument are not guaranteed to be idempotent. Setting num_retries

2617 will override this default behavior and guarantee retries even when

2618 if_generation_match is not set. (Deprecated: This argument

2619 will be removed in a future release.)

2620

2621 :type predefined_acl: str

2622 :param predefined_acl: (Optional) Predefined access control list

2623

2624 :type if_generation_match: long

2625 :param if_generation_match:

2626 (Optional) See :ref:`using-if-generation-match`

2627

2628 :type if_generation_not_match: long

2629 :param if_generation_not_match:

2630 (Optional) See :ref:`using-if-generation-not-match`

2631

2632 :type if_metageneration_match: long

2633 :param if_metageneration_match:

2634 (Optional) See :ref:`using-if-metageneration-match`

2635

2636 :type if_metageneration_not_match: long

2637 :param if_metageneration_not_match:

2638 (Optional) See :ref:`using-if-metageneration-not-match`

2639

2640 :type timeout: float or tuple

2641 :param timeout:

2642 (Optional) The amount of time, in seconds, to wait

2643 for the server response. See: :ref:`configuring_timeouts`

2644

2645 :type checksum: str

2646 :param checksum:

2647 (Optional) The type of checksum to compute to verify

2648 the integrity of the object. If the upload is completed in a single

2649 request, the checksum will be entirely precomputed and the remote

2650 server will handle verification and error handling. If the upload

2651 is too large and must be transmitted in multiple requests, the

2652 checksum will be incrementally computed and the client will handle

2653 verification and error handling, raising

2654 google.resumable_media.common.DataCorruption on a mismatch and

2655 attempting to delete the corrupted file. Supported values are

2656 "md5", "crc32c" and None. The default is None.

2657

2658 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

2659 :param retry: (Optional) How to retry the RPC. A None value will disable

2660 retries. A google.api_core.retry.Retry value will enable retries,

2661 and the object will define retriable response codes and errors and

2662 configure backoff and timeout options.

2663

2664 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

2665 Retry object and activates it only if certain conditions are met.

2666 This class exists to provide safe defaults for RPC calls that are

2667 not technically safe to retry normally (due to potential data

2668 duplication or other side-effects) but become safe to retry if a

2669 condition such as if_generation_match is set.

2670

2671 See the retry.py source code and docstrings in this package

2672 (google.cloud.storage.retry) for information on retry types and how

2673 to configure them.

2674

2675 Media operations (downloads and uploads) do not support non-default

2676 predicates in a Retry object. The default will always be used. Other

2677 configuration changes for Retry objects such as delays and deadlines

2678 are respected.

2679 """

2680 content_type = self._get_content_type(content_type, filename=filename)

2681

2682 with open(filename, "rb") as file_obj:

2683 total_bytes = os.fstat(file_obj.fileno()).st_size

2684 self.upload_from_file(

2685 file_obj,

2686 content_type=content_type,

2687 num_retries=num_retries,

2688 client=client,

2689 size=total_bytes,

2690 predefined_acl=predefined_acl,

2691 if_generation_match=if_generation_match,

2692 if_generation_not_match=if_generation_not_match,

2693 if_metageneration_match=if_metageneration_match,

2694 if_metageneration_not_match=if_metageneration_not_match,

2695 timeout=timeout,

2696 checksum=checksum,

2697 retry=retry,

2698 )

2699

2700 def upload_from_string(

2701 self,

2702 data,

2703 content_type="text/plain",

2704 num_retries=None,

2705 client=None,

2706 predefined_acl=None,

2707 if_generation_match=None,

2708 if_generation_not_match=None,

2709 if_metageneration_match=None,

2710 if_metageneration_not_match=None,

2711 timeout=_DEFAULT_TIMEOUT,

2712 checksum=None,

2713 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

2714 ):

2715 """Upload contents of this blob from the provided string.

2716

2717 .. note::

2718 The effect of uploading to an existing blob depends on the

2719 "versioning" and "lifecycle" policies defined on the blob's

2720 bucket. In the absence of those policies, upload will

2721 overwrite any existing contents.

2722

2723 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)

2724 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)

2725 API documents for details.

2726

2727 If :attr:`user_project` is set on the bucket, bills the API request

2728 to that project.

2729

2730 :type data: bytes or str

2731 :param data:

2732 The data to store in this blob. If the value is text, it will be

2733 encoded as UTF-8.

2734

2735 :type content_type: str

2736 :param content_type:

2737 (Optional) Type of content being uploaded. Defaults to

2738 ``'text/plain'``.

2739

2740 :type num_retries: int

2741 :param num_retries:

2742 Number of upload retries. By default, only uploads with

2743 if_generation_match set will be retried, as uploads without the

2744 argument are not guaranteed to be idempotent. Setting num_retries

2745 will override this default behavior and guarantee retries even when

2746 if_generation_match is not set. (Deprecated: This argument

2747 will be removed in a future release.)

2748

2749 :type client: :class:`~google.cloud.storage.client.Client`

2750 :param client:

2751 (Optional) The client to use. If not passed, falls back to the

2752 ``client`` stored on the blob's bucket.

2753

2754 :type predefined_acl: str

2755 :param predefined_acl: (Optional) Predefined access control list

2756

2757 :type if_generation_match: long

2758 :param if_generation_match:

2759 (Optional) See :ref:`using-if-generation-match`

2760

2761 :type if_generation_not_match: long

2762 :param if_generation_not_match:

2763 (Optional) See :ref:`using-if-generation-not-match`

2764

2765 :type if_metageneration_match: long

2766 :param if_metageneration_match:

2767 (Optional) See :ref:`using-if-metageneration-match`

2768

2769 :type if_metageneration_not_match: long

2770 :param if_metageneration_not_match:

2771 (Optional) See :ref:`using-if-metageneration-not-match`

2772

2773 :type timeout: float or tuple

2774 :param timeout:

2775 (Optional) The amount of time, in seconds, to wait

2776 for the server response. See: :ref:`configuring_timeouts`

2777

2778 :type checksum: str

2779 :param checksum:

2780 (Optional) The type of checksum to compute to verify

2781 the integrity of the object. If the upload is completed in a single

2782 request, the checksum will be entirely precomputed and the remote

2783 server will handle verification and error handling. If the upload

2784 is too large and must be transmitted in multiple requests, the

2785 checksum will be incrementally computed and the client will handle

2786 verification and error handling, raising

2787 google.resumable_media.common.DataCorruption on a mismatch and

2788 attempting to delete the corrupted file. Supported values are

2789 "md5", "crc32c" and None. The default is None.

2790

2791 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

2792 :param retry: (Optional) How to retry the RPC. A None value will disable

2793 retries. A google.api_core.retry.Retry value will enable retries,

2794 and the object will define retriable response codes and errors and

2795 configure backoff and timeout options.

2796

2797 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

2798 Retry object and activates it only if certain conditions are met.

2799 This class exists to provide safe defaults for RPC calls that are

2800 not technically safe to retry normally (due to potential data

2801 duplication or other side-effects) but become safe to retry if a

2802 condition such as if_generation_match is set.

2803

2804 See the retry.py source code and docstrings in this package

2805 (google.cloud.storage.retry) for information on retry types and how

2806 to configure them.

2807

2808 Media operations (downloads and uploads) do not support non-default

2809 predicates in a Retry object. The default will always be used. Other

2810 configuration changes for Retry objects such as delays and deadlines

2811 are respected.

2812 """

2813 data = _to_bytes(data, encoding="utf-8")

2814 string_buffer = BytesIO(data)

2815 self.upload_from_file(

2816 file_obj=string_buffer,

2817 size=len(data),

2818 content_type=content_type,

2819 num_retries=num_retries,

2820 client=client,

2821 predefined_acl=predefined_acl,

2822 if_generation_match=if_generation_match,

2823 if_generation_not_match=if_generation_not_match,

2824 if_metageneration_match=if_metageneration_match,

2825 if_metageneration_not_match=if_metageneration_not_match,

2826 timeout=timeout,

2827 checksum=checksum,

2828 retry=retry,

2829 )

2830

2831 def create_resumable_upload_session(

2832 self,

2833 content_type=None,

2834 size=None,

2835 origin=None,

2836 client=None,

2837 timeout=_DEFAULT_TIMEOUT,

2838 checksum=None,

2839 predefined_acl=None,

2840 if_generation_match=None,

2841 if_generation_not_match=None,

2842 if_metageneration_match=None,

2843 if_metageneration_not_match=None,

2844 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

2845 ):

2846 """Create a resumable upload session.

2847

2848 Resumable upload sessions allow you to start an upload session from

2849 one client and complete the session in another. This method is called

2850 by the initiator to set the metadata and limits. The initiator then

2851 passes the session URL to the client that will upload the binary data.

2852 The client performs a PUT request on the session URL to complete the

2853 upload. This process allows untrusted clients to upload to an

2854 access-controlled bucket.

2855

2856 For more details, see the

2857 documentation on [`signed URLs`](https://cloud.google.com/storage/docs/access-control/signed-urls#signing-resumable).

2858

2859 The content type of the upload will be determined in order

2860 of precedence:

2861

2862 - The value passed in to this method (if not :data:`None`)

2863 - The value stored on the current blob

2864 - The default value ('application/octet-stream')

2865

2866 .. note::

2867 The effect of uploading to an existing blob depends on the

2868 "versioning" and "lifecycle" policies defined on the blob's

2869 bucket. In the absence of those policies, upload will

2870 overwrite any existing contents.

2871

2872 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)

2873 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)

2874 API documents for details.

2875

2876 If :attr:`encryption_key` is set, the blob will be encrypted with

2877 a [`customer-supplied`](https://cloud.google.com/storage/docs/encryption#customer-supplied)

2878 encryption key.

2879

2880 If :attr:`user_project` is set on the bucket, bills the API request

2881 to that project.

2882

2883 :type size: int

2884 :param size:

2885 (Optional) The maximum number of bytes that can be uploaded using

2886 this session. If the size is not known when creating the session,

2887 this should be left blank.

2888

2889 :type content_type: str

2890 :param content_type: (Optional) Type of content being uploaded.

2891

2892 :type origin: str

2893 :param origin:

2894 (Optional) If set, the upload can only be completed by a user-agent

2895 that uploads from the given origin. This can be useful when passing

2896 the session to a web client.

2897

2898 :type client: :class:`~google.cloud.storage.client.Client`

2899 :param client:

2900 (Optional) The client to use. If not passed, falls back to the

2901 ``client`` stored on the blob's bucket.

2902

2903 :type timeout: float or tuple

2904 :param timeout:

2905 (Optional) The amount of time, in seconds, to wait

2906 for the server response. See: :ref:`configuring_timeouts`

2907

2908 :type checksum: str

2909 :param checksum:

2910 (Optional) The type of checksum to compute to verify

2911 the integrity of the object. After the upload is complete, the

2912 server-computed checksum of the resulting object will be checked

2913 and google.resumable_media.common.DataCorruption will be raised on

2914 a mismatch. On a validation failure, the client will attempt to

2915 delete the uploaded object automatically. Supported values

2916 are "md5", "crc32c" and None. The default is None.

2917

2918 :type predefined_acl: str

2919 :param predefined_acl: (Optional) Predefined access control list

2920

2921 :type if_generation_match: long

2922 :param if_generation_match:

2923 (Optional) See :ref:`using-if-generation-match`

2924

2925 :type if_generation_not_match: long

2926 :param if_generation_not_match:

2927 (Optional) See :ref:`using-if-generation-not-match`

2928

2929 :type if_metageneration_match: long

2930 :param if_metageneration_match:

2931 (Optional) See :ref:`using-if-metageneration-match`

2932

2933 :type if_metageneration_not_match: long

2934 :param if_metageneration_not_match:

2935 (Optional) See :ref:`using-if-metageneration-not-match`

2936

2937 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

2938 :param retry: (Optional) How to retry the RPC. A None value will disable

2939 retries. A google.api_core.retry.Retry value will enable retries,

2940 and the object will define retriable response codes and errors and

2941 configure backoff and timeout options.

2942 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a

2943 Retry object and activates it only if certain conditions are met.

2944 This class exists to provide safe defaults for RPC calls that are

2945 not technically safe to retry normally (due to potential data

2946 duplication or other side-effects) but become safe to retry if a

2947 condition such as if_generation_match is set.

2948 See the retry.py source code and docstrings in this package

2949 (google.cloud.storage.retry) for information on retry types and how

2950 to configure them.

2951 Media operations (downloads and uploads) do not support non-default

2952 predicates in a Retry object. The default will always be used. Other

2953 configuration changes for Retry objects such as delays and deadlines

2954 are respected.

2955

2956 :rtype: str

2957 :returns: The resumable upload session URL. The upload can be

2958 completed by making an HTTP PUT request with the

2959 file's contents.

2960

2961 :raises: :class:`google.cloud.exceptions.GoogleCloudError`

2962 if the session creation response returns an error status.

2963 """

2964

2965 # Handle ConditionalRetryPolicy.

2966 if isinstance(retry, ConditionalRetryPolicy):

2967 # Conditional retries are designed for non-media calls, which change

2968 # arguments into query_params dictionaries. Media operations work

2969 # differently, so here we make a "fake" query_params to feed to the

2970 # ConditionalRetryPolicy.

2971 query_params = {

2972 "ifGenerationMatch": if_generation_match,

2973 "ifMetagenerationMatch": if_metageneration_match,

2974 }

2975 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)

2976

2977 extra_headers = {}

2978 if origin is not None:

2979 # This header is specifically for client-side uploads, it

2980 # determines the origins allowed for CORS.

2981 extra_headers["Origin"] = origin

2982

2983 try:

2984 fake_stream = BytesIO(b"")

2985 # Send a fake the chunk size which we **know** will be acceptable

2986 # to the `ResumableUpload` constructor. The chunk size only

2987 # matters when **sending** bytes to an upload.

2988 upload, _ = self._initiate_resumable_upload(

2989 client,

2990 fake_stream,

2991 content_type,

2992 size,

2993 None,

2994 predefined_acl=predefined_acl,

2995 if_generation_match=if_generation_match,

2996 if_generation_not_match=if_generation_not_match,

2997 if_metageneration_match=if_metageneration_match,

2998 if_metageneration_not_match=if_metageneration_not_match,

2999 extra_headers=extra_headers,

3000 chunk_size=self._CHUNK_SIZE_MULTIPLE,

3001 timeout=timeout,

3002 checksum=checksum,

3003 retry=retry,

3004 )

3005

3006 return upload.resumable_url

3007 except resumable_media.InvalidResponse as exc:

3008 _raise_from_invalid_response(exc)

3009

3010 def get_iam_policy(

3011 self,

3012 client=None,

3013 requested_policy_version=None,

3014 timeout=_DEFAULT_TIMEOUT,

3015 retry=DEFAULT_RETRY,

3016 ):

3017 """Retrieve the IAM policy for the object.

3018

3019 .. note::

3020

3021 Blob- / object-level IAM support does not yet exist and methods

3022 currently call an internal ACL backend not providing any utility

3023 beyond the blob's :attr:`acl` at this time. The API may be enhanced

3024 in the future and is currently undocumented. Use :attr:`acl` for

3025 managing object access control.

3026

3027 If :attr:`user_project` is set on the bucket, bills the API request

3028 to that project.

3029

3030 :type client: :class:`~google.cloud.storage.client.Client`

3031 :param client:

3032 (Optional) The client to use. If not passed, falls back to the

3033 ``client`` stored on the current object's bucket.

3034

3035 :type requested_policy_version: int or ``NoneType``

3036 :param requested_policy_version:

3037 (Optional) The version of IAM policies to request. If a policy

3038 with a condition is requested without setting this, the server will

3039 return an error. This must be set to a value of 3 to retrieve IAM

3040 policies containing conditions. This is to prevent client code that

3041 isn't aware of IAM conditions from interpreting and modifying

3042 policies incorrectly. The service might return a policy with

3043 version lower than the one that was requested, based on the feature

3044 syntax in the policy fetched.

3045

3046 :type timeout: float or tuple

3047 :param timeout:

3048 (Optional) The amount of time, in seconds, to wait

3049 for the server response. See: :ref:`configuring_timeouts`

3050

3051 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3052 :param retry:

3053 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3054

3055 :rtype: :class:`google.api_core.iam.Policy`

3056 :returns: the policy instance, based on the resource returned from

3057 the ``getIamPolicy`` API request.

3058 """

3059 client = self._require_client(client)

3060

3061 query_params = {}

3062

3063 if self.user_project is not None:

3064 query_params["userProject"] = self.user_project

3065

3066 if requested_policy_version is not None:

3067 query_params["optionsRequestedPolicyVersion"] = requested_policy_version

3068

3069 info = client._get_resource(

3070 f"{self.path}/iam",

3071 query_params=query_params,

3072 timeout=timeout,

3073 retry=retry,

3074 _target_object=None,

3075 )

3076 return Policy.from_api_repr(info)

3077

3078 def set_iam_policy(

3079 self,

3080 policy,

3081 client=None,

3082 timeout=_DEFAULT_TIMEOUT,

3083 retry=DEFAULT_RETRY_IF_ETAG_IN_JSON,

3084 ):

3085 """Update the IAM policy for the bucket.

3086

3087 .. note::

3088

3089 Blob- / object-level IAM support does not yet exist and methods

3090 currently call an internal ACL backend not providing any utility

3091 beyond the blob's :attr:`acl` at this time. The API may be enhanced

3092 in the future and is currently undocumented. Use :attr:`acl` for

3093 managing object access control.

3094

3095 If :attr:`user_project` is set on the bucket, bills the API request

3096 to that project.

3097

3098 :type policy: :class:`google.api_core.iam.Policy`

3099 :param policy: policy instance used to update bucket's IAM policy.

3100

3101 :type client: :class:`~google.cloud.storage.client.Client`

3102 :param client:

3103 (Optional) The client to use. If not passed, falls back to the

3104 ``client`` stored on the current bucket.

3105

3106 :type timeout: float or tuple

3107 :param timeout:

3108 (Optional) The amount of time, in seconds, to wait

3109 for the server response. See: :ref:`configuring_timeouts`

3110

3111 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3112 :param retry:

3113 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3114

3115 :rtype: :class:`google.api_core.iam.Policy`

3116 :returns: the policy instance, based on the resource returned from

3117 the ``setIamPolicy`` API request.

3118 """

3119 client = self._require_client(client)

3120

3121 query_params = {}

3122

3123 if self.user_project is not None:

3124 query_params["userProject"] = self.user_project

3125

3126 path = f"{self.path}/iam"

3127 resource = policy.to_api_repr()

3128 resource["resourceId"] = self.path

3129 info = client._put_resource(

3130 path,

3131 resource,

3132 query_params=query_params,

3133 timeout=timeout,

3134 retry=retry,

3135 _target_object=None,

3136 )

3137 return Policy.from_api_repr(info)

3138

3139 def test_iam_permissions(

3140 self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY

3141 ):

3142 """API call: test permissions

3143

3144 .. note::

3145

3146 Blob- / object-level IAM support does not yet exist and methods

3147 currently call an internal ACL backend not providing any utility

3148 beyond the blob's :attr:`acl` at this time. The API may be enhanced

3149 in the future and is currently undocumented. Use :attr:`acl` for

3150 managing object access control.

3151

3152 If :attr:`user_project` is set on the bucket, bills the API request

3153 to that project.

3154

3155 :type permissions: list of string

3156 :param permissions: the permissions to check

3157

3158 :type client: :class:`~google.cloud.storage.client.Client`

3159 :param client:

3160 (Optional) The client to use. If not passed, falls back to the

3161 ``client`` stored on the current bucket.

3162

3163 :type timeout: float or tuple

3164 :param timeout:

3165 (Optional) The amount of time, in seconds, to wait

3166 for the server response. See: :ref:`configuring_timeouts`

3167

3168 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3169 :param retry:

3170 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3171

3172 :rtype: list of string

3173 :returns: the permissions returned by the ``testIamPermissions`` API

3174 request.

3175 """

3176 client = self._require_client(client)

3177 query_params = {"permissions": permissions}

3178

3179 if self.user_project is not None:

3180 query_params["userProject"] = self.user_project

3181

3182 path = f"{self.path}/iam/testPermissions"

3183 resp = client._get_resource(

3184 path,

3185 query_params=query_params,

3186 timeout=timeout,

3187 retry=retry,

3188 _target_object=None,

3189 )

3190

3191 return resp.get("permissions", [])

3192

3193 def make_public(

3194 self,

3195 client=None,

3196 timeout=_DEFAULT_TIMEOUT,

3197 if_generation_match=None,

3198 if_generation_not_match=None,

3199 if_metageneration_match=None,

3200 if_metageneration_not_match=None,

3201 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,

3202 ):

3203 """Update blob's ACL, granting read access to anonymous users.

3204

3205 :type client: :class:`~google.cloud.storage.client.Client` or

3206 ``NoneType``

3207 :param client: (Optional) The client to use. If not passed, falls back

3208 to the ``client`` stored on the blob's bucket.

3209

3210 :type timeout: float or tuple

3211 :param timeout:

3212 (Optional) The amount of time, in seconds, to wait

3213 for the server response. See: :ref:`configuring_timeouts`

3214

3215 :type if_generation_match: long

3216 :param if_generation_match:

3217 (Optional) See :ref:`using-if-generation-match`

3218

3219 :type if_generation_not_match: long

3220 :param if_generation_not_match:

3221 (Optional) See :ref:`using-if-generation-not-match`

3222

3223 :type if_metageneration_match: long

3224 :param if_metageneration_match:

3225 (Optional) See :ref:`using-if-metageneration-match`

3226

3227 :type if_metageneration_not_match: long

3228 :param if_metageneration_not_match:

3229 (Optional) See :ref:`using-if-metageneration-not-match`

3230

3231 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3232 :param retry:

3233 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3234 """

3235 self.acl.all().grant_read()

3236 self.acl.save(

3237 client=client,

3238 timeout=timeout,

3239 if_generation_match=if_generation_match,

3240 if_generation_not_match=if_generation_not_match,

3241 if_metageneration_match=if_metageneration_match,

3242 if_metageneration_not_match=if_metageneration_not_match,

3243 retry=retry,

3244 )

3245

3246 def make_private(

3247 self,

3248 client=None,

3249 timeout=_DEFAULT_TIMEOUT,

3250 if_generation_match=None,

3251 if_generation_not_match=None,

3252 if_metageneration_match=None,

3253 if_metageneration_not_match=None,

3254 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,

3255 ):

3256 """Update blob's ACL, revoking read access for anonymous users.

3257

3258 :type client: :class:`~google.cloud.storage.client.Client` or

3259 ``NoneType``

3260 :param client: (Optional) The client to use. If not passed, falls back

3261 to the ``client`` stored on the blob's bucket.

3262

3263 :type timeout: float or tuple

3264 :param timeout:

3265 (Optional) The amount of time, in seconds, to wait

3266 for the server response. See: :ref:`configuring_timeouts`

3267

3268 :type if_generation_match: long

3269 :param if_generation_match:

3270 (Optional) See :ref:`using-if-generation-match`

3271

3272 :type if_generation_not_match: long

3273 :param if_generation_not_match:

3274 (Optional) See :ref:`using-if-generation-not-match`

3275

3276 :type if_metageneration_match: long

3277 :param if_metageneration_match:

3278 (Optional) See :ref:`using-if-metageneration-match`

3279

3280 :type if_metageneration_not_match: long

3281 :param if_metageneration_not_match:

3282 (Optional) See :ref:`using-if-metageneration-not-match`

3283

3284 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3285 :param retry:

3286 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3287 """

3288 self.acl.all().revoke_read()

3289 self.acl.save(

3290 client=client,

3291 timeout=timeout,

3292 if_generation_match=if_generation_match,

3293 if_generation_not_match=if_generation_not_match,

3294 if_metageneration_match=if_metageneration_match,

3295 if_metageneration_not_match=if_metageneration_not_match,

3296 retry=retry,

3297 )

3298

3299 def compose(

3300 self,

3301 sources,

3302 client=None,

3303 timeout=_DEFAULT_TIMEOUT,

3304 if_generation_match=None,

3305 if_metageneration_match=None,

3306 if_source_generation_match=None,

3307 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

3308 ):

3309 """Concatenate source blobs into this one.

3310

3311 If :attr:`user_project` is set on the bucket, bills the API request

3312 to that project.

3313

3314 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/objects/compose)

3315 and a [code sample](https://cloud.google.com/storage/docs/samples/storage-compose-file#storage_compose_file-python).

3316

3317 :type sources: list of :class:`Blob`

3318 :param sources: Blobs whose contents will be composed into this blob.

3319

3320 :type client: :class:`~google.cloud.storage.client.Client`

3321 :param client:

3322 (Optional) The client to use. If not passed, falls back to the

3323 ``client`` stored on the blob's bucket.

3324

3325 :type timeout: float or tuple

3326 :param timeout:

3327 (Optional) The amount of time, in seconds, to wait

3328 for the server response. See: :ref:`configuring_timeouts`

3329

3330 :type if_generation_match: long

3331 :param if_generation_match:

3332 (Optional) Makes the operation conditional on whether the

3333 destination object's current generation matches the given value.

3334 Setting to 0 makes the operation succeed only if there are no live

3335 versions of the object.

3336 Note: In a previous version, this argument worked identically to the

3337 ``if_source_generation_match`` argument. For

3338 backwards-compatibility reasons, if a list is passed in,

3339 this argument will behave like ``if_source_generation_match``

3340 and also issue a DeprecationWarning.

3341

3342 :type if_metageneration_match: long

3343 :param if_metageneration_match:

3344 (Optional) Makes the operation conditional on whether the

3345 destination object's current metageneration matches the given

3346 value.

3347

3348 If a list of long is passed in, no match operation will be

3349 performed. (Deprecated: type(list of long) is supported for

3350 backwards-compatability reasons only.)

3351

3352 :type if_source_generation_match: list of long

3353 :param if_source_generation_match:

3354 (Optional) Makes the operation conditional on whether the current

3355 generation of each source blob matches the corresponding generation.

3356 The list must match ``sources`` item-to-item.

3357

3358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3359 :param retry:

3360 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3361 """

3362 sources_len = len(sources)

3363 client = self._require_client(client)

3364 query_params = {}

3365

3366 if isinstance(if_generation_match, list):

3367 warnings.warn(

3368 _COMPOSE_IF_GENERATION_LIST_DEPRECATED,

3369 DeprecationWarning,

3370 stacklevel=2,

3371 )

3372

3373 if if_source_generation_match is not None:

3374 raise ValueError(

3375 _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR

3376 )

3377

3378 if_source_generation_match = if_generation_match

3379 if_generation_match = None

3380

3381 if isinstance(if_metageneration_match, list):

3382 warnings.warn(

3383 _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED,

3384 DeprecationWarning,

3385 stacklevel=2,

3386 )

3387

3388 if_metageneration_match = None

3389

3390 if if_source_generation_match is None:

3391 if_source_generation_match = [None] * sources_len

3392 if len(if_source_generation_match) != sources_len:

3393 raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR)

3394

3395 source_objects = []

3396 for source, source_generation in zip(sources, if_source_generation_match):

3397 source_object = {"name": source.name, "generation": source.generation}

3398

3399 preconditions = {}

3400 if source_generation is not None:

3401 preconditions["ifGenerationMatch"] = source_generation

3402

3403 if preconditions:

3404 source_object["objectPreconditions"] = preconditions

3405

3406 source_objects.append(source_object)

3407

3408 request = {

3409 "sourceObjects": source_objects,

3410 "destination": self._properties.copy(),

3411 }

3412

3413 if self.user_project is not None:

3414 query_params["userProject"] = self.user_project

3415

3416 _add_generation_match_parameters(

3417 query_params,

3418 if_generation_match=if_generation_match,

3419 if_metageneration_match=if_metageneration_match,

3420 )

3421

3422 api_response = client._post_resource(

3423 f"{self.path}/compose",

3424 request,

3425 query_params=query_params,

3426 timeout=timeout,

3427 retry=retry,

3428 _target_object=self,

3429 )

3430 self._set_properties(api_response)

3431

3432 def rewrite(

3433 self,

3434 source,

3435 token=None,

3436 client=None,

3437 if_generation_match=None,

3438 if_generation_not_match=None,

3439 if_metageneration_match=None,

3440 if_metageneration_not_match=None,

3441 if_source_generation_match=None,

3442 if_source_generation_not_match=None,

3443 if_source_metageneration_match=None,

3444 if_source_metageneration_not_match=None,

3445 timeout=_DEFAULT_TIMEOUT,

3446 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

3447 ):

3448 """Rewrite source blob into this one.

3449

3450 If :attr:`user_project` is set on the bucket, bills the API request

3451 to that project.

3452

3453 .. note::

3454

3455 ``rewrite`` is not supported in a ``Batch`` context.

3456

3457 :type source: :class:`Blob`

3458 :param source: blob whose contents will be rewritten into this blob.

3459

3460 :type token: str

3461 :param token:

3462 (Optional) Token returned from an earlier, not-completed call to

3463 rewrite the same source blob. If passed, result will include

3464 updated status, total bytes written.

3465

3466 :type client: :class:`~google.cloud.storage.client.Client`

3467 :param client:

3468 (Optional) The client to use. If not passed, falls back to the

3469 ``client`` stored on the blob's bucket.

3470

3471 :type if_generation_match: long

3472 :param if_generation_match:

3473 (Optional) See :ref:`using-if-generation-match`

3474 Note that the generation to be matched is that of the

3475 ``destination`` blob.

3476

3477 :type if_generation_not_match: long

3478 :param if_generation_not_match:

3479 (Optional) See :ref:`using-if-generation-not-match`

3480 Note that the generation to be matched is that of the

3481 ``destination`` blob.

3482

3483 :type if_metageneration_match: long

3484 :param if_metageneration_match:

3485 (Optional) See :ref:`using-if-metageneration-match`

3486 Note that the metageneration to be matched is that of the

3487 ``destination`` blob.

3488

3489 :type if_metageneration_not_match: long

3490 :param if_metageneration_not_match:

3491 (Optional) See :ref:`using-if-metageneration-not-match`

3492 Note that the metageneration to be matched is that of the

3493 ``destination`` blob.

3494

3495 :type if_source_generation_match: long

3496 :param if_source_generation_match:

3497 (Optional) Makes the operation conditional on whether the source

3498 object's generation matches the given value.

3499

3500 :type if_source_generation_not_match: long

3501 :param if_source_generation_not_match:

3502 (Optional) Makes the operation conditional on whether the source

3503 object's generation does not match the given value.

3504

3505 :type if_source_metageneration_match: long

3506 :param if_source_metageneration_match:

3507 (Optional) Makes the operation conditional on whether the source

3508 object's current metageneration matches the given value.

3509

3510 :type if_source_metageneration_not_match: long

3511 :param if_source_metageneration_not_match:

3512 (Optional) Makes the operation conditional on whether the source

3513 object's current metageneration does not match the given value.

3514

3515 :type timeout: float or tuple

3516 :param timeout:

3517 (Optional) The amount of time, in seconds, to wait

3518 for the server response. See: :ref:`configuring_timeouts`

3519

3520 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3521 :param retry:

3522 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3523

3524 :rtype: tuple

3525 :returns: ``(token, bytes_rewritten, total_bytes)``, where ``token``

3526 is a rewrite token (``None`` if the rewrite is complete),

3527 ``bytes_rewritten`` is the number of bytes rewritten so far,

3528 and ``total_bytes`` is the total number of bytes to be

3529 rewritten.

3530 """

3531 client = self._require_client(client)

3532 headers = _get_encryption_headers(self._encryption_key)

3533 headers.update(_get_encryption_headers(source._encryption_key, source=True))

3534

3535 query_params = self._query_params

3536 if "generation" in query_params:

3537 del query_params["generation"]

3538

3539 if token:

3540 query_params["rewriteToken"] = token

3541

3542 if source.generation:

3543 query_params["sourceGeneration"] = source.generation

3544

3545 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object

3546 # at rest, object resource metadata will store the version of the Key Management

3547 # Service cryptographic material. If a Blob instance with KMS Key metadata set is

3548 # used to rewrite the object, then the existing kmsKeyName version

3549 # value can't be used in the rewrite request and the client instead ignores it.

3550 if (

3551 self.kms_key_name is not None

3552 and "cryptoKeyVersions" not in self.kms_key_name

3553 ):

3554 query_params["destinationKmsKeyName"] = self.kms_key_name

3555

3556 _add_generation_match_parameters(

3557 query_params,

3558 if_generation_match=if_generation_match,

3559 if_generation_not_match=if_generation_not_match,

3560 if_metageneration_match=if_metageneration_match,

3561 if_metageneration_not_match=if_metageneration_not_match,

3562 if_source_generation_match=if_source_generation_match,

3563 if_source_generation_not_match=if_source_generation_not_match,

3564 if_source_metageneration_match=if_source_metageneration_match,

3565 if_source_metageneration_not_match=if_source_metageneration_not_match,

3566 )

3567

3568 path = f"{source.path}/rewriteTo{self.path}"

3569 api_response = client._post_resource(

3570 path,

3571 self._properties,

3572 query_params=query_params,

3573 headers=headers,

3574 timeout=timeout,

3575 retry=retry,

3576 _target_object=self,

3577 )

3578 rewritten = int(api_response["totalBytesRewritten"])

3579 size = int(api_response["objectSize"])

3580

3581 # The resource key is set if and only if the API response is

3582 # completely done. Additionally, there is no rewrite token to return

3583 # in this case.

3584 if api_response["done"]:

3585 self._set_properties(api_response["resource"])

3586 return None, rewritten, size

3587

3588 return api_response["rewriteToken"], rewritten, size

3589

3590 def update_storage_class(

3591 self,

3592 new_class,

3593 client=None,

3594 if_generation_match=None,

3595 if_generation_not_match=None,

3596 if_metageneration_match=None,

3597 if_metageneration_not_match=None,

3598 if_source_generation_match=None,

3599 if_source_generation_not_match=None,

3600 if_source_metageneration_match=None,

3601 if_source_metageneration_not_match=None,

3602 timeout=_DEFAULT_TIMEOUT,

3603 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,

3604 ):

3605 """Update blob's storage class via a rewrite-in-place. This helper will

3606 wait for the rewrite to complete before returning, so it may take some

3607 time for large files.

3608

3609 See

3610 https://cloud.google.com/storage/docs/per-object-storage-class

3611

3612 If :attr:`user_project` is set on the bucket, bills the API request

3613 to that project.

3614

3615 :type new_class: str

3616 :param new_class:

3617 new storage class for the object. One of:

3618 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,

3619 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,

3620 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,

3621 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,

3622 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,

3623 or

3624 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`.

3625

3626 :type client: :class:`~google.cloud.storage.client.Client`

3627 :param client:

3628 (Optional) The client to use. If not passed, falls back to the

3629 ``client`` stored on the blob's bucket.

3630

3631 :type if_generation_match: long

3632 :param if_generation_match:

3633 (Optional) See :ref:`using-if-generation-match`

3634 Note that the generation to be matched is that of the

3635 ``destination`` blob.

3636

3637 :type if_generation_not_match: long

3638 :param if_generation_not_match:

3639 (Optional) See :ref:`using-if-generation-not-match`

3640 Note that the generation to be matched is that of the

3641 ``destination`` blob.

3642

3643 :type if_metageneration_match: long

3644 :param if_metageneration_match:

3645 (Optional) See :ref:`using-if-metageneration-match`

3646 Note that the metageneration to be matched is that of the

3647 ``destination`` blob.

3648

3649 :type if_metageneration_not_match: long

3650 :param if_metageneration_not_match:

3651 (Optional) See :ref:`using-if-metageneration-not-match`

3652 Note that the metageneration to be matched is that of the

3653 ``destination`` blob.

3654

3655 :type if_source_generation_match: long

3656 :param if_source_generation_match:

3657 (Optional) Makes the operation conditional on whether the source

3658 object's generation matches the given value.

3659

3660 :type if_source_generation_not_match: long

3661 :param if_source_generation_not_match:

3662 (Optional) Makes the operation conditional on whether the source

3663 object's generation does not match the given value.

3664

3665 :type if_source_metageneration_match: long

3666 :param if_source_metageneration_match:

3667 (Optional) Makes the operation conditional on whether the source

3668 object's current metageneration matches the given value.

3669

3670 :type if_source_metageneration_not_match: long

3671 :param if_source_metageneration_not_match:

3672 (Optional) Makes the operation conditional on whether the source

3673 object's current metageneration does not match the given value.

3674

3675 :type timeout: float or tuple

3676 :param timeout:

3677 (Optional) The amount of time, in seconds, to wait

3678 for the server response. See: :ref:`configuring_timeouts`

3679

3680 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

3681 :param retry:

3682 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

3683 """

3684 # Update current blob's storage class prior to rewrite

3685 self._patch_property("storageClass", new_class)

3686

3687 # Execute consecutive rewrite operations until operation is done

3688 token, _, _ = self.rewrite(

3689 self,

3690 if_generation_match=if_generation_match,

3691 if_generation_not_match=if_generation_not_match,

3692 if_metageneration_match=if_metageneration_match,

3693 if_metageneration_not_match=if_metageneration_not_match,

3694 if_source_generation_match=if_source_generation_match,

3695 if_source_generation_not_match=if_source_generation_not_match,

3696 if_source_metageneration_match=if_source_metageneration_match,

3697 if_source_metageneration_not_match=if_source_metageneration_not_match,

3698 timeout=timeout,

3699 retry=retry,

3700 )

3701 while token is not None:

3702 token, _, _ = self.rewrite(

3703 self,

3704 token=token,

3705 if_generation_match=if_generation_match,

3706 if_generation_not_match=if_generation_not_match,

3707 if_metageneration_match=if_metageneration_match,

3708 if_metageneration_not_match=if_metageneration_not_match,

3709 if_source_generation_match=if_source_generation_match,

3710 if_source_generation_not_match=if_source_generation_not_match,

3711 if_source_metageneration_match=if_source_metageneration_match,

3712 if_source_metageneration_not_match=if_source_metageneration_not_match,

3713 timeout=timeout,

3714 retry=retry,

3715 )

3716

3717 def open(

3718 self,

3719 mode="r",

3720 chunk_size=None,

3721 ignore_flush=None,

3722 encoding=None,

3723 errors=None,

3724 newline=None,

3725 **kwargs,

3726 ):

3727 r"""Create a file handler for file-like I/O to or from this blob.

3728

3729 This method can be used as a context manager, just like Python's

3730 built-in 'open()' function.

3731

3732 While reading, as with other read methods, if blob.generation is not set

3733 the most recent blob generation will be used. Because the file-like IO

3734 reader downloads progressively in chunks, this could result in data from

3735 multiple versions being mixed together. If this is a concern, use

3736 either bucket.get_blob(), or blob.reload(), which will download the

3737 latest generation number and set it; or, if the generation is known, set

3738 it manually, for instance with bucket.blob(generation=123456).

3739

3740 Checksumming (hashing) to verify data integrity is disabled for reads

3741 using this feature because reads are implemented using request ranges,

3742 which do not provide checksums to validate. See

3743 https://cloud.google.com/storage/docs/hashes-etags for details.

3744

3745 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_fileio_write_read.py).

3746

3747 Keyword arguments to pass to the underlying API calls.

3748 For both uploads and downloads, the following arguments are

3749 supported:

3750

3751 - ``if_generation_match``

3752 - ``if_generation_not_match``

3753 - ``if_metageneration_match``

3754 - ``if_metageneration_not_match``

3755 - ``timeout``

3756 - ``retry``

3757

3758 For downloads only, the following additional arguments are supported:

3759

3760 - ``raw_download``

3761

3762 For uploads only, the following additional arguments are supported:

3763

3764 - ``content_type``

3765 - ``num_retries``

3766 - ``predefined_acl``

3767 - ``checksum``

3768

3769 .. note::

3770

3771 ``num_retries`` is supported for backwards-compatibility

3772 reasons only; please use ``retry`` with a Retry object or

3773 ConditionalRetryPolicy instead.

3774

3775 :type mode: str

3776 :param mode:

3777 (Optional) A mode string, as per standard Python `open()` semantics.The first

3778 character must be 'r', to open the blob for reading, or 'w' to open

3779 it for writing. The second character, if present, must be 't' for

3780 (unicode) text mode, or 'b' for bytes mode. If the second character

3781 is omitted, text mode is the default.

3782

3783 :type chunk_size: long

3784 :param chunk_size:

3785 (Optional) For reads, the minimum number of bytes to read at a time.

3786 If fewer bytes than the chunk_size are requested, the remainder is

3787 buffered. For writes, the maximum number of bytes to buffer before

3788 sending data to the server, and the size of each request when data

3789 is sent. Writes are implemented as a "resumable upload", so

3790 chunk_size for writes must be exactly a multiple of 256KiB as with

3791 other resumable uploads. The default is 40 MiB.

3792

3793 :type ignore_flush: bool

3794 :param ignore_flush:

3795 (Optional) For non text-mode writes, makes flush() do nothing

3796 instead of raising an error. flush() without closing is not

3797 supported by the remote service and therefore calling it normally

3798 results in io.UnsupportedOperation. However, that behavior is

3799 incompatible with some consumers and wrappers of file objects in

3800 Python, such as zipfile.ZipFile or io.TextIOWrapper. Setting

3801 ignore_flush will cause flush() to successfully do nothing, for

3802 compatibility with those contexts. The correct way to actually flush

3803 data to the remote server is to close() (using a context manager,

3804 such as in the example, will cause this to happen automatically).

3805

3806 :type encoding: str

3807 :param encoding:

3808 (Optional) For text mode only, the name of the encoding that the stream will

3809 be decoded or encoded with. If omitted, it defaults to

3810 locale.getpreferredencoding(False).

3811

3812 :type errors: str

3813 :param errors:

3814 (Optional) For text mode only, an optional string that specifies how encoding

3815 and decoding errors are to be handled. Pass 'strict' to raise a

3816 ValueError exception if there is an encoding error (the default of

3817 None has the same effect), or pass 'ignore' to ignore errors. (Note

3818 that ignoring encoding errors can lead to data loss.) Other more

3819 rarely-used options are also available; see the Python 'io' module

3820 documentation for 'io.TextIOWrapper' for a complete list.

3821

3822 :type newline: str

3823 :param newline:

3824 (Optional) For text mode only, controls how line endings are handled. It can

3825 be None, '', '\n', '\r', and '\r\n'. If None, reads use "universal

3826 newline mode" and writes use the system default. See the Python

3827 'io' module documentation for 'io.TextIOWrapper' for details.

3828

3829 :returns: A 'BlobReader' or 'BlobWriter' from

3830 'google.cloud.storage.fileio', or an 'io.TextIOWrapper' around one

3831 of those classes, depending on the 'mode' argument.

3832 """

3833 if mode == "rb":

3834 if encoding or errors or newline:

3835 raise ValueError(

3836 "encoding, errors and newline arguments are for text mode only"

3837 )

3838 if ignore_flush:

3839 raise ValueError(

3840 "ignore_flush argument is for non-text write mode only"

3841 )

3842 return BlobReader(self, chunk_size=chunk_size, **kwargs)

3843 elif mode == "wb":

3844 if encoding or errors or newline:

3845 raise ValueError(

3846 "encoding, errors and newline arguments are for text mode only"

3847 )

3848 return BlobWriter(

3849 self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs

3850 )

3851 elif mode in ("r", "rt"):

3852 if ignore_flush:

3853 raise ValueError(

3854 "ignore_flush argument is for non-text write mode only"

3855 )

3856 return TextIOWrapper(

3857 BlobReader(self, chunk_size=chunk_size, **kwargs),

3858 encoding=encoding,

3859 errors=errors,

3860 newline=newline,

3861 )

3862 elif mode in ("w", "wt"):

3863 if ignore_flush is False:

3864 raise ValueError(

3865 "ignore_flush is required for text mode writing and "

3866 "cannot be set to False"

3867 )

3868 return TextIOWrapper(

3869 BlobWriter(self, chunk_size=chunk_size, ignore_flush=True, **kwargs),

3870 encoding=encoding,

3871 errors=errors,

3872 newline=newline,

3873 )

3874 else:

3875 raise NotImplementedError(

3876 "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only."

3877 )

3878

3879 cache_control = _scalar_property("cacheControl")

3880 """HTTP 'Cache-Control' header for this object.

3881

3882 See [`RFC 7234`](https://tools.ietf.org/html/rfc7234#section-5.2)

3883 and [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3884

3885 :rtype: str or ``NoneType``

3886

3887 """

3888

3889 content_disposition = _scalar_property("contentDisposition")

3890 """HTTP 'Content-Disposition' header for this object.

3891

3892 See [`RFC 6266`](https://tools.ietf.org/html/rfc7234#section-5.2) and

3893 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3894

3895 :rtype: str or ``NoneType``

3896 """

3897

3898 content_encoding = _scalar_property("contentEncoding")

3899 """HTTP 'Content-Encoding' header for this object.

3900

3901 See [`RFC 7231`](https://tools.ietf.org/html/rfc7231#section-3.1.2.2) and

3902 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3903

3904 :rtype: str or ``NoneType``

3905 """

3906

3907 content_language = _scalar_property("contentLanguage")

3908 """HTTP 'Content-Language' header for this object.

3909

3910 See [`BCP47`](https://tools.ietf.org/html/bcp47) and

3911 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3912

3913 :rtype: str or ``NoneType``

3914 """

3915

3916 content_type = _scalar_property(_CONTENT_TYPE_FIELD)

3917 """HTTP 'Content-Type' header for this object.

3918

3919 See [`RFC 2616`](https://tools.ietf.org/html/rfc2616#section-14.17) and

3920 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3921

3922 :rtype: str or ``NoneType``

3923 """

3924

3925 crc32c = _scalar_property("crc32c")

3926 """CRC32C checksum for this object.

3927

3928 This returns the blob's CRC32C checksum. To retrieve the value, first use a

3929 reload method of the Blob class which loads the blob's properties from the server.

3930

3931 See [`RFC 4960`](https://tools.ietf.org/html/rfc4960#appendix-B) and

3932 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3933

3934 If not set before upload, the server will compute the hash.

3935

3936 :rtype: str or ``NoneType``

3937 """

3938

3939 @property

3940 def component_count(self):

3941 """Number of underlying components that make up this object.

3942

3943 See https://cloud.google.com/storage/docs/json_api/v1/objects

3944

3945 :rtype: int or ``NoneType``

3946 :returns: The component count (in case of a composed object) or

3947 ``None`` if the blob's resource has not been loaded from

3948 the server. This property will not be set on objects

3949 not created via ``compose``.

3950 """

3951 component_count = self._properties.get("componentCount")

3952 if component_count is not None:

3953 return int(component_count)

3954

3955 @property

3956 def etag(self):

3957 """Retrieve the ETag for the object.

3958

3959 See [`RFC 2616 (etags)`](https://tools.ietf.org/html/rfc2616#section-3.11) and

3960 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3961

3962 :rtype: str or ``NoneType``

3963 :returns: The blob etag or ``None`` if the blob's resource has not

3964 been loaded from the server.

3965 """

3966 return self._properties.get("etag")

3967

3968 event_based_hold = _scalar_property("eventBasedHold")

3969 """Is an event-based hold active on the object?

3970

3971 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

3972

3973 If the property is not set locally, returns :data:`None`.

3974

3975 :rtype: bool or ``NoneType``

3976 """

3977

3978 @property

3979 def generation(self):

3980 """Retrieve the generation for the object.

3981

3982 See https://cloud.google.com/storage/docs/json_api/v1/objects

3983

3984 :rtype: int or ``NoneType``

3985 :returns: The generation of the blob or ``None`` if the blob's

3986 resource has not been loaded from the server.

3987 """

3988 generation = self._properties.get("generation")

3989 if generation is not None:

3990 return int(generation)

3991

3992 @property

3993 def id(self):

3994 """Retrieve the ID for the object.

3995

3996 See https://cloud.google.com/storage/docs/json_api/v1/objects

3997

3998 The ID consists of the bucket name, object name, and generation number.

3999

4000 :rtype: str or ``NoneType``

4001 :returns: The ID of the blob or ``None`` if the blob's

4002 resource has not been loaded from the server.

4003 """

4004 return self._properties.get("id")

4005

4006 md5_hash = _scalar_property("md5Hash")

4007 """MD5 hash for this object.

4008

4009 This returns the blob's MD5 hash. To retrieve the value, first use a

4010 reload method of the Blob class which loads the blob's properties from the server.

4011

4012 See [`RFC 1321`](https://tools.ietf.org/html/rfc1321) and

4013 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

4014

4015 If not set before upload, the server will compute the hash.

4016

4017 :rtype: str or ``NoneType``

4018 """

4019

4020 @property

4021 def media_link(self):

4022 """Retrieve the media download URI for the object.

4023

4024 See https://cloud.google.com/storage/docs/json_api/v1/objects

4025

4026 :rtype: str or ``NoneType``

4027 :returns: The media link for the blob or ``None`` if the blob's

4028 resource has not been loaded from the server.

4029 """

4030 return self._properties.get("mediaLink")

4031

4032 @property

4033 def metadata(self):

4034 """Retrieve arbitrary/application specific metadata for the object.

4035

4036 See https://cloud.google.com/storage/docs/json_api/v1/objects

4037

4038 :setter: Update arbitrary/application specific metadata for the

4039 object.

4040 :getter: Retrieve arbitrary/application specific metadata for

4041 the object.

4042

4043 :rtype: dict or ``NoneType``

4044 :returns: The metadata associated with the blob or ``None`` if the

4045 property is not set.

4046 """

4047 return copy.deepcopy(self._properties.get("metadata"))

4048

4049 @metadata.setter

4050 def metadata(self, value):

4051 """Update arbitrary/application specific metadata for the object.

4052

4053 Values are stored to GCS as strings. To delete a key, set its value to

4054 None and call blob.patch().

4055

4056 See https://cloud.google.com/storage/docs/json_api/v1/objects

4057

4058 :type value: dict

4059 :param value: The blob metadata to set.

4060 """

4061 if value is not None:

4062 value = {k: str(v) if v is not None else None for k, v in value.items()}

4063 self._patch_property("metadata", value)

4064

4065 @property

4066 def metageneration(self):

4067 """Retrieve the metageneration for the object.

4068

4069 See https://cloud.google.com/storage/docs/json_api/v1/objects

4070

4071 :rtype: int or ``NoneType``

4072 :returns: The metageneration of the blob or ``None`` if the blob's

4073 resource has not been loaded from the server.

4074 """

4075 metageneration = self._properties.get("metageneration")

4076 if metageneration is not None:

4077 return int(metageneration)

4078

4079 @property

4080 def owner(self):

4081 """Retrieve info about the owner of the object.

4082

4083 See https://cloud.google.com/storage/docs/json_api/v1/objects

4084

4085 :rtype: dict or ``NoneType``

4086 :returns: Mapping of owner's role/ID, or ``None`` if the blob's

4087 resource has not been loaded from the server.

4088 """

4089 return copy.deepcopy(self._properties.get("owner"))

4090

4091 @property

4092 def retention_expiration_time(self):

4093 """Retrieve timestamp at which the object's retention period expires.

4094

4095 See https://cloud.google.com/storage/docs/json_api/v1/objects

4096

4097 :rtype: :class:`datetime.datetime` or ``NoneType``

4098 :returns: Datetime object parsed from RFC3339 valid timestamp, or

4099 ``None`` if the property is not set locally.

4100 """

4101 value = self._properties.get("retentionExpirationTime")

4102 if value is not None:

4103 return _rfc3339_nanos_to_datetime(value)

4104

4105 @property

4106 def self_link(self):

4107 """Retrieve the URI for the object.

4108

4109 See https://cloud.google.com/storage/docs/json_api/v1/objects

4110

4111 :rtype: str or ``NoneType``

4112 :returns: The self link for the blob or ``None`` if the blob's

4113 resource has not been loaded from the server.

4114 """

4115 return self._properties.get("selfLink")

4116

4117 @property

4118 def size(self):

4119 """Size of the object, in bytes.

4120

4121 See https://cloud.google.com/storage/docs/json_api/v1/objects

4122

4123 :rtype: int or ``NoneType``

4124 :returns: The size of the blob or ``None`` if the blob's

4125 resource has not been loaded from the server.

4126 """

4127 size = self._properties.get("size")

4128 if size is not None:

4129 return int(size)

4130

4131 @property

4132 def kms_key_name(self):

4133 """Resource name of Cloud KMS key used to encrypt the blob's contents.

4134

4135 :rtype: str or ``NoneType``

4136 :returns:

4137 The resource name or ``None`` if no Cloud KMS key was used,

4138 or the blob's resource has not been loaded from the server.

4139 """

4140 return self._properties.get("kmsKeyName")

4141

4142 @kms_key_name.setter

4143 def kms_key_name(self, value):

4144 """Set KMS encryption key for object.

4145

4146 :type value: str or ``NoneType``

4147 :param value: new KMS key name (None to clear any existing key).

4148 """

4149 self._patch_property("kmsKeyName", value)

4150

4151 storage_class = _scalar_property("storageClass")

4152 """Retrieve the storage class for the object.

4153

4154 This can only be set at blob / object **creation** time. If you'd

4155 like to change the storage class **after** the blob / object already

4156 exists in a bucket, call :meth:`update_storage_class` (which uses

4157 :meth:`rewrite`).

4158

4159 See https://cloud.google.com/storage/docs/storage-classes

4160

4161 :rtype: str or ``NoneType``

4162 :returns:

4163 If set, one of

4164 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,

4165 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,

4166 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,

4167 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,

4168 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,

4169 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,

4170 :attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`,

4171 else ``None``.

4172 """

4173

4174 temporary_hold = _scalar_property("temporaryHold")

4175 """Is a temporary hold active on the object?

4176

4177 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects).

4178

4179 If the property is not set locally, returns :data:`None`.

4180

4181 :rtype: bool or ``NoneType``

4182 """

4183

4184 @property

4185 def time_deleted(self):

4186 """Retrieve the timestamp at which the object was deleted.

4187

4188 See https://cloud.google.com/storage/docs/json_api/v1/objects

4189

4190 :rtype: :class:`datetime.datetime` or ``NoneType``

4191 :returns: Datetime object parsed from RFC3339 valid timestamp, or

4192 ``None`` if the blob's resource has not been loaded from

4193 the server (see :meth:`reload`). If the blob has

4194 not been deleted, this will never be set.

4195 """

4196 value = self._properties.get("timeDeleted")

4197 if value is not None:

4198 return _rfc3339_nanos_to_datetime(value)

4199

4200 @property

4201 def time_created(self):

4202 """Retrieve the timestamp at which the object was created.

4203

4204 See https://cloud.google.com/storage/docs/json_api/v1/objects

4205

4206 :rtype: :class:`datetime.datetime` or ``NoneType``

4207 :returns: Datetime object parsed from RFC3339 valid timestamp, or

4208 ``None`` if the blob's resource has not been loaded from

4209 the server (see :meth:`reload`).

4210 """

4211 value = self._properties.get("timeCreated")

4212 if value is not None:

4213 return _rfc3339_nanos_to_datetime(value)

4214

4215 @property

4216 def updated(self):

4217 """Retrieve the timestamp at which the object was updated.

4218

4219 See https://cloud.google.com/storage/docs/json_api/v1/objects

4220

4221 :rtype: :class:`datetime.datetime` or ``NoneType``

4222 :returns: Datetime object parsed from RFC3339 valid timestamp, or

4223 ``None`` if the blob's resource has not been loaded from

4224 the server (see :meth:`reload`).

4225 """

4226 value = self._properties.get("updated")

4227 if value is not None:

4228 return _rfc3339_nanos_to_datetime(value)

4229

4230 @property

4231 def custom_time(self):

4232 """Retrieve the custom time for the object.

4233

4234 See https://cloud.google.com/storage/docs/json_api/v1/objects

4235

4236 :rtype: :class:`datetime.datetime` or ``NoneType``

4237 :returns: Datetime object parsed from RFC3339 valid timestamp, or

4238 ``None`` if the blob's resource has not been loaded from

4239 the server (see :meth:`reload`).

4240 """

4241 value = self._properties.get("customTime")

4242 if value is not None:

4243 return _rfc3339_nanos_to_datetime(value)

4244

4245 @custom_time.setter

4246 def custom_time(self, value):

4247 """Set the custom time for the object.

4248

4249 Once set on the server side object, this value can't be unset, but may

4250 only changed to a custom datetime in the future.

4251

4252 If :attr:`custom_time` must be unset, either perform a rewrite

4253 operation or upload the data again.

4254

4255 See https://cloud.google.com/storage/docs/json_api/v1/objects

4256

4257 :type value: :class:`datetime.datetime`

4258 :param value: new value

4259 """

4260 if value is not None:

4261 value = _datetime_to_rfc3339(value)

4262

4263 self._patch_property("customTime", value)

4264

4265

4266def _get_host_name(connection):

4267 """Returns the host name from the given connection.

4268

4269 :type connection: :class:`~google.cloud.storage._http.Connection`

4270 :param connection: The connection object.

4271

4272 :rtype: str

4273 :returns: The host name.

4274 """

4275 # TODO: After google-cloud-core 1.6.0 is stable and we upgrade it

4276 # to 1.6.0 in setup.py, we no longer need to check the attribute

4277 # existence. We can simply return connection.get_api_base_url_for_mtls().

4278 return (

4279 connection.API_BASE_URL

4280 if not hasattr(connection, "get_api_base_url_for_mtls")

4281 else connection.get_api_base_url_for_mtls()

4282 )

4283

4284

4285def _get_encryption_headers(key, source=False):

4286 """Builds customer encryption key headers

4287

4288 :type key: bytes

4289 :param key: 32 byte key to build request key and hash.

4290

4291 :type source: bool

4292 :param source: If true, return headers for the "source" blob; otherwise,

4293 return headers for the "destination" blob.

4294

4295 :rtype: dict

4296 :returns: dict of HTTP headers being sent in request.

4297 """

4298 if key is None:

4299 return {}

4300

4301 key = _to_bytes(key)

4302 key_hash = hashlib.sha256(key).digest()

4303 key_hash = base64.b64encode(key_hash)

4304 key = base64.b64encode(key)

4305

4306 if source:

4307 prefix = "X-Goog-Copy-Source-Encryption-"

4308 else:

4309 prefix = "X-Goog-Encryption-"

4310

4311 return {

4312 prefix + "Algorithm": "AES256",

4313 prefix + "Key": _bytes_to_unicode(key),

4314 prefix + "Key-Sha256": _bytes_to_unicode(key_hash),

4315 }

4316

4317

4318def _quote(value, safe=b"~"):

4319 """URL-quote a string.

4320

4321 If the value is unicode, this method first UTF-8 encodes it as bytes and

4322 then quotes the bytes. (In Python 3, ``urllib.parse.quote`` does this

4323 encoding automatically, but in Python 2, non-ASCII characters cannot be

4324 quoted.)

4325

4326 :type value: str or bytes

4327 :param value: The value to be URL-quoted.

4328

4329 :type safe: bytes

4330 :param safe: Bytes *not* to be quoted. By default, includes only ``b'~'``.

4331

4332 :rtype: str

4333 :returns: The encoded value (bytes in Python 2, unicode in Python 3).

4334 """

4335 value = _to_bytes(value, encoding="utf-8")

4336 return quote(value, safe=safe)

4337

4338

4339def _maybe_rewind(stream, rewind=False):

4340 """Rewind the stream if desired.

4341

4342 :type stream: IO[bytes]

4343 :param stream: A bytes IO object open for reading.

4344

4345 :type rewind: bool

4346 :param rewind: Indicates if we should seek to the beginning of the stream.

4347 """

4348 if rewind:

4349 stream.seek(0, os.SEEK_SET)

4350

4351

4352def _raise_from_invalid_response(error):

4353 """Re-wrap and raise an ``InvalidResponse`` exception.

4354

4355 :type error: :exc:`google.resumable_media.InvalidResponse`

4356 :param error: A caught exception from the ``google-resumable-media``

4357 library.

4358

4359 :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding

4360 to the failed status code

4361 """

4362 response = error.response

4363

4364 # The 'response.text' gives the actual reason of error, where 'error' gives

4365 # the message of expected status code.

4366 if response.text:

4367 error_message = response.text + ": " + str(error)

4368 else:

4369 error_message = str(error)

4370

4371 message = f"{response.request.method} {response.request.url}: {error_message}"

4372

4373 raise exceptions.from_http_status(response.status_code, message, response=response)

4374

4375

4376def _add_query_parameters(base_url, name_value_pairs):

4377 """Add one query parameter to a base URL.

4378

4379 :type base_url: string

4380 :param base_url: Base URL (may already contain query parameters)

4381

4382 :type name_value_pairs: list of (string, string) tuples.

4383 :param name_value_pairs: Names and values of the query parameters to add

4384

4385 :rtype: string

4386 :returns: URL with additional query strings appended.

4387 """

4388 if len(name_value_pairs) == 0:

4389 return base_url

4390

4391 scheme, netloc, path, query, frag = urlsplit(base_url)

4392 query = parse_qsl(query)

4393 query.extend(name_value_pairs)

4394 return urlunsplit((scheme, netloc, path, urlencode(query), frag))