Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/blob.py: 30%

657 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 07:13 +0000

1# Copyright 2014 Google LLC 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15# pylint: disable=too-many-lines 

16 

17"""Create / interact with Google Cloud Storage blobs. 

18""" 

19 

20import base64 

21import copy 

22import hashlib 

23from io import BytesIO 

24from io import TextIOWrapper 

25import logging 

26import mimetypes 

27import os 

28import re 

29from email.parser import HeaderParser 

30from urllib.parse import parse_qsl 

31from urllib.parse import quote 

32from urllib.parse import urlencode 

33from urllib.parse import urlsplit 

34from urllib.parse import urlunsplit 

35import warnings 

36 

37from google import resumable_media 

38from google.resumable_media.requests import ChunkedDownload 

39from google.resumable_media.requests import Download 

40from google.resumable_media.requests import RawDownload 

41from google.resumable_media.requests import RawChunkedDownload 

42from google.resumable_media.requests import MultipartUpload 

43from google.resumable_media.requests import ResumableUpload 

44 

45from google.api_core.iam import Policy 

46from google.cloud import exceptions 

47from google.cloud._helpers import _bytes_to_unicode 

48from google.cloud._helpers import _datetime_to_rfc3339 

49from google.cloud._helpers import _rfc3339_nanos_to_datetime 

50from google.cloud._helpers import _to_bytes 

51from google.cloud.exceptions import NotFound 

52from google.cloud.storage._helpers import _add_etag_match_headers 

53from google.cloud.storage._helpers import _add_generation_match_parameters 

54from google.cloud.storage._helpers import _PropertyMixin 

55from google.cloud.storage._helpers import _scalar_property 

56from google.cloud.storage._helpers import _bucket_bound_hostname_url 

57from google.cloud.storage._helpers import _raise_if_more_than_one_set 

58from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry 

59from google.cloud.storage._helpers import _get_default_headers 

60from google.cloud.storage._signing import generate_signed_url_v2 

61from google.cloud.storage._signing import generate_signed_url_v4 

62from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE 

63from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST 

64from google.cloud.storage._helpers import _API_VERSION 

65from google.cloud.storage.acl import ACL 

66from google.cloud.storage.acl import ObjectACL 

67from google.cloud.storage.constants import _DEFAULT_TIMEOUT 

68from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS 

69from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS 

70from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS 

71from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS 

72from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS 

73from google.cloud.storage.constants import STANDARD_STORAGE_CLASS 

74from google.cloud.storage.retry import ConditionalRetryPolicy 

75from google.cloud.storage.retry import DEFAULT_RETRY 

76from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON 

77from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED 

78from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED 

79from google.cloud.storage.fileio import BlobReader 

80from google.cloud.storage.fileio import BlobWriter 

81 

82 

83_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST 

84_DEFAULT_CONTENT_TYPE = "application/octet-stream" 

85_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media" 

86_BASE_UPLOAD_TEMPLATE = ( 

87 "{hostname}/upload/storage/{api_version}{bucket_path}/o?uploadType=" 

88) 

89_MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "multipart" 

90_RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "resumable" 

91# NOTE: "acl" is also writeable but we defer ACL management to 

92# the classes in the google.cloud.storage.acl module. 

93_CONTENT_TYPE_FIELD = "contentType" 

94_WRITABLE_FIELDS = ( 

95 "cacheControl", 

96 "contentDisposition", 

97 "contentEncoding", 

98 "contentLanguage", 

99 _CONTENT_TYPE_FIELD, 

100 "crc32c", 

101 "customTime", 

102 "md5Hash", 

103 "metadata", 

104 "name", 

105 "storageClass", 

106) 

107_READ_LESS_THAN_SIZE = ( 

108 "Size {:d} was specified but the file-like object only had " "{:d} bytes remaining." 

109) 

110_CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE = ( 

111 "A checksum of type `{}` was requested, but checksumming is not available " 

112 "for downloads when chunk_size is set." 

113) 

114_COMPOSE_IF_GENERATION_LIST_DEPRECATED = ( 

115 "'if_generation_match: type list' is deprecated and supported for " 

116 "backwards-compatability reasons only. Use 'if_source_generation_match' " 

117 "instead' to match source objects' generations." 

118) 

119_COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR = ( 

120 "Use 'if_generation_match' to match the generation of the destination " 

121 "object by passing in a generation number, instead of a list. " 

122 "Use 'if_source_generation_match' to match source objects generations." 

123) 

124_COMPOSE_IF_METAGENERATION_LIST_DEPRECATED = ( 

125 "'if_metageneration_match: type list' is deprecated and supported for " 

126 "backwards-compatability reasons only. Note that the metageneration to " 

127 "be matched is that of the destination blob. Please pass in a single " 

128 "value (type long)." 

129) 

130_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR = ( 

131 "'if_source_generation_match' length must be the same as 'sources' length" 

132) 

133_DOWNLOAD_AS_STRING_DEPRECATED = ( 

134 "Blob.download_as_string() is deprecated and will be removed in future. " 

135 "Use Blob.download_as_bytes() instead." 

136) 

137 

138 

139_DEFAULT_CHUNKSIZE = 104857600 # 1024 * 1024 B * 100 = 100 MB 

140_MAX_MULTIPART_SIZE = 8388608 # 8 MB 

141 

142_logger = logging.getLogger(__name__) 

143 

144 

145class Blob(_PropertyMixin): 

146 """A wrapper around Cloud Storage's concept of an ``Object``. 

147 

148 :type name: str 

149 :param name: The name of the blob. This corresponds to the unique path of 

150 the object in the bucket. If bytes, will be converted to a 

151 unicode object. Blob / object names can contain any sequence 

152 of valid unicode characters, of length 1-1024 bytes when 

153 UTF-8 encoded. 

154 

155 :type bucket: :class:`google.cloud.storage.bucket.Bucket` 

156 :param bucket: The bucket to which this blob belongs. 

157 

158 :type chunk_size: int 

159 :param chunk_size: 

160 (Optional) The size of a chunk of data whenever iterating (in bytes). 

161 This must be a multiple of 256 KB per the API specification. If not 

162 specified, the chunk_size of the blob itself is used. If that is not 

163 specified, a default value of 40 MB is used. 

164 

165 :type encryption_key: bytes 

166 :param encryption_key: 

167 (Optional) 32 byte encryption key for customer-supplied encryption. 

168 See https://cloud.google.com/storage/docs/encryption#customer-supplied. 

169 

170 :type kms_key_name: str 

171 :param kms_key_name: 

172 (Optional) Resource name of Cloud KMS key used to encrypt the blob's 

173 contents. 

174 

175 :type generation: long 

176 :param generation: 

177 (Optional) If present, selects a specific revision of this object. 

178 """ 

179 

180 _chunk_size = None # Default value for each instance. 

181 _CHUNK_SIZE_MULTIPLE = 256 * 1024 

182 """Number (256 KB, in bytes) that must divide the chunk size.""" 

183 

184 STORAGE_CLASSES = ( 

185 STANDARD_STORAGE_CLASS, 

186 NEARLINE_STORAGE_CLASS, 

187 COLDLINE_STORAGE_CLASS, 

188 ARCHIVE_STORAGE_CLASS, 

189 MULTI_REGIONAL_LEGACY_STORAGE_CLASS, 

190 REGIONAL_LEGACY_STORAGE_CLASS, 

191 ) 

192 """Allowed values for :attr:`storage_class`. 

193 

194 See 

195 https://cloud.google.com/storage/docs/json_api/v1/objects#storageClass 

196 https://cloud.google.com/storage/docs/per-object-storage-class 

197 

198 .. note:: 

199 This list does not include 'DURABLE_REDUCED_AVAILABILITY', which 

200 is only documented for buckets (and deprecated). 

201 """ 

202 

203 def __init__( 

204 self, 

205 name, 

206 bucket, 

207 chunk_size=None, 

208 encryption_key=None, 

209 kms_key_name=None, 

210 generation=None, 

211 ): 

212 """ 

213 property :attr:`name` 

214 Get the blob's name. 

215 """ 

216 name = _bytes_to_unicode(name) 

217 super(Blob, self).__init__(name=name) 

218 

219 self.chunk_size = chunk_size # Check that setter accepts value. 

220 self._bucket = bucket 

221 self._acl = ObjectACL(self) 

222 _raise_if_more_than_one_set( 

223 encryption_key=encryption_key, kms_key_name=kms_key_name 

224 ) 

225 

226 self._encryption_key = encryption_key 

227 

228 if kms_key_name is not None: 

229 self._properties["kmsKeyName"] = kms_key_name 

230 

231 if generation is not None: 

232 self._properties["generation"] = generation 

233 

234 @property 

235 def bucket(self): 

236 """Bucket which contains the object. 

237 

238 :rtype: :class:`~google.cloud.storage.bucket.Bucket` 

239 :returns: The object's bucket. 

240 """ 

241 return self._bucket 

242 

243 @property 

244 def chunk_size(self): 

245 """Get the blob's default chunk size. 

246 

247 :rtype: int or ``NoneType`` 

248 :returns: The current blob's chunk size, if it is set. 

249 """ 

250 return self._chunk_size 

251 

252 @chunk_size.setter 

253 def chunk_size(self, value): 

254 """Set the blob's default chunk size. 

255 

256 :type value: int 

257 :param value: (Optional) The current blob's chunk size, if it is set. 

258 

259 :raises: :class:`ValueError` if ``value`` is not ``None`` and is not a 

260 multiple of 256 KB. 

261 """ 

262 if value is not None and value > 0 and value % self._CHUNK_SIZE_MULTIPLE != 0: 

263 raise ValueError( 

264 "Chunk size must be a multiple of %d." % (self._CHUNK_SIZE_MULTIPLE,) 

265 ) 

266 self._chunk_size = value 

267 

268 @property 

269 def encryption_key(self): 

270 """Retrieve the customer-supplied encryption key for the object. 

271 

272 :rtype: bytes or ``NoneType`` 

273 :returns: 

274 The encryption key or ``None`` if no customer-supplied encryption key was used, 

275 or the blob's resource has not been loaded from the server. 

276 """ 

277 return self._encryption_key 

278 

279 @encryption_key.setter 

280 def encryption_key(self, value): 

281 """Set the blob's encryption key. 

282 

283 See https://cloud.google.com/storage/docs/encryption#customer-supplied 

284 

285 To perform a key rotation for an encrypted blob, use :meth:`rewrite`. 

286 See https://cloud.google.com/storage/docs/encryption/using-customer-supplied-keys?hl=ca#rotating 

287 

288 :type value: bytes 

289 :param value: 32 byte encryption key for customer-supplied encryption. 

290 """ 

291 self._encryption_key = value 

292 

293 @staticmethod 

294 def path_helper(bucket_path, blob_name): 

295 """Relative URL path for a blob. 

296 

297 :type bucket_path: str 

298 :param bucket_path: The URL path for a bucket. 

299 

300 :type blob_name: str 

301 :param blob_name: The name of the blob. 

302 

303 :rtype: str 

304 :returns: The relative URL path for ``blob_name``. 

305 """ 

306 return bucket_path + "/o/" + _quote(blob_name) 

307 

308 @property 

309 def acl(self): 

310 """Create our ACL on demand.""" 

311 return self._acl 

312 

313 def __repr__(self): 

314 if self.bucket: 

315 bucket_name = self.bucket.name 

316 else: 

317 bucket_name = None 

318 

319 return f"<Blob: {bucket_name}, {self.name}, {self.generation}>" 

320 

321 @property 

322 def path(self): 

323 """Getter property for the URL path to this Blob. 

324 

325 :rtype: str 

326 :returns: The URL path to this Blob. 

327 """ 

328 if not self.name: 

329 raise ValueError("Cannot determine path without a blob name.") 

330 

331 return self.path_helper(self.bucket.path, self.name) 

332 

333 @property 

334 def client(self): 

335 """The client bound to this blob.""" 

336 return self.bucket.client 

337 

338 @property 

339 def user_project(self): 

340 """Project ID billed for API requests made via this blob. 

341 

342 Derived from bucket's value. 

343 

344 :rtype: str 

345 """ 

346 return self.bucket.user_project 

347 

348 def _encryption_headers(self): 

349 """Return any encryption headers needed to fetch the object. 

350 

351 :rtype: List(Tuple(str, str)) 

352 :returns: a list of tuples to be passed as headers. 

353 """ 

354 return _get_encryption_headers(self._encryption_key) 

355 

356 @property 

357 def _query_params(self): 

358 """Default query parameters.""" 

359 params = {} 

360 if self.generation is not None: 

361 params["generation"] = self.generation 

362 if self.user_project is not None: 

363 params["userProject"] = self.user_project 

364 return params 

365 

366 @property 

367 def public_url(self): 

368 """The public URL for this blob. 

369 

370 Use :meth:`make_public` to enable anonymous access via the returned 

371 URL. 

372 

373 :rtype: `string` 

374 :returns: The public URL for this blob. 

375 """ 

376 return "{storage_base_url}/{bucket_name}/{quoted_name}".format( 

377 storage_base_url=_API_ACCESS_ENDPOINT, 

378 bucket_name=self.bucket.name, 

379 quoted_name=_quote(self.name, safe=b"/~"), 

380 ) 

381 

382 @classmethod 

383 def from_string(cls, uri, client=None): 

384 """Get a constructor for blob object by URI. 

385 

386 .. code-block:: python 

387 

388 from google.cloud import storage 

389 from google.cloud.storage.blob import Blob 

390 client = storage.Client() 

391 blob = Blob.from_string("gs://bucket/object", client=client) 

392 

393 :type uri: str 

394 :param uri: The blob uri pass to get blob object. 

395 

396 :type client: :class:`~google.cloud.storage.client.Client` 

397 :param client: 

398 (Optional) The client to use. Application code should 

399 *always* pass ``client``. 

400 

401 :rtype: :class:`google.cloud.storage.blob.Blob` 

402 :returns: The blob object created. 

403 """ 

404 from google.cloud.storage.bucket import Bucket 

405 

406 scheme, netloc, path, query, frag = urlsplit(uri) 

407 if scheme != "gs": 

408 raise ValueError("URI scheme must be gs") 

409 

410 bucket = Bucket(client, name=netloc) 

411 return cls(path[1:], bucket) 

412 

413 def generate_signed_url( 

414 self, 

415 expiration=None, 

416 api_access_endpoint=_API_ACCESS_ENDPOINT, 

417 method="GET", 

418 content_md5=None, 

419 content_type=None, 

420 response_disposition=None, 

421 response_type=None, 

422 generation=None, 

423 headers=None, 

424 query_parameters=None, 

425 client=None, 

426 credentials=None, 

427 version=None, 

428 service_account_email=None, 

429 access_token=None, 

430 virtual_hosted_style=False, 

431 bucket_bound_hostname=None, 

432 scheme="http", 

433 ): 

434 """Generates a signed URL for this blob. 

435 

436 .. note:: 

437 

438 If you are on Google Compute Engine, you can't generate a signed 

439 URL using GCE service account. 

440 If you'd like to be able to generate a signed URL from GCE, 

441 you can use a standard service account from a JSON file rather 

442 than a GCE service account. 

443 

444 If you have a blob that you want to allow access to for a set 

445 amount of time, you can use this method to generate a URL that 

446 is only valid within a certain time period. 

447 

448 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-generate-signed-url-v4#storage_generate_signed_url_v4-python). 

449 

450 This is particularly useful if you don't want publicly 

451 accessible blobs, but don't want to require users to explicitly 

452 log in. 

453 

454 If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`, 

455 ``https`` works only if using a ``CDN``. 

456 

457 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] 

458 :param expiration: 

459 Point in time when the signed URL should expire. If a ``datetime`` 

460 instance is passed without an explicit ``tzinfo`` set, it will be 

461 assumed to be ``UTC``. 

462 

463 :type api_access_endpoint: str 

464 :param api_access_endpoint: (Optional) URI base. 

465 

466 :type method: str 

467 :param method: The HTTP verb that will be used when requesting the URL. 

468 

469 :type content_md5: str 

470 :param content_md5: 

471 (Optional) The MD5 hash of the object referenced by ``resource``. 

472 

473 :type content_type: str 

474 :param content_type: 

475 (Optional) The content type of the object referenced by 

476 ``resource``. 

477 

478 :type response_disposition: str 

479 :param response_disposition: 

480 (Optional) Content disposition of responses to requests for the 

481 signed URL. For example, to enable the signed URL to initiate a 

482 file of ``blog.png``, use the value ``'attachment; 

483 filename=blob.png'``. 

484 

485 :type response_type: str 

486 :param response_type: 

487 (Optional) Content type of responses to requests for the signed 

488 URL. Ignored if content_type is set on object/blob metadata. 

489 

490 :type generation: str 

491 :param generation: 

492 (Optional) A value that indicates which generation of the resource 

493 to fetch. 

494 

495 :type headers: dict 

496 :param headers: 

497 (Optional) Additional HTTP headers to be included as part of the 

498 signed URLs. See: 

499 https://cloud.google.com/storage/docs/xml-api/reference-headers 

500 Requests using the signed URL *must* pass the specified header 

501 (name and value) with each request for the URL. 

502 

503 :type query_parameters: dict 

504 :param query_parameters: 

505 (Optional) Additional query parameters to be included as part of the 

506 signed URLs. See: 

507 https://cloud.google.com/storage/docs/xml-api/reference-headers#query 

508 

509 :type client: :class:`~google.cloud.storage.client.Client` 

510 :param client: 

511 (Optional) The client to use. If not passed, falls back to the 

512 ``client`` stored on the blob's bucket. 

513 

514 :type credentials: :class:`google.auth.credentials.Credentials` 

515 :param credentials: 

516 (Optional) The authorization credentials to attach to requests. 

517 These credentials identify this application to the service. If 

518 none are specified, the client will attempt to ascertain the 

519 credentials from the environment. 

520 

521 :type version: str 

522 :param version: 

523 (Optional) The version of signed credential to create. Must be one 

524 of 'v2' | 'v4'. 

525 

526 :type service_account_email: str 

527 :param service_account_email: 

528 (Optional) E-mail address of the service account. 

529 

530 :type access_token: str 

531 :param access_token: (Optional) Access token for a service account. 

532 

533 :type virtual_hosted_style: bool 

534 :param virtual_hosted_style: 

535 (Optional) If true, then construct the URL relative the bucket's 

536 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'. 

537 

538 :type bucket_bound_hostname: str 

539 :param bucket_bound_hostname: 

540 (Optional) If passed, then construct the URL relative to the 

541 bucket-bound hostname. Value can be a bare or with scheme, e.g., 

542 'example.com' or 'http://example.com'. See: 

543 https://cloud.google.com/storage/docs/request-endpoints#cname 

544 

545 :type scheme: str 

546 :param scheme: 

547 (Optional) If ``bucket_bound_hostname`` is passed as a bare 

548 hostname, use this value as the scheme. ``https`` will work only 

549 when using a CDN. Defaults to ``"http"``. 

550 

551 :raises: :exc:`ValueError` when version is invalid. 

552 :raises: :exc:`TypeError` when expiration is not a valid type. 

553 :raises: :exc:`AttributeError` if credentials is not an instance 

554 of :class:`google.auth.credentials.Signing`. 

555 

556 :rtype: str 

557 :returns: A signed URL you can use to access the resource 

558 until expiration. 

559 """ 

560 if version is None: 

561 version = "v2" 

562 elif version not in ("v2", "v4"): 

563 raise ValueError("'version' must be either 'v2' or 'v4'") 

564 

565 quoted_name = _quote(self.name, safe=b"/~") 

566 

567 # If you are on Google Compute Engine, you can't generate a signed URL 

568 # using GCE service account. 

569 # See https://github.com/googleapis/google-auth-library-python/issues/50 

570 if virtual_hosted_style: 

571 api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com" 

572 elif bucket_bound_hostname: 

573 api_access_endpoint = _bucket_bound_hostname_url( 

574 bucket_bound_hostname, scheme 

575 ) 

576 else: 

577 resource = f"/{self.bucket.name}/{quoted_name}" 

578 

579 if virtual_hosted_style or bucket_bound_hostname: 

580 resource = f"/{quoted_name}" 

581 

582 if credentials is None: 

583 client = self._require_client(client) 

584 credentials = client._credentials 

585 

586 if version == "v2": 

587 helper = generate_signed_url_v2 

588 else: 

589 helper = generate_signed_url_v4 

590 

591 if self._encryption_key is not None: 

592 encryption_headers = _get_encryption_headers(self._encryption_key) 

593 if headers is None: 

594 headers = {} 

595 if version == "v2": 

596 # See: https://cloud.google.com/storage/docs/access-control/signed-urls-v2#about-canonical-extension-headers 

597 v2_copy_only = "X-Goog-Encryption-Algorithm" 

598 headers[v2_copy_only] = encryption_headers[v2_copy_only] 

599 else: 

600 headers.update(encryption_headers) 

601 

602 return helper( 

603 credentials, 

604 resource=resource, 

605 expiration=expiration, 

606 api_access_endpoint=api_access_endpoint, 

607 method=method.upper(), 

608 content_md5=content_md5, 

609 content_type=content_type, 

610 response_type=response_type, 

611 response_disposition=response_disposition, 

612 generation=generation, 

613 headers=headers, 

614 query_parameters=query_parameters, 

615 service_account_email=service_account_email, 

616 access_token=access_token, 

617 ) 

618 

619 def exists( 

620 self, 

621 client=None, 

622 if_etag_match=None, 

623 if_etag_not_match=None, 

624 if_generation_match=None, 

625 if_generation_not_match=None, 

626 if_metageneration_match=None, 

627 if_metageneration_not_match=None, 

628 timeout=_DEFAULT_TIMEOUT, 

629 retry=DEFAULT_RETRY, 

630 ): 

631 """Determines whether or not this blob exists. 

632 

633 If :attr:`user_project` is set on the bucket, bills the API request 

634 to that project. 

635 

636 :type client: :class:`~google.cloud.storage.client.Client` 

637 :param client: 

638 (Optional) The client to use. If not passed, falls back to the 

639 ``client`` stored on the blob's bucket. 

640 

641 :type if_etag_match: Union[str, Set[str]] 

642 :param if_etag_match: 

643 (Optional) See :ref:`using-if-etag-match` 

644 

645 :type if_etag_not_match: Union[str, Set[str]] 

646 :param if_etag_not_match: 

647 (Optional) See :ref:`using-if-etag-not-match` 

648 

649 :type if_generation_match: long 

650 :param if_generation_match: 

651 (Optional) See :ref:`using-if-generation-match` 

652 

653 :type if_generation_not_match: long 

654 :param if_generation_not_match: 

655 (Optional) See :ref:`using-if-generation-not-match` 

656 

657 :type if_metageneration_match: long 

658 :param if_metageneration_match: 

659 (Optional) See :ref:`using-if-metageneration-match` 

660 

661 :type if_metageneration_not_match: long 

662 :param if_metageneration_not_match: 

663 (Optional) See :ref:`using-if-metageneration-not-match` 

664 

665 :type timeout: float or tuple 

666 :param timeout: 

667 (Optional) The amount of time, in seconds, to wait 

668 for the server response. See: :ref:`configuring_timeouts` 

669 

670 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

671 :param retry: 

672 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

673 

674 :rtype: bool 

675 :returns: True if the blob exists in Cloud Storage. 

676 """ 

677 client = self._require_client(client) 

678 # We only need the status code (200 or not) so we seek to 

679 # minimize the returned payload. 

680 query_params = self._query_params 

681 query_params["fields"] = "name" 

682 

683 _add_generation_match_parameters( 

684 query_params, 

685 if_generation_match=if_generation_match, 

686 if_generation_not_match=if_generation_not_match, 

687 if_metageneration_match=if_metageneration_match, 

688 if_metageneration_not_match=if_metageneration_not_match, 

689 ) 

690 

691 headers = {} 

692 _add_etag_match_headers( 

693 headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match 

694 ) 

695 

696 try: 

697 # We intentionally pass `_target_object=None` since fields=name 

698 # would limit the local properties. 

699 client._get_resource( 

700 self.path, 

701 query_params=query_params, 

702 headers=headers, 

703 timeout=timeout, 

704 retry=retry, 

705 _target_object=None, 

706 ) 

707 except NotFound: 

708 # NOTE: This will not fail immediately in a batch. However, when 

709 # Batch.finish() is called, the resulting `NotFound` will be 

710 # raised. 

711 return False 

712 return True 

713 

714 def delete( 

715 self, 

716 client=None, 

717 if_generation_match=None, 

718 if_generation_not_match=None, 

719 if_metageneration_match=None, 

720 if_metageneration_not_match=None, 

721 timeout=_DEFAULT_TIMEOUT, 

722 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

723 ): 

724 """Deletes a blob from Cloud Storage. 

725 

726 If :attr:`user_project` is set on the bucket, bills the API request 

727 to that project. 

728 

729 :type client: :class:`~google.cloud.storage.client.Client` 

730 :param client: 

731 (Optional) The client to use. If not passed, falls back to the 

732 ``client`` stored on the blob's bucket. 

733 

734 :type if_generation_match: long 

735 :param if_generation_match: 

736 (Optional) See :ref:`using-if-generation-match` 

737 

738 :type if_generation_not_match: long 

739 :param if_generation_not_match: 

740 (Optional) See :ref:`using-if-generation-not-match` 

741 

742 :type if_metageneration_match: long 

743 :param if_metageneration_match: 

744 (Optional) See :ref:`using-if-metageneration-match` 

745 

746 :type if_metageneration_not_match: long 

747 :param if_metageneration_not_match: 

748 (Optional) See :ref:`using-if-metageneration-not-match` 

749 

750 :type timeout: float or tuple 

751 :param timeout: 

752 (Optional) The amount of time, in seconds, to wait 

753 for the server response. See: :ref:`configuring_timeouts` 

754 

755 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

756 :param retry: 

757 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

758 

759 :raises: :class:`google.cloud.exceptions.NotFound` 

760 (propagated from 

761 :meth:`google.cloud.storage.bucket.Bucket.delete_blob`). 

762 """ 

763 self.bucket.delete_blob( 

764 self.name, 

765 client=client, 

766 generation=self.generation, 

767 timeout=timeout, 

768 if_generation_match=if_generation_match, 

769 if_generation_not_match=if_generation_not_match, 

770 if_metageneration_match=if_metageneration_match, 

771 if_metageneration_not_match=if_metageneration_not_match, 

772 retry=retry, 

773 ) 

774 

775 def _get_transport(self, client): 

776 """Return the client's transport. 

777 

778 :type client: :class:`~google.cloud.storage.client.Client` 

779 :param client: 

780 (Optional) The client to use. If not passed, falls back to the 

781 ``client`` stored on the blob's bucket. 

782 

783 :rtype transport: 

784 :class:`~google.auth.transport.requests.AuthorizedSession` 

785 :returns: The transport (with credentials) that will 

786 make authenticated requests. 

787 """ 

788 client = self._require_client(client) 

789 return client._http 

790 

791 def _get_download_url( 

792 self, 

793 client, 

794 if_generation_match=None, 

795 if_generation_not_match=None, 

796 if_metageneration_match=None, 

797 if_metageneration_not_match=None, 

798 ): 

799 """Get the download URL for the current blob. 

800 

801 If the ``media_link`` has been loaded, it will be used, otherwise 

802 the URL will be constructed from the current blob's path (and possibly 

803 generation) to avoid a round trip. 

804 

805 :type client: :class:`~google.cloud.storage.client.Client` 

806 :param client: The client to use. 

807 

808 :type if_generation_match: long 

809 :param if_generation_match: 

810 (Optional) See :ref:`using-if-generation-match` 

811 

812 :type if_generation_not_match: long 

813 :param if_generation_not_match: 

814 (Optional) See :ref:`using-if-generation-not-match` 

815 

816 :type if_metageneration_match: long 

817 :param if_metageneration_match: 

818 (Optional) See :ref:`using-if-metageneration-match` 

819 

820 :type if_metageneration_not_match: long 

821 :param if_metageneration_not_match: 

822 (Optional) See :ref:`using-if-metageneration-not-match` 

823 

824 :rtype: str 

825 :returns: The download URL for the current blob. 

826 """ 

827 name_value_pairs = [] 

828 if self.media_link is None: 

829 hostname = _get_host_name(client._connection) 

830 base_url = _DOWNLOAD_URL_TEMPLATE.format( 

831 hostname=hostname, path=self.path, api_version=_API_VERSION 

832 ) 

833 if self.generation is not None: 

834 name_value_pairs.append(("generation", f"{self.generation:d}")) 

835 else: 

836 base_url = self.media_link 

837 

838 if self.user_project is not None: 

839 name_value_pairs.append(("userProject", self.user_project)) 

840 

841 _add_generation_match_parameters( 

842 name_value_pairs, 

843 if_generation_match=if_generation_match, 

844 if_generation_not_match=if_generation_not_match, 

845 if_metageneration_match=if_metageneration_match, 

846 if_metageneration_not_match=if_metageneration_not_match, 

847 ) 

848 return _add_query_parameters(base_url, name_value_pairs) 

849 

850 def _extract_headers_from_download(self, response): 

851 """Extract headers from a non-chunked request's http object. 

852 

853 This avoids the need to make a second request for commonly used 

854 headers. 

855 

856 :type response: 

857 :class requests.models.Response 

858 :param response: The server response from downloading a non-chunked file 

859 """ 

860 self._properties["contentEncoding"] = response.headers.get( 

861 "Content-Encoding", None 

862 ) 

863 self._properties[_CONTENT_TYPE_FIELD] = response.headers.get( 

864 "Content-Type", None 

865 ) 

866 self._properties["cacheControl"] = response.headers.get("Cache-Control", None) 

867 self._properties["storageClass"] = response.headers.get( 

868 "X-Goog-Storage-Class", None 

869 ) 

870 self._properties["contentLanguage"] = response.headers.get( 

871 "Content-Language", None 

872 ) 

873 self._properties["etag"] = response.headers.get("ETag", None) 

874 self._properties["generation"] = response.headers.get("X-goog-generation", None) 

875 self._properties["metageneration"] = response.headers.get( 

876 "X-goog-metageneration", None 

877 ) 

878 # 'X-Goog-Hash': 'crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==', 

879 x_goog_hash = response.headers.get("X-Goog-Hash", "") 

880 

881 if x_goog_hash: 

882 digests = {} 

883 for encoded_digest in x_goog_hash.split(","): 

884 match = re.match(r"(crc32c|md5)=([\w\d/\+/]+={0,3})", encoded_digest) 

885 if match: 

886 method, digest = match.groups() 

887 digests[method] = digest 

888 

889 self._properties["crc32c"] = digests.get("crc32c", None) 

890 self._properties["md5Hash"] = digests.get("md5", None) 

891 

892 def _do_download( 

893 self, 

894 transport, 

895 file_obj, 

896 download_url, 

897 headers, 

898 start=None, 

899 end=None, 

900 raw_download=False, 

901 timeout=_DEFAULT_TIMEOUT, 

902 checksum="md5", 

903 retry=None, 

904 ): 

905 """Perform a download without any error handling. 

906 

907 This is intended to be called by :meth:`download_to_file` so it can 

908 be wrapped with error handling / remapping. 

909 

910 :type transport: 

911 :class:`~google.auth.transport.requests.AuthorizedSession` 

912 :param transport: 

913 The transport (with credentials) that will make authenticated 

914 requests. 

915 

916 :type file_obj: file 

917 :param file_obj: A file handle to which to write the blob's data. 

918 

919 :type download_url: str 

920 :param download_url: The URL where the media can be accessed. 

921 

922 :type headers: dict 

923 :param headers: Headers to be sent with the request(s). 

924 

925 :type start: int 

926 :param start: (Optional) The first byte in a range to be downloaded. 

927 

928 :type end: int 

929 :param end: (Optional) The last byte in a range to be downloaded. 

930 

931 :type raw_download: bool 

932 :param raw_download: 

933 (Optional) If true, download the object without any expansion. 

934 

935 :type timeout: float or tuple 

936 :param timeout: 

937 (Optional) The amount of time, in seconds, to wait 

938 for the server response. See: :ref:`configuring_timeouts` 

939 

940 :type checksum: str 

941 :param checksum: 

942 (Optional) The type of checksum to compute to verify the integrity 

943 of the object. The response headers must contain a checksum of the 

944 requested type. If the headers lack an appropriate checksum (for 

945 instance in the case of transcoded or ranged downloads where the 

946 remote service does not know the correct checksum, including 

947 downloads where chunk_size is set) an INFO-level log will be 

948 emitted. Supported values are "md5", "crc32c" and None. The default 

949 is "md5". 

950 

951 :type retry: google.api_core.retry.Retry 

952 :param retry: (Optional) How to retry the RPC. A None value will disable 

953 retries. A google.api_core.retry.Retry value will enable retries, 

954 and the object will configure backoff and timeout options. Custom 

955 predicates (customizable error codes) are not supported for media 

956 operations such as this one. 

957 

958 This private method does not accept ConditionalRetryPolicy values 

959 because the information necessary to evaluate the policy is instead 

960 evaluated in client.download_blob_to_file(). 

961 

962 See the retry.py source code and docstrings in this package 

963 (google.cloud.storage.retry) for information on retry types and how 

964 to configure them. 

965 """ 

966 

967 retry_strategy = _api_core_retry_to_resumable_media_retry(retry) 

968 

969 if self.chunk_size is None: 

970 if raw_download: 

971 klass = RawDownload 

972 else: 

973 klass = Download 

974 

975 download = klass( 

976 download_url, 

977 stream=file_obj, 

978 headers=headers, 

979 start=start, 

980 end=end, 

981 checksum=checksum, 

982 ) 

983 download._retry_strategy = retry_strategy 

984 response = download.consume(transport, timeout=timeout) 

985 self._extract_headers_from_download(response) 

986 else: 

987 

988 if checksum: 

989 msg = _CHUNKED_DOWNLOAD_CHECKSUM_MESSAGE.format(checksum) 

990 _logger.info(msg) 

991 

992 if raw_download: 

993 klass = RawChunkedDownload 

994 else: 

995 klass = ChunkedDownload 

996 

997 download = klass( 

998 download_url, 

999 self.chunk_size, 

1000 file_obj, 

1001 headers=headers, 

1002 start=start if start else 0, 

1003 end=end, 

1004 ) 

1005 

1006 download._retry_strategy = retry_strategy 

1007 while not download.finished: 

1008 download.consume_next_chunk(transport, timeout=timeout) 

1009 

1010 def download_to_file( 

1011 self, 

1012 file_obj, 

1013 client=None, 

1014 start=None, 

1015 end=None, 

1016 raw_download=False, 

1017 if_etag_match=None, 

1018 if_etag_not_match=None, 

1019 if_generation_match=None, 

1020 if_generation_not_match=None, 

1021 if_metageneration_match=None, 

1022 if_metageneration_not_match=None, 

1023 timeout=_DEFAULT_TIMEOUT, 

1024 checksum="md5", 

1025 retry=DEFAULT_RETRY, 

1026 ): 

1027 """Download the contents of this blob into a file-like object. 

1028 

1029 .. note:: 

1030 

1031 If the server-set property, :attr:`media_link`, is not yet 

1032 initialized, makes an additional API request to load it. 

1033 

1034 If the :attr:`chunk_size` of a current blob is `None`, will download data 

1035 in single download request otherwise it will download the :attr:`chunk_size` 

1036 of data in each request. 

1037 

1038 For more fine-grained control over the download process, check out 

1039 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html). 

1040 For example, this library allows downloading **parts** of a blob rather than the whole thing. 

1041 

1042 If :attr:`user_project` is set on the bucket, bills the API request 

1043 to that project. 

1044 

1045 :type file_obj: file 

1046 :param file_obj: A file handle to which to write the blob's data. 

1047 

1048 :type client: :class:`~google.cloud.storage.client.Client` 

1049 :param client: 

1050 (Optional) The client to use. If not passed, falls back to the 

1051 ``client`` stored on the blob's bucket. 

1052 

1053 :type start: int 

1054 :param start: (Optional) The first byte in a range to be downloaded. 

1055 

1056 :type end: int 

1057 :param end: (Optional) The last byte in a range to be downloaded. 

1058 

1059 :type raw_download: bool 

1060 :param raw_download: 

1061 (Optional) If true, download the object without any expansion. 

1062 

1063 :type if_etag_match: Union[str, Set[str]] 

1064 :param if_etag_match: 

1065 (Optional) See :ref:`using-if-etag-match` 

1066 

1067 :type if_etag_not_match: Union[str, Set[str]] 

1068 :param if_etag_not_match: 

1069 (Optional) See :ref:`using-if-etag-not-match` 

1070 

1071 :type if_generation_match: long 

1072 :param if_generation_match: 

1073 (Optional) See :ref:`using-if-generation-match` 

1074 

1075 :type if_generation_not_match: long 

1076 :param if_generation_not_match: 

1077 (Optional) See :ref:`using-if-generation-not-match` 

1078 

1079 :type if_metageneration_match: long 

1080 :param if_metageneration_match: 

1081 (Optional) See :ref:`using-if-metageneration-match` 

1082 

1083 :type if_metageneration_not_match: long 

1084 :param if_metageneration_not_match: 

1085 (Optional) See :ref:`using-if-metageneration-not-match` 

1086 

1087 :type timeout: float or tuple 

1088 :param timeout: 

1089 (Optional) The amount of time, in seconds, to wait 

1090 for the server response. See: :ref:`configuring_timeouts` 

1091 

1092 :type checksum: str 

1093 :param checksum: 

1094 (Optional) The type of checksum to compute to verify the integrity 

1095 of the object. The response headers must contain a checksum of the 

1096 requested type. If the headers lack an appropriate checksum (for 

1097 instance in the case of transcoded or ranged downloads where the 

1098 remote service does not know the correct checksum, including 

1099 downloads where chunk_size is set) an INFO-level log will be 

1100 emitted. Supported values are "md5", "crc32c" and None. The default 

1101 is "md5". 

1102 

1103 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1104 :param retry: (Optional) How to retry the RPC. A None value will disable 

1105 retries. A google.api_core.retry.Retry value will enable retries, 

1106 and the object will define retriable response codes and errors and 

1107 configure backoff and timeout options. 

1108 

1109 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1110 Retry object and activates it only if certain conditions are met. 

1111 This class exists to provide safe defaults for RPC calls that are 

1112 not technically safe to retry normally (due to potential data 

1113 duplication or other side-effects) but become safe to retry if a 

1114 condition such as if_metageneration_match is set. 

1115 

1116 See the retry.py source code and docstrings in this package 

1117 (google.cloud.storage.retry) for information on retry types and how 

1118 to configure them. 

1119 

1120 Media operations (downloads and uploads) do not support non-default 

1121 predicates in a Retry object. The default will always be used. Other 

1122 configuration changes for Retry objects such as delays and deadlines 

1123 are respected. 

1124 

1125 :raises: :class:`google.cloud.exceptions.NotFound` 

1126 """ 

1127 client = self._require_client(client) 

1128 

1129 client.download_blob_to_file( 

1130 self, 

1131 file_obj=file_obj, 

1132 start=start, 

1133 end=end, 

1134 raw_download=raw_download, 

1135 if_etag_match=if_etag_match, 

1136 if_etag_not_match=if_etag_not_match, 

1137 if_generation_match=if_generation_match, 

1138 if_generation_not_match=if_generation_not_match, 

1139 if_metageneration_match=if_metageneration_match, 

1140 if_metageneration_not_match=if_metageneration_not_match, 

1141 timeout=timeout, 

1142 checksum=checksum, 

1143 retry=retry, 

1144 ) 

1145 

1146 def download_to_filename( 

1147 self, 

1148 filename, 

1149 client=None, 

1150 start=None, 

1151 end=None, 

1152 raw_download=False, 

1153 if_etag_match=None, 

1154 if_etag_not_match=None, 

1155 if_generation_match=None, 

1156 if_generation_not_match=None, 

1157 if_metageneration_match=None, 

1158 if_metageneration_not_match=None, 

1159 timeout=_DEFAULT_TIMEOUT, 

1160 checksum="md5", 

1161 retry=DEFAULT_RETRY, 

1162 ): 

1163 """Download the contents of this blob into a named file. 

1164 

1165 If :attr:`user_project` is set on the bucket, bills the API request 

1166 to that project. 

1167 

1168 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-download-encrypted-file#storage_download_encrypted_file-python) 

1169 to download a file with a [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied). 

1170 

1171 :type filename: str 

1172 :param filename: A filename to be passed to ``open``. 

1173 

1174 :type client: :class:`~google.cloud.storage.client.Client` 

1175 :param client: 

1176 (Optional) The client to use. If not passed, falls back to the 

1177 ``client`` stored on the blob's bucket. 

1178 

1179 :type start: int 

1180 :param start: (Optional) The first byte in a range to be downloaded. 

1181 

1182 :type end: int 

1183 :param end: (Optional) The last byte in a range to be downloaded. 

1184 

1185 :type raw_download: bool 

1186 :param raw_download: 

1187 (Optional) If true, download the object without any expansion. 

1188 

1189 :type if_etag_match: Union[str, Set[str]] 

1190 :param if_etag_match: 

1191 (Optional) See :ref:`using-if-etag-match` 

1192 

1193 :type if_etag_not_match: Union[str, Set[str]] 

1194 :param if_etag_not_match: 

1195 (Optional) See :ref:`using-if-etag-not-match` 

1196 

1197 :type if_generation_match: long 

1198 :param if_generation_match: 

1199 (Optional) See :ref:`using-if-generation-match` 

1200 

1201 :type if_generation_not_match: long 

1202 :param if_generation_not_match: 

1203 (Optional) See :ref:`using-if-generation-not-match` 

1204 

1205 :type if_metageneration_match: long 

1206 :param if_metageneration_match: 

1207 (Optional) See :ref:`using-if-metageneration-match` 

1208 

1209 :type if_metageneration_not_match: long 

1210 :param if_metageneration_not_match: 

1211 (Optional) See :ref:`using-if-metageneration-not-match` 

1212 

1213 :type timeout: float or tuple 

1214 :param timeout: 

1215 (Optional) The amount of time, in seconds, to wait 

1216 for the server response. See: :ref:`configuring_timeouts` 

1217 

1218 :type checksum: str 

1219 :param checksum: 

1220 (Optional) The type of checksum to compute to verify the integrity 

1221 of the object. The response headers must contain a checksum of the 

1222 requested type. If the headers lack an appropriate checksum (for 

1223 instance in the case of transcoded or ranged downloads where the 

1224 remote service does not know the correct checksum, including 

1225 downloads where chunk_size is set) an INFO-level log will be 

1226 emitted. Supported values are "md5", "crc32c" and None. The default 

1227 is "md5". 

1228 

1229 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1230 :param retry: (Optional) How to retry the RPC. A None value will disable 

1231 retries. A google.api_core.retry.Retry value will enable retries, 

1232 and the object will define retriable response codes and errors and 

1233 configure backoff and timeout options. 

1234 

1235 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1236 Retry object and activates it only if certain conditions are met. 

1237 This class exists to provide safe defaults for RPC calls that are 

1238 not technically safe to retry normally (due to potential data 

1239 duplication or other side-effects) but become safe to retry if a 

1240 condition such as if_metageneration_match is set. 

1241 

1242 See the retry.py source code and docstrings in this package 

1243 (google.cloud.storage.retry) for information on retry types and how 

1244 to configure them. 

1245 

1246 Media operations (downloads and uploads) do not support non-default 

1247 predicates in a Retry object. The default will always be used. Other 

1248 configuration changes for Retry objects such as delays and deadlines 

1249 are respected. 

1250 

1251 :raises: :class:`google.cloud.exceptions.NotFound` 

1252 """ 

1253 client = self._require_client(client) 

1254 try: 

1255 with open(filename, "wb") as file_obj: 

1256 client.download_blob_to_file( 

1257 self, 

1258 file_obj, 

1259 start=start, 

1260 end=end, 

1261 raw_download=raw_download, 

1262 if_etag_match=if_etag_match, 

1263 if_etag_not_match=if_etag_not_match, 

1264 if_generation_match=if_generation_match, 

1265 if_generation_not_match=if_generation_not_match, 

1266 if_metageneration_match=if_metageneration_match, 

1267 if_metageneration_not_match=if_metageneration_not_match, 

1268 timeout=timeout, 

1269 checksum=checksum, 

1270 retry=retry, 

1271 ) 

1272 except resumable_media.DataCorruption: 

1273 # Delete the corrupt downloaded file. 

1274 os.remove(filename) 

1275 raise 

1276 

1277 updated = self.updated 

1278 if updated is not None: 

1279 mtime = updated.timestamp() 

1280 os.utime(file_obj.name, (mtime, mtime)) 

1281 

1282 def download_as_bytes( 

1283 self, 

1284 client=None, 

1285 start=None, 

1286 end=None, 

1287 raw_download=False, 

1288 if_etag_match=None, 

1289 if_etag_not_match=None, 

1290 if_generation_match=None, 

1291 if_generation_not_match=None, 

1292 if_metageneration_match=None, 

1293 if_metageneration_not_match=None, 

1294 timeout=_DEFAULT_TIMEOUT, 

1295 checksum="md5", 

1296 retry=DEFAULT_RETRY, 

1297 ): 

1298 """Download the contents of this blob as a bytes object. 

1299 

1300 If :attr:`user_project` is set on the bucket, bills the API request 

1301 to that project. 

1302 

1303 :type client: :class:`~google.cloud.storage.client.Client` 

1304 :param client: 

1305 (Optional) The client to use. If not passed, falls back to the 

1306 ``client`` stored on the blob's bucket. 

1307 

1308 :type start: int 

1309 :param start: (Optional) The first byte in a range to be downloaded. 

1310 

1311 :type end: int 

1312 :param end: (Optional) The last byte in a range to be downloaded. 

1313 

1314 :type raw_download: bool 

1315 :param raw_download: 

1316 (Optional) If true, download the object without any expansion. 

1317 

1318 :type if_etag_match: Union[str, Set[str]] 

1319 :param if_etag_match: 

1320 (Optional) See :ref:`using-if-etag-match` 

1321 

1322 :type if_etag_not_match: Union[str, Set[str]] 

1323 :param if_etag_not_match: 

1324 (Optional) See :ref:`using-if-etag-not-match` 

1325 

1326 :type if_generation_match: long 

1327 :param if_generation_match: 

1328 (Optional) See :ref:`using-if-generation-match` 

1329 

1330 :type if_generation_not_match: long 

1331 :param if_generation_not_match: 

1332 (Optional) See :ref:`using-if-generation-not-match` 

1333 

1334 :type if_metageneration_match: long 

1335 :param if_metageneration_match: 

1336 (Optional) See :ref:`using-if-metageneration-match` 

1337 

1338 :type if_metageneration_not_match: long 

1339 :param if_metageneration_not_match: 

1340 (Optional) See :ref:`using-if-metageneration-not-match` 

1341 

1342 :type timeout: float or tuple 

1343 :param timeout: 

1344 (Optional) The amount of time, in seconds, to wait 

1345 for the server response. See: :ref:`configuring_timeouts` 

1346 

1347 :type checksum: str 

1348 :param checksum: 

1349 (Optional) The type of checksum to compute to verify the integrity 

1350 of the object. The response headers must contain a checksum of the 

1351 requested type. If the headers lack an appropriate checksum (for 

1352 instance in the case of transcoded or ranged downloads where the 

1353 remote service does not know the correct checksum, including 

1354 downloads where chunk_size is set) an INFO-level log will be 

1355 emitted. Supported values are "md5", "crc32c" and None. The default 

1356 is "md5". 

1357 

1358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1359 :param retry: (Optional) How to retry the RPC. A None value will disable 

1360 retries. A google.api_core.retry.Retry value will enable retries, 

1361 and the object will define retriable response codes and errors and 

1362 configure backoff and timeout options. 

1363 

1364 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1365 Retry object and activates it only if certain conditions are met. 

1366 This class exists to provide safe defaults for RPC calls that are 

1367 not technically safe to retry normally (due to potential data 

1368 duplication or other side-effects) but become safe to retry if a 

1369 condition such as if_metageneration_match is set. 

1370 

1371 See the retry.py source code and docstrings in this package 

1372 (google.cloud.storage.retry) for information on retry types and how 

1373 to configure them. 

1374 

1375 Media operations (downloads and uploads) do not support non-default 

1376 predicates in a Retry object. The default will always be used. Other 

1377 configuration changes for Retry objects such as delays and deadlines 

1378 are respected. 

1379 

1380 :rtype: bytes 

1381 :returns: The data stored in this blob. 

1382 

1383 :raises: :class:`google.cloud.exceptions.NotFound` 

1384 """ 

1385 client = self._require_client(client) 

1386 string_buffer = BytesIO() 

1387 client.download_blob_to_file( 

1388 self, 

1389 string_buffer, 

1390 start=start, 

1391 end=end, 

1392 raw_download=raw_download, 

1393 if_etag_match=if_etag_match, 

1394 if_etag_not_match=if_etag_not_match, 

1395 if_generation_match=if_generation_match, 

1396 if_generation_not_match=if_generation_not_match, 

1397 if_metageneration_match=if_metageneration_match, 

1398 if_metageneration_not_match=if_metageneration_not_match, 

1399 timeout=timeout, 

1400 checksum=checksum, 

1401 retry=retry, 

1402 ) 

1403 return string_buffer.getvalue() 

1404 

1405 def download_as_string( 

1406 self, 

1407 client=None, 

1408 start=None, 

1409 end=None, 

1410 raw_download=False, 

1411 if_etag_match=None, 

1412 if_etag_not_match=None, 

1413 if_generation_match=None, 

1414 if_generation_not_match=None, 

1415 if_metageneration_match=None, 

1416 if_metageneration_not_match=None, 

1417 timeout=_DEFAULT_TIMEOUT, 

1418 retry=DEFAULT_RETRY, 

1419 ): 

1420 """(Deprecated) Download the contents of this blob as a bytes object. 

1421 

1422 If :attr:`user_project` is set on the bucket, bills the API request 

1423 to that project. 

1424 

1425 .. note:: 

1426 Deprecated alias for :meth:`download_as_bytes`. 

1427 

1428 :type client: :class:`~google.cloud.storage.client.Client` 

1429 :param client: 

1430 (Optional) The client to use. If not passed, falls back to the 

1431 ``client`` stored on the blob's bucket. 

1432 

1433 :type start: int 

1434 :param start: (Optional) The first byte in a range to be downloaded. 

1435 

1436 :type end: int 

1437 :param end: (Optional) The last byte in a range to be downloaded. 

1438 

1439 :type raw_download: bool 

1440 :param raw_download: 

1441 (Optional) If true, download the object without any expansion. 

1442 

1443 :type if_etag_match: Union[str, Set[str]] 

1444 :param if_etag_match: 

1445 (Optional) See :ref:`using-if-etag-match` 

1446 

1447 :type if_etag_not_match: Union[str, Set[str]] 

1448 :param if_etag_not_match: 

1449 (Optional) See :ref:`using-if-etag-not-match` 

1450 

1451 :type if_generation_match: long 

1452 :param if_generation_match: 

1453 (Optional) See :ref:`using-if-generation-match` 

1454 

1455 :type if_generation_not_match: long 

1456 :param if_generation_not_match: 

1457 (Optional) See :ref:`using-if-generation-not-match` 

1458 

1459 :type if_metageneration_match: long 

1460 :param if_metageneration_match: 

1461 (Optional) See :ref:`using-if-metageneration-match` 

1462 

1463 :type if_metageneration_not_match: long 

1464 :param if_metageneration_not_match: 

1465 (Optional) See :ref:`using-if-metageneration-not-match` 

1466 

1467 :type timeout: float or tuple 

1468 :param timeout: 

1469 (Optional) The amount of time, in seconds, to wait 

1470 for the server response. See: :ref:`configuring_timeouts` 

1471 

1472 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1473 :param retry: (Optional) How to retry the RPC. A None value will disable 

1474 retries. A google.api_core.retry.Retry value will enable retries, 

1475 and the object will define retriable response codes and errors and 

1476 configure backoff and timeout options. 

1477 

1478 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1479 Retry object and activates it only if certain conditions are met. 

1480 This class exists to provide safe defaults for RPC calls that are 

1481 not technically safe to retry normally (due to potential data 

1482 duplication or other side-effects) but become safe to retry if a 

1483 condition such as if_metageneration_match is set. 

1484 

1485 See the retry.py source code and docstrings in this package 

1486 (google.cloud.storage.retry) for information on retry types and how 

1487 to configure them. 

1488 

1489 Media operations (downloads and uploads) do not support non-default 

1490 predicates in a Retry object. The default will always be used. Other 

1491 configuration changes for Retry objects such as delays and deadlines 

1492 are respected. 

1493 

1494 :rtype: bytes 

1495 :returns: The data stored in this blob. 

1496 

1497 :raises: :class:`google.cloud.exceptions.NotFound` 

1498 """ 

1499 warnings.warn( 

1500 _DOWNLOAD_AS_STRING_DEPRECATED, PendingDeprecationWarning, stacklevel=2 

1501 ) 

1502 return self.download_as_bytes( 

1503 client=client, 

1504 start=start, 

1505 end=end, 

1506 raw_download=raw_download, 

1507 if_etag_match=if_etag_match, 

1508 if_etag_not_match=if_etag_not_match, 

1509 if_generation_match=if_generation_match, 

1510 if_generation_not_match=if_generation_not_match, 

1511 if_metageneration_match=if_metageneration_match, 

1512 if_metageneration_not_match=if_metageneration_not_match, 

1513 timeout=timeout, 

1514 retry=retry, 

1515 ) 

1516 

1517 def download_as_text( 

1518 self, 

1519 client=None, 

1520 start=None, 

1521 end=None, 

1522 raw_download=False, 

1523 encoding=None, 

1524 if_etag_match=None, 

1525 if_etag_not_match=None, 

1526 if_generation_match=None, 

1527 if_generation_not_match=None, 

1528 if_metageneration_match=None, 

1529 if_metageneration_not_match=None, 

1530 timeout=_DEFAULT_TIMEOUT, 

1531 retry=DEFAULT_RETRY, 

1532 ): 

1533 """Download the contents of this blob as text (*not* bytes). 

1534 

1535 If :attr:`user_project` is set on the bucket, bills the API request 

1536 to that project. 

1537 

1538 :type client: :class:`~google.cloud.storage.client.Client` 

1539 :param client: 

1540 (Optional) The client to use. If not passed, falls back to the 

1541 ``client`` stored on the blob's bucket. 

1542 

1543 :type start: int 

1544 :param start: (Optional) The first byte in a range to be downloaded. 

1545 

1546 :type end: int 

1547 :param end: (Optional) The last byte in a range to be downloaded. 

1548 

1549 :type raw_download: bool 

1550 :param raw_download: 

1551 (Optional) If true, download the object without any expansion. 

1552 

1553 :type encoding: str 

1554 :param encoding: (Optional) encoding to be used to decode the 

1555 downloaded bytes. Defaults to the ``charset`` param of 

1556 attr:`content_type`, or else to "utf-8". 

1557 

1558 :type if_etag_match: Union[str, Set[str]] 

1559 :param if_etag_match: 

1560 (Optional) See :ref:`using-if-etag-match` 

1561 

1562 :type if_etag_not_match: Union[str, Set[str]] 

1563 :param if_etag_not_match: 

1564 (Optional) See :ref:`using-if-etag-not-match` 

1565 

1566 :type if_generation_match: long 

1567 :param if_generation_match: 

1568 (Optional) See :ref:`using-if-generation-match` 

1569 

1570 :type if_generation_not_match: long 

1571 :param if_generation_not_match: 

1572 (Optional) See :ref:`using-if-generation-not-match` 

1573 

1574 :type if_metageneration_match: long 

1575 :param if_metageneration_match: 

1576 (Optional) See :ref:`using-if-metageneration-match` 

1577 

1578 :type if_metageneration_not_match: long 

1579 :param if_metageneration_not_match: 

1580 (Optional) See :ref:`using-if-metageneration-not-match` 

1581 

1582 :type timeout: float or tuple 

1583 :param timeout: 

1584 (Optional) The amount of time, in seconds, to wait 

1585 for the server response. See: :ref:`configuring_timeouts` 

1586 

1587 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1588 :param retry: (Optional) How to retry the RPC. A None value will disable 

1589 retries. A google.api_core.retry.Retry value will enable retries, 

1590 and the object will define retriable response codes and errors and 

1591 configure backoff and timeout options. 

1592 

1593 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1594 Retry object and activates it only if certain conditions are met. 

1595 This class exists to provide safe defaults for RPC calls that are 

1596 not technically safe to retry normally (due to potential data 

1597 duplication or other side-effects) but become safe to retry if a 

1598 condition such as if_metageneration_match is set. 

1599 

1600 See the retry.py source code and docstrings in this package 

1601 (google.cloud.storage.retry) for information on retry types and how 

1602 to configure them. 

1603 

1604 Media operations (downloads and uploads) do not support non-default 

1605 predicates in a Retry object. The default will always be used. Other 

1606 configuration changes for Retry objects such as delays and deadlines 

1607 are respected. 

1608 

1609 :rtype: text 

1610 :returns: The data stored in this blob, decoded to text. 

1611 """ 

1612 data = self.download_as_bytes( 

1613 client=client, 

1614 start=start, 

1615 end=end, 

1616 raw_download=raw_download, 

1617 if_etag_match=if_etag_match, 

1618 if_etag_not_match=if_etag_not_match, 

1619 if_generation_match=if_generation_match, 

1620 if_generation_not_match=if_generation_not_match, 

1621 if_metageneration_match=if_metageneration_match, 

1622 if_metageneration_not_match=if_metageneration_not_match, 

1623 timeout=timeout, 

1624 retry=retry, 

1625 ) 

1626 

1627 if encoding is not None: 

1628 return data.decode(encoding) 

1629 

1630 if self.content_type is not None: 

1631 msg = HeaderParser().parsestr("Content-Type: " + self.content_type) 

1632 params = dict(msg.get_params()[1:]) 

1633 if "charset" in params: 

1634 return data.decode(params["charset"]) 

1635 

1636 return data.decode("utf-8") 

1637 

1638 def _get_content_type(self, content_type, filename=None): 

1639 """Determine the content type from the current object. 

1640 

1641 The return value will be determined in order of precedence: 

1642 

1643 - The value passed in to this method (if not :data:`None`) 

1644 - The value stored on the current blob 

1645 - The default value ('application/octet-stream') 

1646 

1647 :type content_type: str 

1648 :param content_type: (Optional) Type of content. 

1649 

1650 :type filename: str 

1651 :param filename: 

1652 (Optional) The name of the file where the content is stored. 

1653 

1654 :rtype: str 

1655 :returns: Type of content gathered from the object. 

1656 """ 

1657 if content_type is None: 

1658 content_type = self.content_type 

1659 

1660 if content_type is None and filename is not None: 

1661 content_type, _ = mimetypes.guess_type(filename) 

1662 

1663 if content_type is None: 

1664 content_type = _DEFAULT_CONTENT_TYPE 

1665 

1666 return content_type 

1667 

1668 def _get_writable_metadata(self): 

1669 """Get the object / blob metadata which is writable. 

1670 

1671 This is intended to be used when creating a new object / blob. 

1672 

1673 See the [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects) 

1674 for more information, the fields marked as writable are: 

1675 

1676 * ``acl`` 

1677 * ``cacheControl`` 

1678 * ``contentDisposition`` 

1679 * ``contentEncoding`` 

1680 * ``contentLanguage`` 

1681 * ``contentType`` 

1682 * ``crc32c`` 

1683 * ``customTime`` 

1684 * ``md5Hash`` 

1685 * ``metadata`` 

1686 * ``name`` 

1687 * ``storageClass`` 

1688 

1689 For now, we don't support ``acl``, access control lists should be 

1690 managed directly through :class:`ObjectACL` methods. 

1691 """ 

1692 # NOTE: This assumes `self.name` is unicode. 

1693 object_metadata = {"name": self.name} 

1694 for key in self._changes: 

1695 if key in _WRITABLE_FIELDS: 

1696 object_metadata[key] = self._properties[key] 

1697 

1698 return object_metadata 

1699 

1700 def _get_upload_arguments(self, client, content_type): 

1701 """Get required arguments for performing an upload. 

1702 

1703 The content type returned will be determined in order of precedence: 

1704 

1705 - The value passed in to this method (if not :data:`None`) 

1706 - The value stored on the current blob 

1707 - The default value ('application/octet-stream') 

1708 

1709 :type content_type: str 

1710 :param content_type: Type of content being uploaded (or :data:`None`). 

1711 

1712 :rtype: tuple 

1713 :returns: A triple of 

1714 

1715 * A header dictionary 

1716 * An object metadata dictionary 

1717 * The ``content_type`` as a string (according to precedence) 

1718 """ 

1719 content_type = self._get_content_type(content_type) 

1720 headers = { 

1721 **_get_default_headers(client._connection.user_agent, content_type), 

1722 **_get_encryption_headers(self._encryption_key), 

1723 } 

1724 object_metadata = self._get_writable_metadata() 

1725 return headers, object_metadata, content_type 

1726 

1727 def _do_multipart_upload( 

1728 self, 

1729 client, 

1730 stream, 

1731 content_type, 

1732 size, 

1733 num_retries, 

1734 predefined_acl, 

1735 if_generation_match, 

1736 if_generation_not_match, 

1737 if_metageneration_match, 

1738 if_metageneration_not_match, 

1739 timeout=_DEFAULT_TIMEOUT, 

1740 checksum=None, 

1741 retry=None, 

1742 ): 

1743 """Perform a multipart upload. 

1744 

1745 The content type of the upload will be determined in order 

1746 of precedence: 

1747 

1748 - The value passed in to this method (if not :data:`None`) 

1749 - The value stored on the current blob 

1750 - The default value ('application/octet-stream') 

1751 

1752 :type client: :class:`~google.cloud.storage.client.Client` 

1753 :param client: 

1754 (Optional) The client to use. If not passed, falls back to the 

1755 ``client`` stored on the blob's bucket. 

1756 

1757 :type stream: IO[bytes] 

1758 :param stream: A bytes IO object open for reading. 

1759 

1760 :type content_type: str 

1761 :param content_type: Type of content being uploaded (or :data:`None`). 

1762 

1763 :type size: int 

1764 :param size: 

1765 The number of bytes to be uploaded (which will be read from 

1766 ``stream``). If not provided, the upload will be concluded once 

1767 ``stream`` is exhausted (or :data:`None`). 

1768 

1769 :type num_retries: int 

1770 :param num_retries: 

1771 Number of upload retries. By default, only uploads with 

1772 if_generation_match set will be retried, as uploads without the 

1773 argument are not guaranteed to be idempotent. Setting num_retries 

1774 will override this default behavior and guarantee retries even when 

1775 if_generation_match is not set. (Deprecated: This argument 

1776 will be removed in a future release.) 

1777 

1778 :type predefined_acl: str 

1779 :param predefined_acl: (Optional) Predefined access control list 

1780 

1781 :type if_generation_match: long 

1782 :param if_generation_match: 

1783 (Optional) See :ref:`using-if-generation-match` 

1784 

1785 :type if_generation_not_match: long 

1786 :param if_generation_not_match: 

1787 (Optional) See :ref:`using-if-generation-not-match` 

1788 

1789 :type if_metageneration_match: long 

1790 :param if_metageneration_match: 

1791 (Optional) See :ref:`using-if-metageneration-match` 

1792 

1793 :type if_metageneration_not_match: long 

1794 :param if_metageneration_not_match: 

1795 (Optional) See :ref:`using-if-metageneration-not-match` 

1796 

1797 :type timeout: float or tuple 

1798 :param timeout: 

1799 (Optional) The amount of time, in seconds, to wait 

1800 for the server response. See: :ref:`configuring_timeouts` 

1801 

1802 :type checksum: str 

1803 :param checksum: 

1804 (Optional) The type of checksum to compute to verify 

1805 the integrity of the object. The request metadata will be amended 

1806 to include the computed value. Using this option will override a 

1807 manually-set checksum value. Supported values are "md5", 

1808 "crc32c" and None. The default is None. 

1809 

1810 :type retry: google.api_core.retry.Retry 

1811 :param retry: (Optional) How to retry the RPC. A None value will disable 

1812 retries. A google.api_core.retry.Retry value will enable retries, 

1813 and the object will configure backoff and timeout options. Custom 

1814 predicates (customizable error codes) are not supported for media 

1815 operations such as this one. 

1816 

1817 This private method does not accept ConditionalRetryPolicy values 

1818 because the information necessary to evaluate the policy is instead 

1819 evaluated in blob._do_upload(). 

1820 

1821 See the retry.py source code and docstrings in this package 

1822 (google.cloud.storage.retry) for information on retry types and how 

1823 to configure them. 

1824 

1825 :rtype: :class:`~requests.Response` 

1826 :returns: The "200 OK" response object returned after the multipart 

1827 upload request. 

1828 :raises: :exc:`ValueError` if ``size`` is not :data:`None` but the 

1829 ``stream`` has fewer than ``size`` bytes remaining. 

1830 """ 

1831 if size is None: 

1832 data = stream.read() 

1833 else: 

1834 data = stream.read(size) 

1835 if len(data) < size: 

1836 msg = _READ_LESS_THAN_SIZE.format(size, len(data)) 

1837 raise ValueError(msg) 

1838 

1839 client = self._require_client(client) 

1840 transport = self._get_transport(client) 

1841 if "metadata" in self._properties and "metadata" not in self._changes: 

1842 self._changes.add("metadata") 

1843 info = self._get_upload_arguments(client, content_type) 

1844 headers, object_metadata, content_type = info 

1845 

1846 hostname = _get_host_name(client._connection) 

1847 base_url = _MULTIPART_URL_TEMPLATE.format( 

1848 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION 

1849 ) 

1850 name_value_pairs = [] 

1851 

1852 if self.user_project is not None: 

1853 name_value_pairs.append(("userProject", self.user_project)) 

1854 

1855 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object 

1856 # at rest, object resource metadata will store the version of the Key Management 

1857 # Service cryptographic material. If a Blob instance with KMS Key metadata set is 

1858 # used to upload a new version of the object then the existing kmsKeyName version 

1859 # value can't be used in the upload request and the client instead ignores it. 

1860 if ( 

1861 self.kms_key_name is not None 

1862 and "cryptoKeyVersions" not in self.kms_key_name 

1863 ): 

1864 name_value_pairs.append(("kmsKeyName", self.kms_key_name)) 

1865 

1866 if predefined_acl is not None: 

1867 name_value_pairs.append(("predefinedAcl", predefined_acl)) 

1868 

1869 if if_generation_match is not None: 

1870 name_value_pairs.append(("ifGenerationMatch", if_generation_match)) 

1871 

1872 if if_generation_not_match is not None: 

1873 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match)) 

1874 

1875 if if_metageneration_match is not None: 

1876 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match)) 

1877 

1878 if if_metageneration_not_match is not None: 

1879 name_value_pairs.append( 

1880 ("ifMetaGenerationNotMatch", if_metageneration_not_match) 

1881 ) 

1882 

1883 upload_url = _add_query_parameters(base_url, name_value_pairs) 

1884 upload = MultipartUpload(upload_url, headers=headers, checksum=checksum) 

1885 

1886 upload._retry_strategy = _api_core_retry_to_resumable_media_retry( 

1887 retry, num_retries 

1888 ) 

1889 

1890 response = upload.transmit( 

1891 transport, data, object_metadata, content_type, timeout=timeout 

1892 ) 

1893 

1894 return response 

1895 

1896 def _initiate_resumable_upload( 

1897 self, 

1898 client, 

1899 stream, 

1900 content_type, 

1901 size, 

1902 num_retries, 

1903 predefined_acl=None, 

1904 extra_headers=None, 

1905 chunk_size=None, 

1906 if_generation_match=None, 

1907 if_generation_not_match=None, 

1908 if_metageneration_match=None, 

1909 if_metageneration_not_match=None, 

1910 timeout=_DEFAULT_TIMEOUT, 

1911 checksum=None, 

1912 retry=None, 

1913 ): 

1914 """Initiate a resumable upload. 

1915 

1916 The content type of the upload will be determined in order 

1917 of precedence: 

1918 

1919 - The value passed in to this method (if not :data:`None`) 

1920 - The value stored on the current blob 

1921 - The default value ('application/octet-stream') 

1922 

1923 :type client: :class:`~google.cloud.storage.client.Client` 

1924 :param client: 

1925 (Optional) The client to use. If not passed, falls back to the 

1926 ``client`` stored on the blob's bucket. 

1927 

1928 :type stream: IO[bytes] 

1929 :param stream: A bytes IO object open for reading. 

1930 

1931 :type content_type: str 

1932 :param content_type: Type of content being uploaded (or :data:`None`). 

1933 

1934 :type size: int 

1935 :param size: 

1936 The number of bytes to be uploaded (which will be read from 

1937 ``stream``). If not provided, the upload will be concluded once 

1938 ``stream`` is exhausted (or :data:`None`). 

1939 

1940 :type predefined_acl: str 

1941 :param predefined_acl: (Optional) Predefined access control list 

1942 

1943 :type num_retries: int 

1944 :param num_retries: 

1945 Number of upload retries. By default, only uploads with 

1946 if_generation_match set will be retried, as uploads without the 

1947 argument are not guaranteed to be idempotent. Setting num_retries 

1948 will override this default behavior and guarantee retries even when 

1949 if_generation_match is not set. (Deprecated: This argument 

1950 will be removed in a future release.) 

1951 

1952 :type extra_headers: dict 

1953 :param extra_headers: 

1954 (Optional) Extra headers to add to standard headers. 

1955 

1956 :type chunk_size: int 

1957 :param chunk_size: 

1958 (Optional) Chunk size to use when creating a 

1959 :class:`~google.resumable_media.requests.ResumableUpload`. 

1960 If not passed, will fall back to the chunk size on the 

1961 current blob, if the chunk size of a current blob is also 

1962 `None`, will set the default value. 

1963 The default value of ``chunk_size`` is 100 MB. 

1964 

1965 :type if_generation_match: long 

1966 :param if_generation_match: 

1967 (Optional) See :ref:`using-if-generation-match` 

1968 

1969 :type if_generation_not_match: long 

1970 :param if_generation_not_match: 

1971 (Optional) See :ref:`using-if-generation-not-match` 

1972 

1973 :type if_metageneration_match: long 

1974 :param if_metageneration_match: 

1975 (Optional) See :ref:`using-if-metageneration-match` 

1976 

1977 :type if_metageneration_not_match: long 

1978 :param if_metageneration_not_match: 

1979 (Optional) See :ref:`using-if-metageneration-not-match` 

1980 

1981 :type timeout: float or tuple 

1982 :param timeout: 

1983 (Optional) The amount of time, in seconds, to wait 

1984 for the server response. See: :ref:`configuring_timeouts` 

1985 

1986 :type checksum: str 

1987 :param checksum: 

1988 (Optional) The type of checksum to compute to verify 

1989 the integrity of the object. After the upload is complete, the 

1990 server-computed checksum of the resulting object will be checked 

1991 and google.resumable_media.common.DataCorruption will be raised on 

1992 a mismatch. On a validation failure, the client will attempt to 

1993 delete the uploaded object automatically. Supported values 

1994 are "md5", "crc32c" and None. The default is None. 

1995 

1996 :type retry: google.api_core.retry.Retry 

1997 :param retry: (Optional) How to retry the RPC. A None value will disable 

1998 retries. A google.api_core.retry.Retry value will enable retries, 

1999 and the object will configure backoff and timeout options. Custom 

2000 predicates (customizable error codes) are not supported for media 

2001 operations such as this one. 

2002 

2003 This private method does not accept ConditionalRetryPolicy values 

2004 because the information necessary to evaluate the policy is instead 

2005 evaluated in blob._do_upload(). 

2006 

2007 See the retry.py source code and docstrings in this package 

2008 (google.cloud.storage.retry) for information on retry types and how 

2009 to configure them. 

2010 

2011 :rtype: tuple 

2012 :returns: 

2013 Pair of 

2014 

2015 * The :class:`~google.resumable_media.requests.ResumableUpload` 

2016 that was created 

2017 * The ``transport`` used to initiate the upload. 

2018 """ 

2019 client = self._require_client(client) 

2020 if chunk_size is None: 

2021 chunk_size = self.chunk_size 

2022 if chunk_size is None: 

2023 chunk_size = _DEFAULT_CHUNKSIZE 

2024 

2025 transport = self._get_transport(client) 

2026 if "metadata" in self._properties and "metadata" not in self._changes: 

2027 self._changes.add("metadata") 

2028 info = self._get_upload_arguments(client, content_type) 

2029 headers, object_metadata, content_type = info 

2030 if extra_headers is not None: 

2031 headers.update(extra_headers) 

2032 

2033 hostname = _get_host_name(client._connection) 

2034 base_url = _RESUMABLE_URL_TEMPLATE.format( 

2035 hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION 

2036 ) 

2037 name_value_pairs = [] 

2038 

2039 if self.user_project is not None: 

2040 name_value_pairs.append(("userProject", self.user_project)) 

2041 

2042 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object 

2043 # at rest, object resource metadata will store the version of the Key Management 

2044 # Service cryptographic material. If a Blob instance with KMS Key metadata set is 

2045 # used to upload a new version of the object then the existing kmsKeyName version 

2046 # value can't be used in the upload request and the client instead ignores it. 

2047 if ( 

2048 self.kms_key_name is not None 

2049 and "cryptoKeyVersions" not in self.kms_key_name 

2050 ): 

2051 name_value_pairs.append(("kmsKeyName", self.kms_key_name)) 

2052 

2053 if predefined_acl is not None: 

2054 name_value_pairs.append(("predefinedAcl", predefined_acl)) 

2055 

2056 if if_generation_match is not None: 

2057 name_value_pairs.append(("ifGenerationMatch", if_generation_match)) 

2058 

2059 if if_generation_not_match is not None: 

2060 name_value_pairs.append(("ifGenerationNotMatch", if_generation_not_match)) 

2061 

2062 if if_metageneration_match is not None: 

2063 name_value_pairs.append(("ifMetagenerationMatch", if_metageneration_match)) 

2064 

2065 if if_metageneration_not_match is not None: 

2066 name_value_pairs.append( 

2067 ("ifMetaGenerationNotMatch", if_metageneration_not_match) 

2068 ) 

2069 

2070 upload_url = _add_query_parameters(base_url, name_value_pairs) 

2071 upload = ResumableUpload( 

2072 upload_url, chunk_size, headers=headers, checksum=checksum 

2073 ) 

2074 

2075 upload._retry_strategy = _api_core_retry_to_resumable_media_retry( 

2076 retry, num_retries 

2077 ) 

2078 

2079 upload.initiate( 

2080 transport, 

2081 stream, 

2082 object_metadata, 

2083 content_type, 

2084 total_bytes=size, 

2085 stream_final=False, 

2086 timeout=timeout, 

2087 ) 

2088 

2089 return upload, transport 

2090 

2091 def _do_resumable_upload( 

2092 self, 

2093 client, 

2094 stream, 

2095 content_type, 

2096 size, 

2097 num_retries, 

2098 predefined_acl, 

2099 if_generation_match, 

2100 if_generation_not_match, 

2101 if_metageneration_match, 

2102 if_metageneration_not_match, 

2103 timeout=_DEFAULT_TIMEOUT, 

2104 checksum=None, 

2105 retry=None, 

2106 ): 

2107 """Perform a resumable upload. 

2108 

2109 Assumes ``chunk_size`` is not :data:`None` on the current blob. 

2110 The default value of ``chunk_size`` is 100 MB. 

2111 

2112 The content type of the upload will be determined in order 

2113 of precedence: 

2114 

2115 - The value passed in to this method (if not :data:`None`) 

2116 - The value stored on the current blob 

2117 - The default value ('application/octet-stream') 

2118 

2119 :type client: :class:`~google.cloud.storage.client.Client` 

2120 :param client: 

2121 (Optional) The client to use. If not passed, falls back to the 

2122 ``client`` stored on the blob's bucket. 

2123 

2124 :type stream: IO[bytes] 

2125 :param stream: A bytes IO object open for reading. 

2126 

2127 :type content_type: str 

2128 :param content_type: Type of content being uploaded (or :data:`None`). 

2129 

2130 :type size: int 

2131 :param size: 

2132 The number of bytes to be uploaded (which will be read from 

2133 ``stream``). If not provided, the upload will be concluded once 

2134 ``stream`` is exhausted (or :data:`None`). 

2135 

2136 :type num_retries: int 

2137 :param num_retries: 

2138 Number of upload retries. By default, only uploads with 

2139 if_generation_match set will be retried, as uploads without the 

2140 argument are not guaranteed to be idempotent. Setting num_retries 

2141 will override this default behavior and guarantee retries even when 

2142 if_generation_match is not set. (Deprecated: This argument 

2143 will be removed in a future release.) 

2144 

2145 :type predefined_acl: str 

2146 :param predefined_acl: (Optional) Predefined access control list 

2147 

2148 :type if_generation_match: long 

2149 :param if_generation_match: 

2150 (Optional) See :ref:`using-if-generation-match` 

2151 

2152 :type if_generation_not_match: long 

2153 :param if_generation_not_match: 

2154 (Optional) See :ref:`using-if-generation-not-match` 

2155 

2156 :type if_metageneration_match: long 

2157 :param if_metageneration_match: 

2158 (Optional) See :ref:`using-if-metageneration-match` 

2159 

2160 :type if_metageneration_not_match: long 

2161 :param if_metageneration_not_match: 

2162 (Optional) See :ref:`using-if-metageneration-not-match` 

2163 

2164 :type timeout: float or tuple 

2165 :param timeout: 

2166 (Optional) The amount of time, in seconds, to wait 

2167 for the server response. See: :ref:`configuring_timeouts` 

2168 

2169 :type checksum: str 

2170 :param checksum: 

2171 (Optional) The type of checksum to compute to verify 

2172 the integrity of the object. After the upload is complete, the 

2173 server-computed checksum of the resulting object will be checked 

2174 and google.resumable_media.common.DataCorruption will be raised on 

2175 a mismatch. On a validation failure, the client will attempt to 

2176 delete the uploaded object automatically. Supported values 

2177 are "md5", "crc32c" and None. The default is None. 

2178 

2179 :type retry: google.api_core.retry.Retry 

2180 :param retry: (Optional) How to retry the RPC. A None value will disable 

2181 retries. A google.api_core.retry.Retry value will enable retries, 

2182 and the object will configure backoff and timeout options. Custom 

2183 predicates (customizable error codes) are not supported for media 

2184 operations such as this one. 

2185 

2186 This private method does not accept ConditionalRetryPolicy values 

2187 because the information necessary to evaluate the policy is instead 

2188 evaluated in blob._do_upload(). 

2189 

2190 See the retry.py source code and docstrings in this package 

2191 (google.cloud.storage.retry) for information on retry types and how 

2192 to configure them. 

2193 

2194 :rtype: :class:`~requests.Response` 

2195 :returns: The "200 OK" response object returned after the final chunk 

2196 is uploaded. 

2197 """ 

2198 upload, transport = self._initiate_resumable_upload( 

2199 client, 

2200 stream, 

2201 content_type, 

2202 size, 

2203 num_retries, 

2204 predefined_acl=predefined_acl, 

2205 if_generation_match=if_generation_match, 

2206 if_generation_not_match=if_generation_not_match, 

2207 if_metageneration_match=if_metageneration_match, 

2208 if_metageneration_not_match=if_metageneration_not_match, 

2209 timeout=timeout, 

2210 checksum=checksum, 

2211 retry=retry, 

2212 ) 

2213 while not upload.finished: 

2214 try: 

2215 response = upload.transmit_next_chunk(transport, timeout=timeout) 

2216 except resumable_media.DataCorruption: 

2217 # Attempt to delete the corrupted object. 

2218 self.delete() 

2219 raise 

2220 return response 

2221 

2222 def _do_upload( 

2223 self, 

2224 client, 

2225 stream, 

2226 content_type, 

2227 size, 

2228 num_retries, 

2229 predefined_acl, 

2230 if_generation_match, 

2231 if_generation_not_match, 

2232 if_metageneration_match, 

2233 if_metageneration_not_match, 

2234 timeout=_DEFAULT_TIMEOUT, 

2235 checksum=None, 

2236 retry=None, 

2237 ): 

2238 """Determine an upload strategy and then perform the upload. 

2239 

2240 If the size of the data to be uploaded exceeds 8 MB a resumable media 

2241 request will be used, otherwise the content and the metadata will be 

2242 uploaded in a single multipart upload request. 

2243 

2244 The content type of the upload will be determined in order 

2245 of precedence: 

2246 

2247 - The value passed in to this method (if not :data:`None`) 

2248 - The value stored on the current blob 

2249 - The default value ('application/octet-stream') 

2250 

2251 :type client: :class:`~google.cloud.storage.client.Client` 

2252 :param client: 

2253 (Optional) The client to use. If not passed, falls back to the 

2254 ``client`` stored on the blob's bucket. 

2255 

2256 :type stream: IO[bytes] 

2257 :param stream: A bytes IO object open for reading. 

2258 

2259 :type content_type: str 

2260 :param content_type: Type of content being uploaded (or :data:`None`). 

2261 

2262 :type size: int 

2263 :param size: 

2264 The number of bytes to be uploaded (which will be read from 

2265 ``stream``). If not provided, the upload will be concluded once 

2266 ``stream`` is exhausted (or :data:`None`). 

2267 

2268 :type num_retries: int 

2269 :param num_retries: 

2270 Number of upload retries. By default, only uploads with 

2271 if_generation_match set will be retried, as uploads without the 

2272 argument are not guaranteed to be idempotent. Setting num_retries 

2273 will override this default behavior and guarantee retries even when 

2274 if_generation_match is not set. (Deprecated: This argument 

2275 will be removed in a future release.) 

2276 

2277 :type predefined_acl: str 

2278 :param predefined_acl: (Optional) Predefined access control list 

2279 

2280 :type if_generation_match: long 

2281 :param if_generation_match: 

2282 (Optional) See :ref:`using-if-generation-match` 

2283 

2284 :type if_generation_not_match: long 

2285 :param if_generation_not_match: 

2286 (Optional) See :ref:`using-if-generation-not-match` 

2287 

2288 :type if_metageneration_match: long 

2289 :param if_metageneration_match: 

2290 (Optional) See :ref:`using-if-metageneration-match` 

2291 

2292 :type if_metageneration_not_match: long 

2293 :param if_metageneration_not_match: 

2294 (Optional) See :ref:`using-if-metageneration-not-match` 

2295 

2296 :type timeout: float or tuple 

2297 :param timeout: 

2298 (Optional) The amount of time, in seconds, to wait 

2299 for the server response. See: :ref:`configuring_timeouts` 

2300 

2301 :type checksum: str 

2302 :param checksum: 

2303 (Optional) The type of checksum to compute to verify 

2304 the integrity of the object. If the upload is completed in a single 

2305 request, the checksum will be entirely precomputed and the remote 

2306 server will handle verification and error handling. If the upload 

2307 is too large and must be transmitted in multiple requests, the 

2308 checksum will be incrementally computed and the client will handle 

2309 verification and error handling, raising 

2310 google.resumable_media.common.DataCorruption on a mismatch and 

2311 attempting to delete the corrupted file. Supported values are 

2312 "md5", "crc32c" and None. The default is None. 

2313 

2314 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

2315 :param retry: (Optional) How to retry the RPC. A None value will disable 

2316 retries. A google.api_core.retry.Retry value will enable retries, 

2317 and the object will define retriable response codes and errors and 

2318 configure backoff and timeout options. 

2319 

2320 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

2321 Retry object and activates it only if certain conditions are met. 

2322 This class exists to provide safe defaults for RPC calls that are 

2323 not technically safe to retry normally (due to potential data 

2324 duplication or other side-effects) but become safe to retry if a 

2325 condition such as if_generation_match is set. 

2326 

2327 See the retry.py source code and docstrings in this package 

2328 (google.cloud.storage.retry) for information on retry types and how 

2329 to configure them. 

2330 

2331 Media operations (downloads and uploads) do not support non-default 

2332 predicates in a Retry object. The default will always be used. Other 

2333 configuration changes for Retry objects such as delays and deadlines 

2334 are respected. 

2335 

2336 :rtype: dict 

2337 :returns: The parsed JSON from the "200 OK" response. This will be the 

2338 **only** response in the multipart case and it will be the 

2339 **final** response in the resumable case. 

2340 """ 

2341 

2342 # Handle ConditionalRetryPolicy. 

2343 if isinstance(retry, ConditionalRetryPolicy): 

2344 # Conditional retries are designed for non-media calls, which change 

2345 # arguments into query_params dictionaries. Media operations work 

2346 # differently, so here we make a "fake" query_params to feed to the 

2347 # ConditionalRetryPolicy. 

2348 query_params = { 

2349 "ifGenerationMatch": if_generation_match, 

2350 "ifMetagenerationMatch": if_metageneration_match, 

2351 } 

2352 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) 

2353 

2354 if size is not None and size <= _MAX_MULTIPART_SIZE: 

2355 response = self._do_multipart_upload( 

2356 client, 

2357 stream, 

2358 content_type, 

2359 size, 

2360 num_retries, 

2361 predefined_acl, 

2362 if_generation_match, 

2363 if_generation_not_match, 

2364 if_metageneration_match, 

2365 if_metageneration_not_match, 

2366 timeout=timeout, 

2367 checksum=checksum, 

2368 retry=retry, 

2369 ) 

2370 else: 

2371 response = self._do_resumable_upload( 

2372 client, 

2373 stream, 

2374 content_type, 

2375 size, 

2376 num_retries, 

2377 predefined_acl, 

2378 if_generation_match, 

2379 if_generation_not_match, 

2380 if_metageneration_match, 

2381 if_metageneration_not_match, 

2382 timeout=timeout, 

2383 checksum=checksum, 

2384 retry=retry, 

2385 ) 

2386 

2387 return response.json() 

2388 

2389 def upload_from_file( 

2390 self, 

2391 file_obj, 

2392 rewind=False, 

2393 size=None, 

2394 content_type=None, 

2395 num_retries=None, 

2396 client=None, 

2397 predefined_acl=None, 

2398 if_generation_match=None, 

2399 if_generation_not_match=None, 

2400 if_metageneration_match=None, 

2401 if_metageneration_not_match=None, 

2402 timeout=_DEFAULT_TIMEOUT, 

2403 checksum=None, 

2404 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

2405 ): 

2406 """Upload the contents of this blob from a file-like object. 

2407 

2408 The content type of the upload will be determined in order 

2409 of precedence: 

2410 

2411 - The value passed in to this method (if not :data:`None`) 

2412 - The value stored on the current blob 

2413 - The default value ('application/octet-stream') 

2414 

2415 .. note:: 

2416 The effect of uploading to an existing blob depends on the 

2417 "versioning" and "lifecycle" policies defined on the blob's 

2418 bucket. In the absence of those policies, upload will 

2419 overwrite any existing contents. 

2420 

2421 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning) 

2422 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle) 

2423 API documents for details. 

2424 

2425 If the size of the data to be uploaded exceeds 8 MB a resumable media 

2426 request will be used, otherwise the content and the metadata will be 

2427 uploaded in a single multipart upload request. 

2428 

2429 For more fine-grained over the upload process, check out 

2430 [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html). 

2431 

2432 If :attr:`user_project` is set on the bucket, bills the API request 

2433 to that project. 

2434 

2435 :type file_obj: file 

2436 :param file_obj: A file handle opened in binary mode for reading. 

2437 

2438 :type rewind: bool 

2439 :param rewind: 

2440 If True, seek to the beginning of the file handle before writing 

2441 the file to Cloud Storage. 

2442 

2443 :type size: int 

2444 :param size: 

2445 The number of bytes to be uploaded (which will be read from 

2446 ``file_obj``). If not provided, the upload will be concluded once 

2447 ``file_obj`` is exhausted. 

2448 

2449 :type content_type: str 

2450 :param content_type: (Optional) Type of content being uploaded. 

2451 

2452 :type num_retries: int 

2453 :param num_retries: 

2454 Number of upload retries. By default, only uploads with 

2455 if_generation_match set will be retried, as uploads without the 

2456 argument are not guaranteed to be idempotent. Setting num_retries 

2457 will override this default behavior and guarantee retries even when 

2458 if_generation_match is not set. (Deprecated: This argument 

2459 will be removed in a future release.) 

2460 

2461 :type client: :class:`~google.cloud.storage.client.Client` 

2462 :param client: 

2463 (Optional) The client to use. If not passed, falls back to the 

2464 ``client`` stored on the blob's bucket. 

2465 

2466 :type predefined_acl: str 

2467 :param predefined_acl: (Optional) Predefined access control list 

2468 

2469 :type if_generation_match: long 

2470 :param if_generation_match: 

2471 (Optional) See :ref:`using-if-generation-match` 

2472 

2473 :type if_generation_not_match: long 

2474 :param if_generation_not_match: 

2475 (Optional) See :ref:`using-if-generation-not-match` 

2476 

2477 :type if_metageneration_match: long 

2478 :param if_metageneration_match: 

2479 (Optional) See :ref:`using-if-metageneration-match` 

2480 

2481 :type if_metageneration_not_match: long 

2482 :param if_metageneration_not_match: 

2483 (Optional) See :ref:`using-if-metageneration-not-match` 

2484 

2485 :type timeout: float or tuple 

2486 :param timeout: 

2487 (Optional) The amount of time, in seconds, to wait 

2488 for the server response. See: :ref:`configuring_timeouts` 

2489 

2490 :type checksum: str 

2491 :param checksum: 

2492 (Optional) The type of checksum to compute to verify 

2493 the integrity of the object. If the upload is completed in a single 

2494 request, the checksum will be entirely precomputed and the remote 

2495 server will handle verification and error handling. If the upload 

2496 is too large and must be transmitted in multiple requests, the 

2497 checksum will be incrementally computed and the client will handle 

2498 verification and error handling, raising 

2499 google.resumable_media.common.DataCorruption on a mismatch and 

2500 attempting to delete the corrupted file. Supported values are 

2501 "md5", "crc32c" and None. The default is None. 

2502 

2503 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

2504 :param retry: (Optional) How to retry the RPC. A None value will disable 

2505 retries. A google.api_core.retry.Retry value will enable retries, 

2506 and the object will define retriable response codes and errors and 

2507 configure backoff and timeout options. 

2508 

2509 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

2510 Retry object and activates it only if certain conditions are met. 

2511 This class exists to provide safe defaults for RPC calls that are 

2512 not technically safe to retry normally (due to potential data 

2513 duplication or other side-effects) but become safe to retry if a 

2514 condition such as if_generation_match is set. 

2515 

2516 See the retry.py source code and docstrings in this package 

2517 (google.cloud.storage.retry) for information on retry types and how 

2518 to configure them. 

2519 

2520 Media operations (downloads and uploads) do not support non-default 

2521 predicates in a Retry object. The default will always be used. Other 

2522 configuration changes for Retry objects such as delays and deadlines 

2523 are respected. 

2524 

2525 :raises: :class:`~google.cloud.exceptions.GoogleCloudError` 

2526 if the upload response returns an error status. 

2527 """ 

2528 if num_retries is not None: 

2529 warnings.warn(_NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2) 

2530 # num_retries and retry are mutually exclusive. If num_retries is 

2531 # set and retry is exactly the default, then nullify retry for 

2532 # backwards compatibility. 

2533 if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED: 

2534 retry = None 

2535 

2536 _maybe_rewind(file_obj, rewind=rewind) 

2537 predefined_acl = ACL.validate_predefined(predefined_acl) 

2538 

2539 try: 

2540 created_json = self._do_upload( 

2541 client, 

2542 file_obj, 

2543 content_type, 

2544 size, 

2545 num_retries, 

2546 predefined_acl, 

2547 if_generation_match, 

2548 if_generation_not_match, 

2549 if_metageneration_match, 

2550 if_metageneration_not_match, 

2551 timeout=timeout, 

2552 checksum=checksum, 

2553 retry=retry, 

2554 ) 

2555 self._set_properties(created_json) 

2556 except resumable_media.InvalidResponse as exc: 

2557 _raise_from_invalid_response(exc) 

2558 

2559 def upload_from_filename( 

2560 self, 

2561 filename, 

2562 content_type=None, 

2563 num_retries=None, 

2564 client=None, 

2565 predefined_acl=None, 

2566 if_generation_match=None, 

2567 if_generation_not_match=None, 

2568 if_metageneration_match=None, 

2569 if_metageneration_not_match=None, 

2570 timeout=_DEFAULT_TIMEOUT, 

2571 checksum=None, 

2572 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

2573 ): 

2574 """Upload this blob's contents from the content of a named file. 

2575 

2576 The content type of the upload will be determined in order 

2577 of precedence: 

2578 

2579 - The value passed in to this method (if not :data:`None`) 

2580 - The value stored on the current blob 

2581 - The value given by ``mimetypes.guess_type`` 

2582 - The default value ('application/octet-stream') 

2583 

2584 .. note:: 

2585 The effect of uploading to an existing blob depends on the 

2586 "versioning" and "lifecycle" policies defined on the blob's 

2587 bucket. In the absence of those policies, upload will 

2588 overwrite any existing contents. 

2589 

2590 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning) 

2591 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle) 

2592 API documents for details. 

2593 

2594 If :attr:`user_project` is set on the bucket, bills the API request 

2595 to that project. 

2596 

2597 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-upload-encrypted-file#storage_upload_encrypted_file-python) 

2598 to upload a file with a 

2599 [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied). 

2600 

2601 :type filename: str 

2602 :param filename: The path to the file. 

2603 

2604 :type content_type: str 

2605 :param content_type: (Optional) Type of content being uploaded. 

2606 

2607 :type client: :class:`~google.cloud.storage.client.Client` 

2608 :param client: 

2609 (Optional) The client to use. If not passed, falls back to the 

2610 ``client`` stored on the blob's bucket. 

2611 

2612 :type num_retries: int 

2613 :param num_retries: 

2614 Number of upload retries. By default, only uploads with 

2615 if_generation_match set will be retried, as uploads without the 

2616 argument are not guaranteed to be idempotent. Setting num_retries 

2617 will override this default behavior and guarantee retries even when 

2618 if_generation_match is not set. (Deprecated: This argument 

2619 will be removed in a future release.) 

2620 

2621 :type predefined_acl: str 

2622 :param predefined_acl: (Optional) Predefined access control list 

2623 

2624 :type if_generation_match: long 

2625 :param if_generation_match: 

2626 (Optional) See :ref:`using-if-generation-match` 

2627 

2628 :type if_generation_not_match: long 

2629 :param if_generation_not_match: 

2630 (Optional) See :ref:`using-if-generation-not-match` 

2631 

2632 :type if_metageneration_match: long 

2633 :param if_metageneration_match: 

2634 (Optional) See :ref:`using-if-metageneration-match` 

2635 

2636 :type if_metageneration_not_match: long 

2637 :param if_metageneration_not_match: 

2638 (Optional) See :ref:`using-if-metageneration-not-match` 

2639 

2640 :type timeout: float or tuple 

2641 :param timeout: 

2642 (Optional) The amount of time, in seconds, to wait 

2643 for the server response. See: :ref:`configuring_timeouts` 

2644 

2645 :type checksum: str 

2646 :param checksum: 

2647 (Optional) The type of checksum to compute to verify 

2648 the integrity of the object. If the upload is completed in a single 

2649 request, the checksum will be entirely precomputed and the remote 

2650 server will handle verification and error handling. If the upload 

2651 is too large and must be transmitted in multiple requests, the 

2652 checksum will be incrementally computed and the client will handle 

2653 verification and error handling, raising 

2654 google.resumable_media.common.DataCorruption on a mismatch and 

2655 attempting to delete the corrupted file. Supported values are 

2656 "md5", "crc32c" and None. The default is None. 

2657 

2658 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

2659 :param retry: (Optional) How to retry the RPC. A None value will disable 

2660 retries. A google.api_core.retry.Retry value will enable retries, 

2661 and the object will define retriable response codes and errors and 

2662 configure backoff and timeout options. 

2663 

2664 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

2665 Retry object and activates it only if certain conditions are met. 

2666 This class exists to provide safe defaults for RPC calls that are 

2667 not technically safe to retry normally (due to potential data 

2668 duplication or other side-effects) but become safe to retry if a 

2669 condition such as if_generation_match is set. 

2670 

2671 See the retry.py source code and docstrings in this package 

2672 (google.cloud.storage.retry) for information on retry types and how 

2673 to configure them. 

2674 

2675 Media operations (downloads and uploads) do not support non-default 

2676 predicates in a Retry object. The default will always be used. Other 

2677 configuration changes for Retry objects such as delays and deadlines 

2678 are respected. 

2679 """ 

2680 content_type = self._get_content_type(content_type, filename=filename) 

2681 

2682 with open(filename, "rb") as file_obj: 

2683 total_bytes = os.fstat(file_obj.fileno()).st_size 

2684 self.upload_from_file( 

2685 file_obj, 

2686 content_type=content_type, 

2687 num_retries=num_retries, 

2688 client=client, 

2689 size=total_bytes, 

2690 predefined_acl=predefined_acl, 

2691 if_generation_match=if_generation_match, 

2692 if_generation_not_match=if_generation_not_match, 

2693 if_metageneration_match=if_metageneration_match, 

2694 if_metageneration_not_match=if_metageneration_not_match, 

2695 timeout=timeout, 

2696 checksum=checksum, 

2697 retry=retry, 

2698 ) 

2699 

2700 def upload_from_string( 

2701 self, 

2702 data, 

2703 content_type="text/plain", 

2704 num_retries=None, 

2705 client=None, 

2706 predefined_acl=None, 

2707 if_generation_match=None, 

2708 if_generation_not_match=None, 

2709 if_metageneration_match=None, 

2710 if_metageneration_not_match=None, 

2711 timeout=_DEFAULT_TIMEOUT, 

2712 checksum=None, 

2713 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

2714 ): 

2715 """Upload contents of this blob from the provided string. 

2716 

2717 .. note:: 

2718 The effect of uploading to an existing blob depends on the 

2719 "versioning" and "lifecycle" policies defined on the blob's 

2720 bucket. In the absence of those policies, upload will 

2721 overwrite any existing contents. 

2722 

2723 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning) 

2724 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle) 

2725 API documents for details. 

2726 

2727 If :attr:`user_project` is set on the bucket, bills the API request 

2728 to that project. 

2729 

2730 :type data: bytes or str 

2731 :param data: 

2732 The data to store in this blob. If the value is text, it will be 

2733 encoded as UTF-8. 

2734 

2735 :type content_type: str 

2736 :param content_type: 

2737 (Optional) Type of content being uploaded. Defaults to 

2738 ``'text/plain'``. 

2739 

2740 :type num_retries: int 

2741 :param num_retries: 

2742 Number of upload retries. By default, only uploads with 

2743 if_generation_match set will be retried, as uploads without the 

2744 argument are not guaranteed to be idempotent. Setting num_retries 

2745 will override this default behavior and guarantee retries even when 

2746 if_generation_match is not set. (Deprecated: This argument 

2747 will be removed in a future release.) 

2748 

2749 :type client: :class:`~google.cloud.storage.client.Client` 

2750 :param client: 

2751 (Optional) The client to use. If not passed, falls back to the 

2752 ``client`` stored on the blob's bucket. 

2753 

2754 :type predefined_acl: str 

2755 :param predefined_acl: (Optional) Predefined access control list 

2756 

2757 :type if_generation_match: long 

2758 :param if_generation_match: 

2759 (Optional) See :ref:`using-if-generation-match` 

2760 

2761 :type if_generation_not_match: long 

2762 :param if_generation_not_match: 

2763 (Optional) See :ref:`using-if-generation-not-match` 

2764 

2765 :type if_metageneration_match: long 

2766 :param if_metageneration_match: 

2767 (Optional) See :ref:`using-if-metageneration-match` 

2768 

2769 :type if_metageneration_not_match: long 

2770 :param if_metageneration_not_match: 

2771 (Optional) See :ref:`using-if-metageneration-not-match` 

2772 

2773 :type timeout: float or tuple 

2774 :param timeout: 

2775 (Optional) The amount of time, in seconds, to wait 

2776 for the server response. See: :ref:`configuring_timeouts` 

2777 

2778 :type checksum: str 

2779 :param checksum: 

2780 (Optional) The type of checksum to compute to verify 

2781 the integrity of the object. If the upload is completed in a single 

2782 request, the checksum will be entirely precomputed and the remote 

2783 server will handle verification and error handling. If the upload 

2784 is too large and must be transmitted in multiple requests, the 

2785 checksum will be incrementally computed and the client will handle 

2786 verification and error handling, raising 

2787 google.resumable_media.common.DataCorruption on a mismatch and 

2788 attempting to delete the corrupted file. Supported values are 

2789 "md5", "crc32c" and None. The default is None. 

2790 

2791 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

2792 :param retry: (Optional) How to retry the RPC. A None value will disable 

2793 retries. A google.api_core.retry.Retry value will enable retries, 

2794 and the object will define retriable response codes and errors and 

2795 configure backoff and timeout options. 

2796 

2797 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

2798 Retry object and activates it only if certain conditions are met. 

2799 This class exists to provide safe defaults for RPC calls that are 

2800 not technically safe to retry normally (due to potential data 

2801 duplication or other side-effects) but become safe to retry if a 

2802 condition such as if_generation_match is set. 

2803 

2804 See the retry.py source code and docstrings in this package 

2805 (google.cloud.storage.retry) for information on retry types and how 

2806 to configure them. 

2807 

2808 Media operations (downloads and uploads) do not support non-default 

2809 predicates in a Retry object. The default will always be used. Other 

2810 configuration changes for Retry objects such as delays and deadlines 

2811 are respected. 

2812 """ 

2813 data = _to_bytes(data, encoding="utf-8") 

2814 string_buffer = BytesIO(data) 

2815 self.upload_from_file( 

2816 file_obj=string_buffer, 

2817 size=len(data), 

2818 content_type=content_type, 

2819 num_retries=num_retries, 

2820 client=client, 

2821 predefined_acl=predefined_acl, 

2822 if_generation_match=if_generation_match, 

2823 if_generation_not_match=if_generation_not_match, 

2824 if_metageneration_match=if_metageneration_match, 

2825 if_metageneration_not_match=if_metageneration_not_match, 

2826 timeout=timeout, 

2827 checksum=checksum, 

2828 retry=retry, 

2829 ) 

2830 

2831 def create_resumable_upload_session( 

2832 self, 

2833 content_type=None, 

2834 size=None, 

2835 origin=None, 

2836 client=None, 

2837 timeout=_DEFAULT_TIMEOUT, 

2838 checksum=None, 

2839 predefined_acl=None, 

2840 if_generation_match=None, 

2841 if_generation_not_match=None, 

2842 if_metageneration_match=None, 

2843 if_metageneration_not_match=None, 

2844 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

2845 ): 

2846 """Create a resumable upload session. 

2847 

2848 Resumable upload sessions allow you to start an upload session from 

2849 one client and complete the session in another. This method is called 

2850 by the initiator to set the metadata and limits. The initiator then 

2851 passes the session URL to the client that will upload the binary data. 

2852 The client performs a PUT request on the session URL to complete the 

2853 upload. This process allows untrusted clients to upload to an 

2854 access-controlled bucket. 

2855 

2856 For more details, see the 

2857 documentation on [`signed URLs`](https://cloud.google.com/storage/docs/access-control/signed-urls#signing-resumable). 

2858 

2859 The content type of the upload will be determined in order 

2860 of precedence: 

2861 

2862 - The value passed in to this method (if not :data:`None`) 

2863 - The value stored on the current blob 

2864 - The default value ('application/octet-stream') 

2865 

2866 .. note:: 

2867 The effect of uploading to an existing blob depends on the 

2868 "versioning" and "lifecycle" policies defined on the blob's 

2869 bucket. In the absence of those policies, upload will 

2870 overwrite any existing contents. 

2871 

2872 See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning) 

2873 and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle) 

2874 API documents for details. 

2875 

2876 If :attr:`encryption_key` is set, the blob will be encrypted with 

2877 a [`customer-supplied`](https://cloud.google.com/storage/docs/encryption#customer-supplied) 

2878 encryption key. 

2879 

2880 If :attr:`user_project` is set on the bucket, bills the API request 

2881 to that project. 

2882 

2883 :type size: int 

2884 :param size: 

2885 (Optional) The maximum number of bytes that can be uploaded using 

2886 this session. If the size is not known when creating the session, 

2887 this should be left blank. 

2888 

2889 :type content_type: str 

2890 :param content_type: (Optional) Type of content being uploaded. 

2891 

2892 :type origin: str 

2893 :param origin: 

2894 (Optional) If set, the upload can only be completed by a user-agent 

2895 that uploads from the given origin. This can be useful when passing 

2896 the session to a web client. 

2897 

2898 :type client: :class:`~google.cloud.storage.client.Client` 

2899 :param client: 

2900 (Optional) The client to use. If not passed, falls back to the 

2901 ``client`` stored on the blob's bucket. 

2902 

2903 :type timeout: float or tuple 

2904 :param timeout: 

2905 (Optional) The amount of time, in seconds, to wait 

2906 for the server response. See: :ref:`configuring_timeouts` 

2907 

2908 :type checksum: str 

2909 :param checksum: 

2910 (Optional) The type of checksum to compute to verify 

2911 the integrity of the object. After the upload is complete, the 

2912 server-computed checksum of the resulting object will be checked 

2913 and google.resumable_media.common.DataCorruption will be raised on 

2914 a mismatch. On a validation failure, the client will attempt to 

2915 delete the uploaded object automatically. Supported values 

2916 are "md5", "crc32c" and None. The default is None. 

2917 

2918 :type predefined_acl: str 

2919 :param predefined_acl: (Optional) Predefined access control list 

2920 

2921 :type if_generation_match: long 

2922 :param if_generation_match: 

2923 (Optional) See :ref:`using-if-generation-match` 

2924 

2925 :type if_generation_not_match: long 

2926 :param if_generation_not_match: 

2927 (Optional) See :ref:`using-if-generation-not-match` 

2928 

2929 :type if_metageneration_match: long 

2930 :param if_metageneration_match: 

2931 (Optional) See :ref:`using-if-metageneration-match` 

2932 

2933 :type if_metageneration_not_match: long 

2934 :param if_metageneration_not_match: 

2935 (Optional) See :ref:`using-if-metageneration-not-match` 

2936 

2937 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

2938 :param retry: (Optional) How to retry the RPC. A None value will disable 

2939 retries. A google.api_core.retry.Retry value will enable retries, 

2940 and the object will define retriable response codes and errors and 

2941 configure backoff and timeout options. 

2942 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

2943 Retry object and activates it only if certain conditions are met. 

2944 This class exists to provide safe defaults for RPC calls that are 

2945 not technically safe to retry normally (due to potential data 

2946 duplication or other side-effects) but become safe to retry if a 

2947 condition such as if_generation_match is set. 

2948 See the retry.py source code and docstrings in this package 

2949 (google.cloud.storage.retry) for information on retry types and how 

2950 to configure them. 

2951 Media operations (downloads and uploads) do not support non-default 

2952 predicates in a Retry object. The default will always be used. Other 

2953 configuration changes for Retry objects such as delays and deadlines 

2954 are respected. 

2955 

2956 :rtype: str 

2957 :returns: The resumable upload session URL. The upload can be 

2958 completed by making an HTTP PUT request with the 

2959 file's contents. 

2960 

2961 :raises: :class:`google.cloud.exceptions.GoogleCloudError` 

2962 if the session creation response returns an error status. 

2963 """ 

2964 

2965 # Handle ConditionalRetryPolicy. 

2966 if isinstance(retry, ConditionalRetryPolicy): 

2967 # Conditional retries are designed for non-media calls, which change 

2968 # arguments into query_params dictionaries. Media operations work 

2969 # differently, so here we make a "fake" query_params to feed to the 

2970 # ConditionalRetryPolicy. 

2971 query_params = { 

2972 "ifGenerationMatch": if_generation_match, 

2973 "ifMetagenerationMatch": if_metageneration_match, 

2974 } 

2975 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) 

2976 

2977 extra_headers = {} 

2978 if origin is not None: 

2979 # This header is specifically for client-side uploads, it 

2980 # determines the origins allowed for CORS. 

2981 extra_headers["Origin"] = origin 

2982 

2983 try: 

2984 fake_stream = BytesIO(b"") 

2985 # Send a fake the chunk size which we **know** will be acceptable 

2986 # to the `ResumableUpload` constructor. The chunk size only 

2987 # matters when **sending** bytes to an upload. 

2988 upload, _ = self._initiate_resumable_upload( 

2989 client, 

2990 fake_stream, 

2991 content_type, 

2992 size, 

2993 None, 

2994 predefined_acl=predefined_acl, 

2995 if_generation_match=if_generation_match, 

2996 if_generation_not_match=if_generation_not_match, 

2997 if_metageneration_match=if_metageneration_match, 

2998 if_metageneration_not_match=if_metageneration_not_match, 

2999 extra_headers=extra_headers, 

3000 chunk_size=self._CHUNK_SIZE_MULTIPLE, 

3001 timeout=timeout, 

3002 checksum=checksum, 

3003 retry=retry, 

3004 ) 

3005 

3006 return upload.resumable_url 

3007 except resumable_media.InvalidResponse as exc: 

3008 _raise_from_invalid_response(exc) 

3009 

3010 def get_iam_policy( 

3011 self, 

3012 client=None, 

3013 requested_policy_version=None, 

3014 timeout=_DEFAULT_TIMEOUT, 

3015 retry=DEFAULT_RETRY, 

3016 ): 

3017 """Retrieve the IAM policy for the object. 

3018 

3019 .. note:: 

3020 

3021 Blob- / object-level IAM support does not yet exist and methods 

3022 currently call an internal ACL backend not providing any utility 

3023 beyond the blob's :attr:`acl` at this time. The API may be enhanced 

3024 in the future and is currently undocumented. Use :attr:`acl` for 

3025 managing object access control. 

3026 

3027 If :attr:`user_project` is set on the bucket, bills the API request 

3028 to that project. 

3029 

3030 :type client: :class:`~google.cloud.storage.client.Client` 

3031 :param client: 

3032 (Optional) The client to use. If not passed, falls back to the 

3033 ``client`` stored on the current object's bucket. 

3034 

3035 :type requested_policy_version: int or ``NoneType`` 

3036 :param requested_policy_version: 

3037 (Optional) The version of IAM policies to request. If a policy 

3038 with a condition is requested without setting this, the server will 

3039 return an error. This must be set to a value of 3 to retrieve IAM 

3040 policies containing conditions. This is to prevent client code that 

3041 isn't aware of IAM conditions from interpreting and modifying 

3042 policies incorrectly. The service might return a policy with 

3043 version lower than the one that was requested, based on the feature 

3044 syntax in the policy fetched. 

3045 

3046 :type timeout: float or tuple 

3047 :param timeout: 

3048 (Optional) The amount of time, in seconds, to wait 

3049 for the server response. See: :ref:`configuring_timeouts` 

3050 

3051 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3052 :param retry: 

3053 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3054 

3055 :rtype: :class:`google.api_core.iam.Policy` 

3056 :returns: the policy instance, based on the resource returned from 

3057 the ``getIamPolicy`` API request. 

3058 """ 

3059 client = self._require_client(client) 

3060 

3061 query_params = {} 

3062 

3063 if self.user_project is not None: 

3064 query_params["userProject"] = self.user_project 

3065 

3066 if requested_policy_version is not None: 

3067 query_params["optionsRequestedPolicyVersion"] = requested_policy_version 

3068 

3069 info = client._get_resource( 

3070 f"{self.path}/iam", 

3071 query_params=query_params, 

3072 timeout=timeout, 

3073 retry=retry, 

3074 _target_object=None, 

3075 ) 

3076 return Policy.from_api_repr(info) 

3077 

3078 def set_iam_policy( 

3079 self, 

3080 policy, 

3081 client=None, 

3082 timeout=_DEFAULT_TIMEOUT, 

3083 retry=DEFAULT_RETRY_IF_ETAG_IN_JSON, 

3084 ): 

3085 """Update the IAM policy for the bucket. 

3086 

3087 .. note:: 

3088 

3089 Blob- / object-level IAM support does not yet exist and methods 

3090 currently call an internal ACL backend not providing any utility 

3091 beyond the blob's :attr:`acl` at this time. The API may be enhanced 

3092 in the future and is currently undocumented. Use :attr:`acl` for 

3093 managing object access control. 

3094 

3095 If :attr:`user_project` is set on the bucket, bills the API request 

3096 to that project. 

3097 

3098 :type policy: :class:`google.api_core.iam.Policy` 

3099 :param policy: policy instance used to update bucket's IAM policy. 

3100 

3101 :type client: :class:`~google.cloud.storage.client.Client` 

3102 :param client: 

3103 (Optional) The client to use. If not passed, falls back to the 

3104 ``client`` stored on the current bucket. 

3105 

3106 :type timeout: float or tuple 

3107 :param timeout: 

3108 (Optional) The amount of time, in seconds, to wait 

3109 for the server response. See: :ref:`configuring_timeouts` 

3110 

3111 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3112 :param retry: 

3113 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3114 

3115 :rtype: :class:`google.api_core.iam.Policy` 

3116 :returns: the policy instance, based on the resource returned from 

3117 the ``setIamPolicy`` API request. 

3118 """ 

3119 client = self._require_client(client) 

3120 

3121 query_params = {} 

3122 

3123 if self.user_project is not None: 

3124 query_params["userProject"] = self.user_project 

3125 

3126 path = f"{self.path}/iam" 

3127 resource = policy.to_api_repr() 

3128 resource["resourceId"] = self.path 

3129 info = client._put_resource( 

3130 path, 

3131 resource, 

3132 query_params=query_params, 

3133 timeout=timeout, 

3134 retry=retry, 

3135 _target_object=None, 

3136 ) 

3137 return Policy.from_api_repr(info) 

3138 

3139 def test_iam_permissions( 

3140 self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY 

3141 ): 

3142 """API call: test permissions 

3143 

3144 .. note:: 

3145 

3146 Blob- / object-level IAM support does not yet exist and methods 

3147 currently call an internal ACL backend not providing any utility 

3148 beyond the blob's :attr:`acl` at this time. The API may be enhanced 

3149 in the future and is currently undocumented. Use :attr:`acl` for 

3150 managing object access control. 

3151 

3152 If :attr:`user_project` is set on the bucket, bills the API request 

3153 to that project. 

3154 

3155 :type permissions: list of string 

3156 :param permissions: the permissions to check 

3157 

3158 :type client: :class:`~google.cloud.storage.client.Client` 

3159 :param client: 

3160 (Optional) The client to use. If not passed, falls back to the 

3161 ``client`` stored on the current bucket. 

3162 

3163 :type timeout: float or tuple 

3164 :param timeout: 

3165 (Optional) The amount of time, in seconds, to wait 

3166 for the server response. See: :ref:`configuring_timeouts` 

3167 

3168 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3169 :param retry: 

3170 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3171 

3172 :rtype: list of string 

3173 :returns: the permissions returned by the ``testIamPermissions`` API 

3174 request. 

3175 """ 

3176 client = self._require_client(client) 

3177 query_params = {"permissions": permissions} 

3178 

3179 if self.user_project is not None: 

3180 query_params["userProject"] = self.user_project 

3181 

3182 path = f"{self.path}/iam/testPermissions" 

3183 resp = client._get_resource( 

3184 path, 

3185 query_params=query_params, 

3186 timeout=timeout, 

3187 retry=retry, 

3188 _target_object=None, 

3189 ) 

3190 

3191 return resp.get("permissions", []) 

3192 

3193 def make_public( 

3194 self, 

3195 client=None, 

3196 timeout=_DEFAULT_TIMEOUT, 

3197 if_generation_match=None, 

3198 if_generation_not_match=None, 

3199 if_metageneration_match=None, 

3200 if_metageneration_not_match=None, 

3201 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, 

3202 ): 

3203 """Update blob's ACL, granting read access to anonymous users. 

3204 

3205 :type client: :class:`~google.cloud.storage.client.Client` or 

3206 ``NoneType`` 

3207 :param client: (Optional) The client to use. If not passed, falls back 

3208 to the ``client`` stored on the blob's bucket. 

3209 

3210 :type timeout: float or tuple 

3211 :param timeout: 

3212 (Optional) The amount of time, in seconds, to wait 

3213 for the server response. See: :ref:`configuring_timeouts` 

3214 

3215 :type if_generation_match: long 

3216 :param if_generation_match: 

3217 (Optional) See :ref:`using-if-generation-match` 

3218 

3219 :type if_generation_not_match: long 

3220 :param if_generation_not_match: 

3221 (Optional) See :ref:`using-if-generation-not-match` 

3222 

3223 :type if_metageneration_match: long 

3224 :param if_metageneration_match: 

3225 (Optional) See :ref:`using-if-metageneration-match` 

3226 

3227 :type if_metageneration_not_match: long 

3228 :param if_metageneration_not_match: 

3229 (Optional) See :ref:`using-if-metageneration-not-match` 

3230 

3231 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3232 :param retry: 

3233 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3234 """ 

3235 self.acl.all().grant_read() 

3236 self.acl.save( 

3237 client=client, 

3238 timeout=timeout, 

3239 if_generation_match=if_generation_match, 

3240 if_generation_not_match=if_generation_not_match, 

3241 if_metageneration_match=if_metageneration_match, 

3242 if_metageneration_not_match=if_metageneration_not_match, 

3243 retry=retry, 

3244 ) 

3245 

3246 def make_private( 

3247 self, 

3248 client=None, 

3249 timeout=_DEFAULT_TIMEOUT, 

3250 if_generation_match=None, 

3251 if_generation_not_match=None, 

3252 if_metageneration_match=None, 

3253 if_metageneration_not_match=None, 

3254 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, 

3255 ): 

3256 """Update blob's ACL, revoking read access for anonymous users. 

3257 

3258 :type client: :class:`~google.cloud.storage.client.Client` or 

3259 ``NoneType`` 

3260 :param client: (Optional) The client to use. If not passed, falls back 

3261 to the ``client`` stored on the blob's bucket. 

3262 

3263 :type timeout: float or tuple 

3264 :param timeout: 

3265 (Optional) The amount of time, in seconds, to wait 

3266 for the server response. See: :ref:`configuring_timeouts` 

3267 

3268 :type if_generation_match: long 

3269 :param if_generation_match: 

3270 (Optional) See :ref:`using-if-generation-match` 

3271 

3272 :type if_generation_not_match: long 

3273 :param if_generation_not_match: 

3274 (Optional) See :ref:`using-if-generation-not-match` 

3275 

3276 :type if_metageneration_match: long 

3277 :param if_metageneration_match: 

3278 (Optional) See :ref:`using-if-metageneration-match` 

3279 

3280 :type if_metageneration_not_match: long 

3281 :param if_metageneration_not_match: 

3282 (Optional) See :ref:`using-if-metageneration-not-match` 

3283 

3284 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3285 :param retry: 

3286 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3287 """ 

3288 self.acl.all().revoke_read() 

3289 self.acl.save( 

3290 client=client, 

3291 timeout=timeout, 

3292 if_generation_match=if_generation_match, 

3293 if_generation_not_match=if_generation_not_match, 

3294 if_metageneration_match=if_metageneration_match, 

3295 if_metageneration_not_match=if_metageneration_not_match, 

3296 retry=retry, 

3297 ) 

3298 

3299 def compose( 

3300 self, 

3301 sources, 

3302 client=None, 

3303 timeout=_DEFAULT_TIMEOUT, 

3304 if_generation_match=None, 

3305 if_metageneration_match=None, 

3306 if_source_generation_match=None, 

3307 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

3308 ): 

3309 """Concatenate source blobs into this one. 

3310 

3311 If :attr:`user_project` is set on the bucket, bills the API request 

3312 to that project. 

3313 

3314 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/objects/compose) 

3315 and a [code sample](https://cloud.google.com/storage/docs/samples/storage-compose-file#storage_compose_file-python). 

3316 

3317 :type sources: list of :class:`Blob` 

3318 :param sources: Blobs whose contents will be composed into this blob. 

3319 

3320 :type client: :class:`~google.cloud.storage.client.Client` 

3321 :param client: 

3322 (Optional) The client to use. If not passed, falls back to the 

3323 ``client`` stored on the blob's bucket. 

3324 

3325 :type timeout: float or tuple 

3326 :param timeout: 

3327 (Optional) The amount of time, in seconds, to wait 

3328 for the server response. See: :ref:`configuring_timeouts` 

3329 

3330 :type if_generation_match: long 

3331 :param if_generation_match: 

3332 (Optional) Makes the operation conditional on whether the 

3333 destination object's current generation matches the given value. 

3334 Setting to 0 makes the operation succeed only if there are no live 

3335 versions of the object. 

3336 Note: In a previous version, this argument worked identically to the 

3337 ``if_source_generation_match`` argument. For 

3338 backwards-compatibility reasons, if a list is passed in, 

3339 this argument will behave like ``if_source_generation_match`` 

3340 and also issue a DeprecationWarning. 

3341 

3342 :type if_metageneration_match: long 

3343 :param if_metageneration_match: 

3344 (Optional) Makes the operation conditional on whether the 

3345 destination object's current metageneration matches the given 

3346 value. 

3347 

3348 If a list of long is passed in, no match operation will be 

3349 performed. (Deprecated: type(list of long) is supported for 

3350 backwards-compatability reasons only.) 

3351 

3352 :type if_source_generation_match: list of long 

3353 :param if_source_generation_match: 

3354 (Optional) Makes the operation conditional on whether the current 

3355 generation of each source blob matches the corresponding generation. 

3356 The list must match ``sources`` item-to-item. 

3357 

3358 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3359 :param retry: 

3360 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3361 """ 

3362 sources_len = len(sources) 

3363 client = self._require_client(client) 

3364 query_params = {} 

3365 

3366 if isinstance(if_generation_match, list): 

3367 warnings.warn( 

3368 _COMPOSE_IF_GENERATION_LIST_DEPRECATED, 

3369 DeprecationWarning, 

3370 stacklevel=2, 

3371 ) 

3372 

3373 if if_source_generation_match is not None: 

3374 raise ValueError( 

3375 _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR 

3376 ) 

3377 

3378 if_source_generation_match = if_generation_match 

3379 if_generation_match = None 

3380 

3381 if isinstance(if_metageneration_match, list): 

3382 warnings.warn( 

3383 _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED, 

3384 DeprecationWarning, 

3385 stacklevel=2, 

3386 ) 

3387 

3388 if_metageneration_match = None 

3389 

3390 if if_source_generation_match is None: 

3391 if_source_generation_match = [None] * sources_len 

3392 if len(if_source_generation_match) != sources_len: 

3393 raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR) 

3394 

3395 source_objects = [] 

3396 for source, source_generation in zip(sources, if_source_generation_match): 

3397 source_object = {"name": source.name, "generation": source.generation} 

3398 

3399 preconditions = {} 

3400 if source_generation is not None: 

3401 preconditions["ifGenerationMatch"] = source_generation 

3402 

3403 if preconditions: 

3404 source_object["objectPreconditions"] = preconditions 

3405 

3406 source_objects.append(source_object) 

3407 

3408 request = { 

3409 "sourceObjects": source_objects, 

3410 "destination": self._properties.copy(), 

3411 } 

3412 

3413 if self.user_project is not None: 

3414 query_params["userProject"] = self.user_project 

3415 

3416 _add_generation_match_parameters( 

3417 query_params, 

3418 if_generation_match=if_generation_match, 

3419 if_metageneration_match=if_metageneration_match, 

3420 ) 

3421 

3422 api_response = client._post_resource( 

3423 f"{self.path}/compose", 

3424 request, 

3425 query_params=query_params, 

3426 timeout=timeout, 

3427 retry=retry, 

3428 _target_object=self, 

3429 ) 

3430 self._set_properties(api_response) 

3431 

3432 def rewrite( 

3433 self, 

3434 source, 

3435 token=None, 

3436 client=None, 

3437 if_generation_match=None, 

3438 if_generation_not_match=None, 

3439 if_metageneration_match=None, 

3440 if_metageneration_not_match=None, 

3441 if_source_generation_match=None, 

3442 if_source_generation_not_match=None, 

3443 if_source_metageneration_match=None, 

3444 if_source_metageneration_not_match=None, 

3445 timeout=_DEFAULT_TIMEOUT, 

3446 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

3447 ): 

3448 """Rewrite source blob into this one. 

3449 

3450 If :attr:`user_project` is set on the bucket, bills the API request 

3451 to that project. 

3452 

3453 .. note:: 

3454 

3455 ``rewrite`` is not supported in a ``Batch`` context. 

3456 

3457 :type source: :class:`Blob` 

3458 :param source: blob whose contents will be rewritten into this blob. 

3459 

3460 :type token: str 

3461 :param token: 

3462 (Optional) Token returned from an earlier, not-completed call to 

3463 rewrite the same source blob. If passed, result will include 

3464 updated status, total bytes written. 

3465 

3466 :type client: :class:`~google.cloud.storage.client.Client` 

3467 :param client: 

3468 (Optional) The client to use. If not passed, falls back to the 

3469 ``client`` stored on the blob's bucket. 

3470 

3471 :type if_generation_match: long 

3472 :param if_generation_match: 

3473 (Optional) See :ref:`using-if-generation-match` 

3474 Note that the generation to be matched is that of the 

3475 ``destination`` blob. 

3476 

3477 :type if_generation_not_match: long 

3478 :param if_generation_not_match: 

3479 (Optional) See :ref:`using-if-generation-not-match` 

3480 Note that the generation to be matched is that of the 

3481 ``destination`` blob. 

3482 

3483 :type if_metageneration_match: long 

3484 :param if_metageneration_match: 

3485 (Optional) See :ref:`using-if-metageneration-match` 

3486 Note that the metageneration to be matched is that of the 

3487 ``destination`` blob. 

3488 

3489 :type if_metageneration_not_match: long 

3490 :param if_metageneration_not_match: 

3491 (Optional) See :ref:`using-if-metageneration-not-match` 

3492 Note that the metageneration to be matched is that of the 

3493 ``destination`` blob. 

3494 

3495 :type if_source_generation_match: long 

3496 :param if_source_generation_match: 

3497 (Optional) Makes the operation conditional on whether the source 

3498 object's generation matches the given value. 

3499 

3500 :type if_source_generation_not_match: long 

3501 :param if_source_generation_not_match: 

3502 (Optional) Makes the operation conditional on whether the source 

3503 object's generation does not match the given value. 

3504 

3505 :type if_source_metageneration_match: long 

3506 :param if_source_metageneration_match: 

3507 (Optional) Makes the operation conditional on whether the source 

3508 object's current metageneration matches the given value. 

3509 

3510 :type if_source_metageneration_not_match: long 

3511 :param if_source_metageneration_not_match: 

3512 (Optional) Makes the operation conditional on whether the source 

3513 object's current metageneration does not match the given value. 

3514 

3515 :type timeout: float or tuple 

3516 :param timeout: 

3517 (Optional) The amount of time, in seconds, to wait 

3518 for the server response. See: :ref:`configuring_timeouts` 

3519 

3520 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3521 :param retry: 

3522 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3523 

3524 :rtype: tuple 

3525 :returns: ``(token, bytes_rewritten, total_bytes)``, where ``token`` 

3526 is a rewrite token (``None`` if the rewrite is complete), 

3527 ``bytes_rewritten`` is the number of bytes rewritten so far, 

3528 and ``total_bytes`` is the total number of bytes to be 

3529 rewritten. 

3530 """ 

3531 client = self._require_client(client) 

3532 headers = _get_encryption_headers(self._encryption_key) 

3533 headers.update(_get_encryption_headers(source._encryption_key, source=True)) 

3534 

3535 query_params = self._query_params 

3536 if "generation" in query_params: 

3537 del query_params["generation"] 

3538 

3539 if token: 

3540 query_params["rewriteToken"] = token 

3541 

3542 if source.generation: 

3543 query_params["sourceGeneration"] = source.generation 

3544 

3545 # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object 

3546 # at rest, object resource metadata will store the version of the Key Management 

3547 # Service cryptographic material. If a Blob instance with KMS Key metadata set is 

3548 # used to rewrite the object, then the existing kmsKeyName version 

3549 # value can't be used in the rewrite request and the client instead ignores it. 

3550 if ( 

3551 self.kms_key_name is not None 

3552 and "cryptoKeyVersions" not in self.kms_key_name 

3553 ): 

3554 query_params["destinationKmsKeyName"] = self.kms_key_name 

3555 

3556 _add_generation_match_parameters( 

3557 query_params, 

3558 if_generation_match=if_generation_match, 

3559 if_generation_not_match=if_generation_not_match, 

3560 if_metageneration_match=if_metageneration_match, 

3561 if_metageneration_not_match=if_metageneration_not_match, 

3562 if_source_generation_match=if_source_generation_match, 

3563 if_source_generation_not_match=if_source_generation_not_match, 

3564 if_source_metageneration_match=if_source_metageneration_match, 

3565 if_source_metageneration_not_match=if_source_metageneration_not_match, 

3566 ) 

3567 

3568 path = f"{source.path}/rewriteTo{self.path}" 

3569 api_response = client._post_resource( 

3570 path, 

3571 self._properties, 

3572 query_params=query_params, 

3573 headers=headers, 

3574 timeout=timeout, 

3575 retry=retry, 

3576 _target_object=self, 

3577 ) 

3578 rewritten = int(api_response["totalBytesRewritten"]) 

3579 size = int(api_response["objectSize"]) 

3580 

3581 # The resource key is set if and only if the API response is 

3582 # completely done. Additionally, there is no rewrite token to return 

3583 # in this case. 

3584 if api_response["done"]: 

3585 self._set_properties(api_response["resource"]) 

3586 return None, rewritten, size 

3587 

3588 return api_response["rewriteToken"], rewritten, size 

3589 

3590 def update_storage_class( 

3591 self, 

3592 new_class, 

3593 client=None, 

3594 if_generation_match=None, 

3595 if_generation_not_match=None, 

3596 if_metageneration_match=None, 

3597 if_metageneration_not_match=None, 

3598 if_source_generation_match=None, 

3599 if_source_generation_not_match=None, 

3600 if_source_metageneration_match=None, 

3601 if_source_metageneration_not_match=None, 

3602 timeout=_DEFAULT_TIMEOUT, 

3603 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 

3604 ): 

3605 """Update blob's storage class via a rewrite-in-place. This helper will 

3606 wait for the rewrite to complete before returning, so it may take some 

3607 time for large files. 

3608 

3609 See 

3610 https://cloud.google.com/storage/docs/per-object-storage-class 

3611 

3612 If :attr:`user_project` is set on the bucket, bills the API request 

3613 to that project. 

3614 

3615 :type new_class: str 

3616 :param new_class: 

3617 new storage class for the object. One of: 

3618 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, 

3619 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, 

3620 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, 

3621 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, 

3622 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, 

3623 or 

3624 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`. 

3625 

3626 :type client: :class:`~google.cloud.storage.client.Client` 

3627 :param client: 

3628 (Optional) The client to use. If not passed, falls back to the 

3629 ``client`` stored on the blob's bucket. 

3630 

3631 :type if_generation_match: long 

3632 :param if_generation_match: 

3633 (Optional) See :ref:`using-if-generation-match` 

3634 Note that the generation to be matched is that of the 

3635 ``destination`` blob. 

3636 

3637 :type if_generation_not_match: long 

3638 :param if_generation_not_match: 

3639 (Optional) See :ref:`using-if-generation-not-match` 

3640 Note that the generation to be matched is that of the 

3641 ``destination`` blob. 

3642 

3643 :type if_metageneration_match: long 

3644 :param if_metageneration_match: 

3645 (Optional) See :ref:`using-if-metageneration-match` 

3646 Note that the metageneration to be matched is that of the 

3647 ``destination`` blob. 

3648 

3649 :type if_metageneration_not_match: long 

3650 :param if_metageneration_not_match: 

3651 (Optional) See :ref:`using-if-metageneration-not-match` 

3652 Note that the metageneration to be matched is that of the 

3653 ``destination`` blob. 

3654 

3655 :type if_source_generation_match: long 

3656 :param if_source_generation_match: 

3657 (Optional) Makes the operation conditional on whether the source 

3658 object's generation matches the given value. 

3659 

3660 :type if_source_generation_not_match: long 

3661 :param if_source_generation_not_match: 

3662 (Optional) Makes the operation conditional on whether the source 

3663 object's generation does not match the given value. 

3664 

3665 :type if_source_metageneration_match: long 

3666 :param if_source_metageneration_match: 

3667 (Optional) Makes the operation conditional on whether the source 

3668 object's current metageneration matches the given value. 

3669 

3670 :type if_source_metageneration_not_match: long 

3671 :param if_source_metageneration_not_match: 

3672 (Optional) Makes the operation conditional on whether the source 

3673 object's current metageneration does not match the given value. 

3674 

3675 :type timeout: float or tuple 

3676 :param timeout: 

3677 (Optional) The amount of time, in seconds, to wait 

3678 for the server response. See: :ref:`configuring_timeouts` 

3679 

3680 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

3681 :param retry: 

3682 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

3683 """ 

3684 # Update current blob's storage class prior to rewrite 

3685 self._patch_property("storageClass", new_class) 

3686 

3687 # Execute consecutive rewrite operations until operation is done 

3688 token, _, _ = self.rewrite( 

3689 self, 

3690 if_generation_match=if_generation_match, 

3691 if_generation_not_match=if_generation_not_match, 

3692 if_metageneration_match=if_metageneration_match, 

3693 if_metageneration_not_match=if_metageneration_not_match, 

3694 if_source_generation_match=if_source_generation_match, 

3695 if_source_generation_not_match=if_source_generation_not_match, 

3696 if_source_metageneration_match=if_source_metageneration_match, 

3697 if_source_metageneration_not_match=if_source_metageneration_not_match, 

3698 timeout=timeout, 

3699 retry=retry, 

3700 ) 

3701 while token is not None: 

3702 token, _, _ = self.rewrite( 

3703 self, 

3704 token=token, 

3705 if_generation_match=if_generation_match, 

3706 if_generation_not_match=if_generation_not_match, 

3707 if_metageneration_match=if_metageneration_match, 

3708 if_metageneration_not_match=if_metageneration_not_match, 

3709 if_source_generation_match=if_source_generation_match, 

3710 if_source_generation_not_match=if_source_generation_not_match, 

3711 if_source_metageneration_match=if_source_metageneration_match, 

3712 if_source_metageneration_not_match=if_source_metageneration_not_match, 

3713 timeout=timeout, 

3714 retry=retry, 

3715 ) 

3716 

3717 def open( 

3718 self, 

3719 mode="r", 

3720 chunk_size=None, 

3721 ignore_flush=None, 

3722 encoding=None, 

3723 errors=None, 

3724 newline=None, 

3725 **kwargs, 

3726 ): 

3727 r"""Create a file handler for file-like I/O to or from this blob. 

3728 

3729 This method can be used as a context manager, just like Python's 

3730 built-in 'open()' function. 

3731 

3732 While reading, as with other read methods, if blob.generation is not set 

3733 the most recent blob generation will be used. Because the file-like IO 

3734 reader downloads progressively in chunks, this could result in data from 

3735 multiple versions being mixed together. If this is a concern, use 

3736 either bucket.get_blob(), or blob.reload(), which will download the 

3737 latest generation number and set it; or, if the generation is known, set 

3738 it manually, for instance with bucket.blob(generation=123456). 

3739 

3740 Checksumming (hashing) to verify data integrity is disabled for reads 

3741 using this feature because reads are implemented using request ranges, 

3742 which do not provide checksums to validate. See 

3743 https://cloud.google.com/storage/docs/hashes-etags for details. 

3744 

3745 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_fileio_write_read.py). 

3746 

3747 Keyword arguments to pass to the underlying API calls. 

3748 For both uploads and downloads, the following arguments are 

3749 supported: 

3750 

3751 - ``if_generation_match`` 

3752 - ``if_generation_not_match`` 

3753 - ``if_metageneration_match`` 

3754 - ``if_metageneration_not_match`` 

3755 - ``timeout`` 

3756 - ``retry`` 

3757 

3758 For downloads only, the following additional arguments are supported: 

3759 

3760 - ``raw_download`` 

3761 

3762 For uploads only, the following additional arguments are supported: 

3763 

3764 - ``content_type`` 

3765 - ``num_retries`` 

3766 - ``predefined_acl`` 

3767 - ``checksum`` 

3768 

3769 .. note:: 

3770 

3771 ``num_retries`` is supported for backwards-compatibility 

3772 reasons only; please use ``retry`` with a Retry object or 

3773 ConditionalRetryPolicy instead. 

3774 

3775 :type mode: str 

3776 :param mode: 

3777 (Optional) A mode string, as per standard Python `open()` semantics.The first 

3778 character must be 'r', to open the blob for reading, or 'w' to open 

3779 it for writing. The second character, if present, must be 't' for 

3780 (unicode) text mode, or 'b' for bytes mode. If the second character 

3781 is omitted, text mode is the default. 

3782 

3783 :type chunk_size: long 

3784 :param chunk_size: 

3785 (Optional) For reads, the minimum number of bytes to read at a time. 

3786 If fewer bytes than the chunk_size are requested, the remainder is 

3787 buffered. For writes, the maximum number of bytes to buffer before 

3788 sending data to the server, and the size of each request when data 

3789 is sent. Writes are implemented as a "resumable upload", so 

3790 chunk_size for writes must be exactly a multiple of 256KiB as with 

3791 other resumable uploads. The default is 40 MiB. 

3792 

3793 :type ignore_flush: bool 

3794 :param ignore_flush: 

3795 (Optional) For non text-mode writes, makes flush() do nothing 

3796 instead of raising an error. flush() without closing is not 

3797 supported by the remote service and therefore calling it normally 

3798 results in io.UnsupportedOperation. However, that behavior is 

3799 incompatible with some consumers and wrappers of file objects in 

3800 Python, such as zipfile.ZipFile or io.TextIOWrapper. Setting 

3801 ignore_flush will cause flush() to successfully do nothing, for 

3802 compatibility with those contexts. The correct way to actually flush 

3803 data to the remote server is to close() (using a context manager, 

3804 such as in the example, will cause this to happen automatically). 

3805 

3806 :type encoding: str 

3807 :param encoding: 

3808 (Optional) For text mode only, the name of the encoding that the stream will 

3809 be decoded or encoded with. If omitted, it defaults to 

3810 locale.getpreferredencoding(False). 

3811 

3812 :type errors: str 

3813 :param errors: 

3814 (Optional) For text mode only, an optional string that specifies how encoding 

3815 and decoding errors are to be handled. Pass 'strict' to raise a 

3816 ValueError exception if there is an encoding error (the default of 

3817 None has the same effect), or pass 'ignore' to ignore errors. (Note 

3818 that ignoring encoding errors can lead to data loss.) Other more 

3819 rarely-used options are also available; see the Python 'io' module 

3820 documentation for 'io.TextIOWrapper' for a complete list. 

3821 

3822 :type newline: str 

3823 :param newline: 

3824 (Optional) For text mode only, controls how line endings are handled. It can 

3825 be None, '', '\n', '\r', and '\r\n'. If None, reads use "universal 

3826 newline mode" and writes use the system default. See the Python 

3827 'io' module documentation for 'io.TextIOWrapper' for details. 

3828 

3829 :returns: A 'BlobReader' or 'BlobWriter' from 

3830 'google.cloud.storage.fileio', or an 'io.TextIOWrapper' around one 

3831 of those classes, depending on the 'mode' argument. 

3832 """ 

3833 if mode == "rb": 

3834 if encoding or errors or newline: 

3835 raise ValueError( 

3836 "encoding, errors and newline arguments are for text mode only" 

3837 ) 

3838 if ignore_flush: 

3839 raise ValueError( 

3840 "ignore_flush argument is for non-text write mode only" 

3841 ) 

3842 return BlobReader(self, chunk_size=chunk_size, **kwargs) 

3843 elif mode == "wb": 

3844 if encoding or errors or newline: 

3845 raise ValueError( 

3846 "encoding, errors and newline arguments are for text mode only" 

3847 ) 

3848 return BlobWriter( 

3849 self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs 

3850 ) 

3851 elif mode in ("r", "rt"): 

3852 if ignore_flush: 

3853 raise ValueError( 

3854 "ignore_flush argument is for non-text write mode only" 

3855 ) 

3856 return TextIOWrapper( 

3857 BlobReader(self, chunk_size=chunk_size, **kwargs), 

3858 encoding=encoding, 

3859 errors=errors, 

3860 newline=newline, 

3861 ) 

3862 elif mode in ("w", "wt"): 

3863 if ignore_flush is False: 

3864 raise ValueError( 

3865 "ignore_flush is required for text mode writing and " 

3866 "cannot be set to False" 

3867 ) 

3868 return TextIOWrapper( 

3869 BlobWriter(self, chunk_size=chunk_size, ignore_flush=True, **kwargs), 

3870 encoding=encoding, 

3871 errors=errors, 

3872 newline=newline, 

3873 ) 

3874 else: 

3875 raise NotImplementedError( 

3876 "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only." 

3877 ) 

3878 

3879 cache_control = _scalar_property("cacheControl") 

3880 """HTTP 'Cache-Control' header for this object. 

3881 

3882 See [`RFC 7234`](https://tools.ietf.org/html/rfc7234#section-5.2) 

3883 and [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3884 

3885 :rtype: str or ``NoneType`` 

3886 

3887 """ 

3888 

3889 content_disposition = _scalar_property("contentDisposition") 

3890 """HTTP 'Content-Disposition' header for this object. 

3891 

3892 See [`RFC 6266`](https://tools.ietf.org/html/rfc7234#section-5.2) and 

3893 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3894 

3895 :rtype: str or ``NoneType`` 

3896 """ 

3897 

3898 content_encoding = _scalar_property("contentEncoding") 

3899 """HTTP 'Content-Encoding' header for this object. 

3900 

3901 See [`RFC 7231`](https://tools.ietf.org/html/rfc7231#section-3.1.2.2) and 

3902 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3903 

3904 :rtype: str or ``NoneType`` 

3905 """ 

3906 

3907 content_language = _scalar_property("contentLanguage") 

3908 """HTTP 'Content-Language' header for this object. 

3909 

3910 See [`BCP47`](https://tools.ietf.org/html/bcp47) and 

3911 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3912 

3913 :rtype: str or ``NoneType`` 

3914 """ 

3915 

3916 content_type = _scalar_property(_CONTENT_TYPE_FIELD) 

3917 """HTTP 'Content-Type' header for this object. 

3918 

3919 See [`RFC 2616`](https://tools.ietf.org/html/rfc2616#section-14.17) and 

3920 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3921 

3922 :rtype: str or ``NoneType`` 

3923 """ 

3924 

3925 crc32c = _scalar_property("crc32c") 

3926 """CRC32C checksum for this object. 

3927 

3928 This returns the blob's CRC32C checksum. To retrieve the value, first use a 

3929 reload method of the Blob class which loads the blob's properties from the server. 

3930 

3931 See [`RFC 4960`](https://tools.ietf.org/html/rfc4960#appendix-B) and 

3932 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3933 

3934 If not set before upload, the server will compute the hash. 

3935 

3936 :rtype: str or ``NoneType`` 

3937 """ 

3938 

3939 @property 

3940 def component_count(self): 

3941 """Number of underlying components that make up this object. 

3942 

3943 See https://cloud.google.com/storage/docs/json_api/v1/objects 

3944 

3945 :rtype: int or ``NoneType`` 

3946 :returns: The component count (in case of a composed object) or 

3947 ``None`` if the blob's resource has not been loaded from 

3948 the server. This property will not be set on objects 

3949 not created via ``compose``. 

3950 """ 

3951 component_count = self._properties.get("componentCount") 

3952 if component_count is not None: 

3953 return int(component_count) 

3954 

3955 @property 

3956 def etag(self): 

3957 """Retrieve the ETag for the object. 

3958 

3959 See [`RFC 2616 (etags)`](https://tools.ietf.org/html/rfc2616#section-3.11) and 

3960 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3961 

3962 :rtype: str or ``NoneType`` 

3963 :returns: The blob etag or ``None`` if the blob's resource has not 

3964 been loaded from the server. 

3965 """ 

3966 return self._properties.get("etag") 

3967 

3968 event_based_hold = _scalar_property("eventBasedHold") 

3969 """Is an event-based hold active on the object? 

3970 

3971 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

3972 

3973 If the property is not set locally, returns :data:`None`. 

3974 

3975 :rtype: bool or ``NoneType`` 

3976 """ 

3977 

3978 @property 

3979 def generation(self): 

3980 """Retrieve the generation for the object. 

3981 

3982 See https://cloud.google.com/storage/docs/json_api/v1/objects 

3983 

3984 :rtype: int or ``NoneType`` 

3985 :returns: The generation of the blob or ``None`` if the blob's 

3986 resource has not been loaded from the server. 

3987 """ 

3988 generation = self._properties.get("generation") 

3989 if generation is not None: 

3990 return int(generation) 

3991 

3992 @property 

3993 def id(self): 

3994 """Retrieve the ID for the object. 

3995 

3996 See https://cloud.google.com/storage/docs/json_api/v1/objects 

3997 

3998 The ID consists of the bucket name, object name, and generation number. 

3999 

4000 :rtype: str or ``NoneType`` 

4001 :returns: The ID of the blob or ``None`` if the blob's 

4002 resource has not been loaded from the server. 

4003 """ 

4004 return self._properties.get("id") 

4005 

4006 md5_hash = _scalar_property("md5Hash") 

4007 """MD5 hash for this object. 

4008 

4009 This returns the blob's MD5 hash. To retrieve the value, first use a 

4010 reload method of the Blob class which loads the blob's properties from the server. 

4011 

4012 See [`RFC 1321`](https://tools.ietf.org/html/rfc1321) and 

4013 [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

4014 

4015 If not set before upload, the server will compute the hash. 

4016 

4017 :rtype: str or ``NoneType`` 

4018 """ 

4019 

4020 @property 

4021 def media_link(self): 

4022 """Retrieve the media download URI for the object. 

4023 

4024 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4025 

4026 :rtype: str or ``NoneType`` 

4027 :returns: The media link for the blob or ``None`` if the blob's 

4028 resource has not been loaded from the server. 

4029 """ 

4030 return self._properties.get("mediaLink") 

4031 

4032 @property 

4033 def metadata(self): 

4034 """Retrieve arbitrary/application specific metadata for the object. 

4035 

4036 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4037 

4038 :setter: Update arbitrary/application specific metadata for the 

4039 object. 

4040 :getter: Retrieve arbitrary/application specific metadata for 

4041 the object. 

4042 

4043 :rtype: dict or ``NoneType`` 

4044 :returns: The metadata associated with the blob or ``None`` if the 

4045 property is not set. 

4046 """ 

4047 return copy.deepcopy(self._properties.get("metadata")) 

4048 

4049 @metadata.setter 

4050 def metadata(self, value): 

4051 """Update arbitrary/application specific metadata for the object. 

4052 

4053 Values are stored to GCS as strings. To delete a key, set its value to 

4054 None and call blob.patch(). 

4055 

4056 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4057 

4058 :type value: dict 

4059 :param value: The blob metadata to set. 

4060 """ 

4061 if value is not None: 

4062 value = {k: str(v) if v is not None else None for k, v in value.items()} 

4063 self._patch_property("metadata", value) 

4064 

4065 @property 

4066 def metageneration(self): 

4067 """Retrieve the metageneration for the object. 

4068 

4069 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4070 

4071 :rtype: int or ``NoneType`` 

4072 :returns: The metageneration of the blob or ``None`` if the blob's 

4073 resource has not been loaded from the server. 

4074 """ 

4075 metageneration = self._properties.get("metageneration") 

4076 if metageneration is not None: 

4077 return int(metageneration) 

4078 

4079 @property 

4080 def owner(self): 

4081 """Retrieve info about the owner of the object. 

4082 

4083 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4084 

4085 :rtype: dict or ``NoneType`` 

4086 :returns: Mapping of owner's role/ID, or ``None`` if the blob's 

4087 resource has not been loaded from the server. 

4088 """ 

4089 return copy.deepcopy(self._properties.get("owner")) 

4090 

4091 @property 

4092 def retention_expiration_time(self): 

4093 """Retrieve timestamp at which the object's retention period expires. 

4094 

4095 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4096 

4097 :rtype: :class:`datetime.datetime` or ``NoneType`` 

4098 :returns: Datetime object parsed from RFC3339 valid timestamp, or 

4099 ``None`` if the property is not set locally. 

4100 """ 

4101 value = self._properties.get("retentionExpirationTime") 

4102 if value is not None: 

4103 return _rfc3339_nanos_to_datetime(value) 

4104 

4105 @property 

4106 def self_link(self): 

4107 """Retrieve the URI for the object. 

4108 

4109 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4110 

4111 :rtype: str or ``NoneType`` 

4112 :returns: The self link for the blob or ``None`` if the blob's 

4113 resource has not been loaded from the server. 

4114 """ 

4115 return self._properties.get("selfLink") 

4116 

4117 @property 

4118 def size(self): 

4119 """Size of the object, in bytes. 

4120 

4121 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4122 

4123 :rtype: int or ``NoneType`` 

4124 :returns: The size of the blob or ``None`` if the blob's 

4125 resource has not been loaded from the server. 

4126 """ 

4127 size = self._properties.get("size") 

4128 if size is not None: 

4129 return int(size) 

4130 

4131 @property 

4132 def kms_key_name(self): 

4133 """Resource name of Cloud KMS key used to encrypt the blob's contents. 

4134 

4135 :rtype: str or ``NoneType`` 

4136 :returns: 

4137 The resource name or ``None`` if no Cloud KMS key was used, 

4138 or the blob's resource has not been loaded from the server. 

4139 """ 

4140 return self._properties.get("kmsKeyName") 

4141 

4142 @kms_key_name.setter 

4143 def kms_key_name(self, value): 

4144 """Set KMS encryption key for object. 

4145 

4146 :type value: str or ``NoneType`` 

4147 :param value: new KMS key name (None to clear any existing key). 

4148 """ 

4149 self._patch_property("kmsKeyName", value) 

4150 

4151 storage_class = _scalar_property("storageClass") 

4152 """Retrieve the storage class for the object. 

4153 

4154 This can only be set at blob / object **creation** time. If you'd 

4155 like to change the storage class **after** the blob / object already 

4156 exists in a bucket, call :meth:`update_storage_class` (which uses 

4157 :meth:`rewrite`). 

4158 

4159 See https://cloud.google.com/storage/docs/storage-classes 

4160 

4161 :rtype: str or ``NoneType`` 

4162 :returns: 

4163 If set, one of 

4164 :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, 

4165 :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, 

4166 :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, 

4167 :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, 

4168 :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, 

4169 :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`, 

4170 :attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`, 

4171 else ``None``. 

4172 """ 

4173 

4174 temporary_hold = _scalar_property("temporaryHold") 

4175 """Is a temporary hold active on the object? 

4176 

4177 See [`API reference docs`](https://cloud.google.com/storage/docs/json_api/v1/objects). 

4178 

4179 If the property is not set locally, returns :data:`None`. 

4180 

4181 :rtype: bool or ``NoneType`` 

4182 """ 

4183 

4184 @property 

4185 def time_deleted(self): 

4186 """Retrieve the timestamp at which the object was deleted. 

4187 

4188 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4189 

4190 :rtype: :class:`datetime.datetime` or ``NoneType`` 

4191 :returns: Datetime object parsed from RFC3339 valid timestamp, or 

4192 ``None`` if the blob's resource has not been loaded from 

4193 the server (see :meth:`reload`). If the blob has 

4194 not been deleted, this will never be set. 

4195 """ 

4196 value = self._properties.get("timeDeleted") 

4197 if value is not None: 

4198 return _rfc3339_nanos_to_datetime(value) 

4199 

4200 @property 

4201 def time_created(self): 

4202 """Retrieve the timestamp at which the object was created. 

4203 

4204 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4205 

4206 :rtype: :class:`datetime.datetime` or ``NoneType`` 

4207 :returns: Datetime object parsed from RFC3339 valid timestamp, or 

4208 ``None`` if the blob's resource has not been loaded from 

4209 the server (see :meth:`reload`). 

4210 """ 

4211 value = self._properties.get("timeCreated") 

4212 if value is not None: 

4213 return _rfc3339_nanos_to_datetime(value) 

4214 

4215 @property 

4216 def updated(self): 

4217 """Retrieve the timestamp at which the object was updated. 

4218 

4219 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4220 

4221 :rtype: :class:`datetime.datetime` or ``NoneType`` 

4222 :returns: Datetime object parsed from RFC3339 valid timestamp, or 

4223 ``None`` if the blob's resource has not been loaded from 

4224 the server (see :meth:`reload`). 

4225 """ 

4226 value = self._properties.get("updated") 

4227 if value is not None: 

4228 return _rfc3339_nanos_to_datetime(value) 

4229 

4230 @property 

4231 def custom_time(self): 

4232 """Retrieve the custom time for the object. 

4233 

4234 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4235 

4236 :rtype: :class:`datetime.datetime` or ``NoneType`` 

4237 :returns: Datetime object parsed from RFC3339 valid timestamp, or 

4238 ``None`` if the blob's resource has not been loaded from 

4239 the server (see :meth:`reload`). 

4240 """ 

4241 value = self._properties.get("customTime") 

4242 if value is not None: 

4243 return _rfc3339_nanos_to_datetime(value) 

4244 

4245 @custom_time.setter 

4246 def custom_time(self, value): 

4247 """Set the custom time for the object. 

4248 

4249 Once set on the server side object, this value can't be unset, but may 

4250 only changed to a custom datetime in the future. 

4251 

4252 If :attr:`custom_time` must be unset, either perform a rewrite 

4253 operation or upload the data again. 

4254 

4255 See https://cloud.google.com/storage/docs/json_api/v1/objects 

4256 

4257 :type value: :class:`datetime.datetime` 

4258 :param value: new value 

4259 """ 

4260 if value is not None: 

4261 value = _datetime_to_rfc3339(value) 

4262 

4263 self._patch_property("customTime", value) 

4264 

4265 

4266def _get_host_name(connection): 

4267 """Returns the host name from the given connection. 

4268 

4269 :type connection: :class:`~google.cloud.storage._http.Connection` 

4270 :param connection: The connection object. 

4271 

4272 :rtype: str 

4273 :returns: The host name. 

4274 """ 

4275 # TODO: After google-cloud-core 1.6.0 is stable and we upgrade it 

4276 # to 1.6.0 in setup.py, we no longer need to check the attribute 

4277 # existence. We can simply return connection.get_api_base_url_for_mtls(). 

4278 return ( 

4279 connection.API_BASE_URL 

4280 if not hasattr(connection, "get_api_base_url_for_mtls") 

4281 else connection.get_api_base_url_for_mtls() 

4282 ) 

4283 

4284 

4285def _get_encryption_headers(key, source=False): 

4286 """Builds customer encryption key headers 

4287 

4288 :type key: bytes 

4289 :param key: 32 byte key to build request key and hash. 

4290 

4291 :type source: bool 

4292 :param source: If true, return headers for the "source" blob; otherwise, 

4293 return headers for the "destination" blob. 

4294 

4295 :rtype: dict 

4296 :returns: dict of HTTP headers being sent in request. 

4297 """ 

4298 if key is None: 

4299 return {} 

4300 

4301 key = _to_bytes(key) 

4302 key_hash = hashlib.sha256(key).digest() 

4303 key_hash = base64.b64encode(key_hash) 

4304 key = base64.b64encode(key) 

4305 

4306 if source: 

4307 prefix = "X-Goog-Copy-Source-Encryption-" 

4308 else: 

4309 prefix = "X-Goog-Encryption-" 

4310 

4311 return { 

4312 prefix + "Algorithm": "AES256", 

4313 prefix + "Key": _bytes_to_unicode(key), 

4314 prefix + "Key-Sha256": _bytes_to_unicode(key_hash), 

4315 } 

4316 

4317 

4318def _quote(value, safe=b"~"): 

4319 """URL-quote a string. 

4320 

4321 If the value is unicode, this method first UTF-8 encodes it as bytes and 

4322 then quotes the bytes. (In Python 3, ``urllib.parse.quote`` does this 

4323 encoding automatically, but in Python 2, non-ASCII characters cannot be 

4324 quoted.) 

4325 

4326 :type value: str or bytes 

4327 :param value: The value to be URL-quoted. 

4328 

4329 :type safe: bytes 

4330 :param safe: Bytes *not* to be quoted. By default, includes only ``b'~'``. 

4331 

4332 :rtype: str 

4333 :returns: The encoded value (bytes in Python 2, unicode in Python 3). 

4334 """ 

4335 value = _to_bytes(value, encoding="utf-8") 

4336 return quote(value, safe=safe) 

4337 

4338 

4339def _maybe_rewind(stream, rewind=False): 

4340 """Rewind the stream if desired. 

4341 

4342 :type stream: IO[bytes] 

4343 :param stream: A bytes IO object open for reading. 

4344 

4345 :type rewind: bool 

4346 :param rewind: Indicates if we should seek to the beginning of the stream. 

4347 """ 

4348 if rewind: 

4349 stream.seek(0, os.SEEK_SET) 

4350 

4351 

4352def _raise_from_invalid_response(error): 

4353 """Re-wrap and raise an ``InvalidResponse`` exception. 

4354 

4355 :type error: :exc:`google.resumable_media.InvalidResponse` 

4356 :param error: A caught exception from the ``google-resumable-media`` 

4357 library. 

4358 

4359 :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding 

4360 to the failed status code 

4361 """ 

4362 response = error.response 

4363 

4364 # The 'response.text' gives the actual reason of error, where 'error' gives 

4365 # the message of expected status code. 

4366 if response.text: 

4367 error_message = response.text + ": " + str(error) 

4368 else: 

4369 error_message = str(error) 

4370 

4371 message = f"{response.request.method} {response.request.url}: {error_message}" 

4372 

4373 raise exceptions.from_http_status(response.status_code, message, response=response) 

4374 

4375 

4376def _add_query_parameters(base_url, name_value_pairs): 

4377 """Add one query parameter to a base URL. 

4378 

4379 :type base_url: string 

4380 :param base_url: Base URL (may already contain query parameters) 

4381 

4382 :type name_value_pairs: list of (string, string) tuples. 

4383 :param name_value_pairs: Names and values of the query parameters to add 

4384 

4385 :rtype: string 

4386 :returns: URL with additional query strings appended. 

4387 """ 

4388 if len(name_value_pairs) == 0: 

4389 return base_url 

4390 

4391 scheme, netloc, path, query, frag = urlsplit(base_url) 

4392 query = parse_qsl(query) 

4393 query.extend(name_value_pairs) 

4394 return urlunsplit((scheme, netloc, path, urlencode(query), frag))