Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/client.py: 23%

290 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 07:13 +0000

1# Copyright 2015 Google LLC 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15"""Client for interacting with the Google Cloud Storage API.""" 

16 

17import base64 

18import binascii 

19import collections 

20import datetime 

21import functools 

22import json 

23import warnings 

24import google.api_core.client_options 

25 

26from google.auth.credentials import AnonymousCredentials 

27 

28from google import resumable_media 

29 

30from google.api_core import page_iterator 

31from google.cloud._helpers import _LocalStack, _NOW 

32from google.cloud.client import ClientWithProject 

33from google.cloud.exceptions import NotFound 

34from google.cloud.storage._helpers import _get_default_headers 

35from google.cloud.storage._helpers import _get_environ_project 

36from google.cloud.storage._helpers import _get_storage_host 

37from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST 

38from google.cloud.storage._helpers import _bucket_bound_hostname_url 

39from google.cloud.storage._helpers import _add_etag_match_headers 

40from google.cloud.storage._http import Connection 

41from google.cloud.storage._signing import ( 

42 get_expiration_seconds_v4, 

43 get_v4_now_dtstamps, 

44 ensure_signed_credentials, 

45 _sign_message, 

46) 

47from google.cloud.storage.batch import Batch 

48from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start 

49from google.cloud.storage.blob import ( 

50 Blob, 

51 _get_encryption_headers, 

52 _raise_from_invalid_response, 

53) 

54from google.cloud.storage.hmac_key import HMACKeyMetadata 

55from google.cloud.storage.acl import BucketACL 

56from google.cloud.storage.acl import DefaultObjectACL 

57from google.cloud.storage.constants import _DEFAULT_TIMEOUT 

58from google.cloud.storage.retry import DEFAULT_RETRY 

59from google.cloud.storage.retry import ConditionalRetryPolicy 

60 

61 

62_marker = object() 

63 

64 

65class Client(ClientWithProject): 

66 """Client to bundle configuration needed for API requests. 

67 

68 :type project: str or None 

69 :param project: the project which the client acts on behalf of. Will be 

70 passed when creating a topic. If not passed, 

71 falls back to the default inferred from the environment. 

72 

73 :type credentials: :class:`~google.auth.credentials.Credentials` 

74 :param credentials: (Optional) The OAuth2 Credentials to use for this 

75 client. If not passed (and if no ``_http`` object is 

76 passed), falls back to the default inferred from the 

77 environment. 

78 

79 :type _http: :class:`~requests.Session` 

80 :param _http: (Optional) HTTP object to make requests. Can be any object 

81 that defines ``request()`` with the same interface as 

82 :meth:`requests.Session.request`. If not passed, an 

83 ``_http`` object is created that is bound to the 

84 ``credentials`` for the current object. 

85 This parameter should be considered private, and could 

86 change in the future. 

87 

88 :type client_info: :class:`~google.api_core.client_info.ClientInfo` 

89 :param client_info: 

90 The client info used to send a user-agent string along with API 

91 requests. If ``None``, then default info will be used. Generally, 

92 you only need to set this if you're developing your own library 

93 or partner tool. 

94 

95 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict` 

96 :param client_options: (Optional) Client options used to set user options on the client. 

97 API Endpoint should be set through client_options. 

98 

99 :type use_auth_w_custom_endpoint: bool 

100 :param use_auth_w_custom_endpoint: 

101 (Optional) Whether authentication is required under custom endpoints. 

102 If false, uses AnonymousCredentials and bypasses authentication. 

103 Defaults to True. Note this is only used when a custom endpoint is set in conjunction. 

104 """ 

105 

106 SCOPE = ( 

107 "https://www.googleapis.com/auth/devstorage.full_control", 

108 "https://www.googleapis.com/auth/devstorage.read_only", 

109 "https://www.googleapis.com/auth/devstorage.read_write", 

110 ) 

111 """The scopes required for authenticating as a Cloud Storage consumer.""" 

112 

113 def __init__( 

114 self, 

115 project=_marker, 

116 credentials=None, 

117 _http=None, 

118 client_info=None, 

119 client_options=None, 

120 use_auth_w_custom_endpoint=True, 

121 ): 

122 self._base_connection = None 

123 

124 if project is None: 

125 no_project = True 

126 project = "<none>" 

127 else: 

128 no_project = False 

129 

130 if project is _marker: 

131 project = None 

132 

133 # Save the initial value of constructor arguments before they 

134 # are passed along, for use in __reduce__ defined elsewhere. 

135 self._initial_client_info = client_info 

136 self._initial_client_options = client_options 

137 self._initial_credentials = credentials 

138 

139 kw_args = {"client_info": client_info} 

140 

141 # `api_endpoint` should be only set by the user via `client_options`, 

142 # or if the _get_storage_host() returns a non-default value (_is_emulator_set). 

143 # `api_endpoint` plays an important role for mTLS, if it is not set, 

144 # then mTLS logic will be applied to decide which endpoint will be used. 

145 storage_host = _get_storage_host() 

146 _is_emulator_set = storage_host != _DEFAULT_STORAGE_HOST 

147 kw_args["api_endpoint"] = storage_host if _is_emulator_set else None 

148 

149 if client_options: 

150 if type(client_options) == dict: 

151 client_options = google.api_core.client_options.from_dict( 

152 client_options 

153 ) 

154 if client_options.api_endpoint: 

155 api_endpoint = client_options.api_endpoint 

156 kw_args["api_endpoint"] = api_endpoint 

157 

158 # If a custom endpoint is set, the client checks for credentials 

159 # or finds the default credentials based on the current environment. 

160 # Authentication may be bypassed under certain conditions: 

161 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR 

162 # (2) use_auth_w_custom_endpoint is set to False. 

163 if kw_args["api_endpoint"] is not None: 

164 if _is_emulator_set or not use_auth_w_custom_endpoint: 

165 if credentials is None: 

166 credentials = AnonymousCredentials() 

167 if project is None: 

168 project = _get_environ_project() 

169 if project is None: 

170 no_project = True 

171 project = "<none>" 

172 

173 super(Client, self).__init__( 

174 project=project, 

175 credentials=credentials, 

176 client_options=client_options, 

177 _http=_http, 

178 ) 

179 

180 if no_project: 

181 self.project = None 

182 

183 self._connection = Connection(self, **kw_args) 

184 self._batch_stack = _LocalStack() 

185 

186 @classmethod 

187 def create_anonymous_client(cls): 

188 """Factory: return client with anonymous credentials. 

189 

190 .. note:: 

191 

192 Such a client has only limited access to "public" buckets: 

193 listing their contents and downloading their blobs. 

194 

195 :rtype: :class:`google.cloud.storage.client.Client` 

196 :returns: Instance w/ anonymous credentials and no project. 

197 """ 

198 client = cls(project="<none>", credentials=AnonymousCredentials()) 

199 client.project = None 

200 return client 

201 

202 @property 

203 def _connection(self): 

204 """Get connection or batch on the client. 

205 

206 :rtype: :class:`google.cloud.storage._http.Connection` 

207 :returns: The connection set on the client, or the batch 

208 if one is set. 

209 """ 

210 if self.current_batch is not None: 

211 return self.current_batch 

212 else: 

213 return self._base_connection 

214 

215 @_connection.setter 

216 def _connection(self, value): 

217 """Set connection on the client. 

218 

219 Intended to be used by constructor (since the base class calls) 

220 self._connection = connection 

221 Will raise if the connection is set more than once. 

222 

223 :type value: :class:`google.cloud.storage._http.Connection` 

224 :param value: The connection set on the client. 

225 

226 :raises: :class:`ValueError` if connection has already been set. 

227 """ 

228 if self._base_connection is not None: 

229 raise ValueError("Connection already set on client") 

230 self._base_connection = value 

231 

232 def _push_batch(self, batch): 

233 """Push a batch onto our stack. 

234 

235 "Protected", intended for use by batch context mgrs. 

236 

237 :type batch: :class:`google.cloud.storage.batch.Batch` 

238 :param batch: newly-active batch 

239 """ 

240 self._batch_stack.push(batch) 

241 

242 def _pop_batch(self): 

243 """Pop a batch from our stack. 

244 

245 "Protected", intended for use by batch context mgrs. 

246 

247 :raises: IndexError if the stack is empty. 

248 :rtype: :class:`google.cloud.storage.batch.Batch` 

249 :returns: the top-most batch/transaction, after removing it. 

250 """ 

251 return self._batch_stack.pop() 

252 

253 @property 

254 def current_batch(self): 

255 """Currently-active batch. 

256 

257 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if 

258 no batch is active). 

259 :returns: The batch at the top of the batch stack. 

260 """ 

261 return self._batch_stack.top 

262 

263 def get_service_account_email( 

264 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY 

265 ): 

266 """Get the email address of the project's GCS service account 

267 

268 :type project: str 

269 :param project: 

270 (Optional) Project ID to use for retreiving GCS service account 

271 email address. Defaults to the client's project. 

272 :type timeout: float or tuple 

273 :param timeout: 

274 (Optional) The amount of time, in seconds, to wait 

275 for the server response. See: :ref:`configuring_timeouts` 

276 

277 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

278 :param retry: 

279 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

280 

281 :rtype: str 

282 :returns: service account email address 

283 """ 

284 if project is None: 

285 project = self.project 

286 

287 path = f"/projects/{project}/serviceAccount" 

288 api_response = self._get_resource(path, timeout=timeout, retry=retry) 

289 return api_response["email_address"] 

290 

291 def bucket(self, bucket_name, user_project=None): 

292 """Factory constructor for bucket object. 

293 

294 .. note:: 

295 This will not make an HTTP request; it simply instantiates 

296 a bucket object owned by this client. 

297 

298 :type bucket_name: str 

299 :param bucket_name: The name of the bucket to be instantiated. 

300 

301 :type user_project: str 

302 :param user_project: (Optional) The project ID to be billed for API 

303 requests made via the bucket. 

304 

305 :rtype: :class:`google.cloud.storage.bucket.Bucket` 

306 :returns: The bucket object created. 

307 """ 

308 return Bucket(client=self, name=bucket_name, user_project=user_project) 

309 

310 def batch(self, raise_exception=True): 

311 """Factory constructor for batch object. 

312 

313 .. note:: 

314 This will not make an HTTP request; it simply instantiates 

315 a batch object owned by this client. 

316 

317 :type raise_exception: bool 

318 :param raise_exception: 

319 (Optional) Defaults to True. If True, instead of adding exceptions 

320 to the list of return responses, the final exception will be raised. 

321 Note that exceptions are unwrapped after all operations are complete 

322 in success or failure, and only the last exception is raised. 

323 

324 :rtype: :class:`google.cloud.storage.batch.Batch` 

325 :returns: The batch object created. 

326 """ 

327 return Batch(client=self, raise_exception=raise_exception) 

328 

329 def _get_resource( 

330 self, 

331 path, 

332 query_params=None, 

333 headers=None, 

334 timeout=_DEFAULT_TIMEOUT, 

335 retry=DEFAULT_RETRY, 

336 _target_object=None, 

337 ): 

338 """Helper for bucket / blob methods making API 'GET' calls. 

339 

340 Args: 

341 path str: 

342 The path of the resource to fetch. 

343 

344 query_params Optional[dict]: 

345 HTTP query parameters to be passed 

346 

347 headers Optional[dict]: 

348 HTTP headers to be passed 

349 

350 timeout (Optional[Union[float, Tuple[float, float]]]): 

351 The amount of time, in seconds, to wait for the server response. 

352 

353 Can also be passed as a tuple (connect_timeout, read_timeout). 

354 See :meth:`requests.Session.request` documentation for details. 

355 

356 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

357 How to retry the RPC. A None value will disable retries. 

358 A google.api_core.retry.Retry value will enable retries, and the object will 

359 define retriable response codes and errors and configure backoff and timeout options. 

360 

361 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

362 activates it only if certain conditions are met. This class exists to provide safe defaults 

363 for RPC calls that are not technically safe to retry normally (due to potential data 

364 duplication or other side-effects) but become safe to retry if a condition such as 

365 if_metageneration_match is set. 

366 

367 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

368 information on retry types and how to configure them. 

369 

370 _target_object (Union[ \ 

371 :class:`~google.cloud.storage.bucket.Bucket`, \ 

372 :class:`~google.cloud.storage.bucket.blob`, \ 

373 ]): 

374 Object to which future data is to be applied -- only relevant 

375 in the context of a batch. 

376 

377 Returns: 

378 dict 

379 The JSON resource fetched 

380 

381 Raises: 

382 google.cloud.exceptions.NotFound 

383 If the bucket is not found. 

384 """ 

385 return self._connection.api_request( 

386 method="GET", 

387 path=path, 

388 query_params=query_params, 

389 headers=headers, 

390 timeout=timeout, 

391 retry=retry, 

392 _target_object=_target_object, 

393 ) 

394 

395 def _list_resource( 

396 self, 

397 path, 

398 item_to_value, 

399 page_token=None, 

400 max_results=None, 

401 extra_params=None, 

402 page_start=page_iterator._do_nothing_page_start, 

403 page_size=None, 

404 timeout=_DEFAULT_TIMEOUT, 

405 retry=DEFAULT_RETRY, 

406 ): 

407 api_request = functools.partial( 

408 self._connection.api_request, timeout=timeout, retry=retry 

409 ) 

410 return page_iterator.HTTPIterator( 

411 client=self, 

412 api_request=api_request, 

413 path=path, 

414 item_to_value=item_to_value, 

415 page_token=page_token, 

416 max_results=max_results, 

417 extra_params=extra_params, 

418 page_start=page_start, 

419 page_size=page_size, 

420 ) 

421 

422 def _patch_resource( 

423 self, 

424 path, 

425 data, 

426 query_params=None, 

427 headers=None, 

428 timeout=_DEFAULT_TIMEOUT, 

429 retry=None, 

430 _target_object=None, 

431 ): 

432 """Helper for bucket / blob methods making API 'PATCH' calls. 

433 

434 Args: 

435 path str: 

436 The path of the resource to fetch. 

437 

438 data dict: 

439 The data to be patched. 

440 

441 query_params Optional[dict]: 

442 HTTP query parameters to be passed 

443 

444 headers Optional[dict]: 

445 HTTP headers to be passed 

446 

447 timeout (Optional[Union[float, Tuple[float, float]]]): 

448 The amount of time, in seconds, to wait for the server response. 

449 

450 Can also be passed as a tuple (connect_timeout, read_timeout). 

451 See :meth:`requests.Session.request` documentation for details. 

452 

453 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

454 How to retry the RPC. A None value will disable retries. 

455 A google.api_core.retry.Retry value will enable retries, and the object will 

456 define retriable response codes and errors and configure backoff and timeout options. 

457 

458 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

459 activates it only if certain conditions are met. This class exists to provide safe defaults 

460 for RPC calls that are not technically safe to retry normally (due to potential data 

461 duplication or other side-effects) but become safe to retry if a condition such as 

462 if_metageneration_match is set. 

463 

464 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

465 information on retry types and how to configure them. 

466 

467 _target_object (Union[ \ 

468 :class:`~google.cloud.storage.bucket.Bucket`, \ 

469 :class:`~google.cloud.storage.bucket.blob`, \ 

470 ]): 

471 Object to which future data is to be applied -- only relevant 

472 in the context of a batch. 

473 

474 Returns: 

475 dict 

476 The JSON resource fetched 

477 

478 Raises: 

479 google.cloud.exceptions.NotFound 

480 If the bucket is not found. 

481 """ 

482 return self._connection.api_request( 

483 method="PATCH", 

484 path=path, 

485 data=data, 

486 query_params=query_params, 

487 headers=headers, 

488 timeout=timeout, 

489 retry=retry, 

490 _target_object=_target_object, 

491 ) 

492 

493 def _put_resource( 

494 self, 

495 path, 

496 data, 

497 query_params=None, 

498 headers=None, 

499 timeout=_DEFAULT_TIMEOUT, 

500 retry=None, 

501 _target_object=None, 

502 ): 

503 """Helper for bucket / blob methods making API 'PUT' calls. 

504 

505 Args: 

506 path str: 

507 The path of the resource to fetch. 

508 

509 data dict: 

510 The data to be patched. 

511 

512 query_params Optional[dict]: 

513 HTTP query parameters to be passed 

514 

515 headers Optional[dict]: 

516 HTTP headers to be passed 

517 

518 timeout (Optional[Union[float, Tuple[float, float]]]): 

519 The amount of time, in seconds, to wait for the server response. 

520 

521 Can also be passed as a tuple (connect_timeout, read_timeout). 

522 See :meth:`requests.Session.request` documentation for details. 

523 

524 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

525 How to retry the RPC. A None value will disable retries. 

526 A google.api_core.retry.Retry value will enable retries, and the object will 

527 define retriable response codes and errors and configure backoff and timeout options. 

528 

529 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

530 activates it only if certain conditions are met. This class exists to provide safe defaults 

531 for RPC calls that are not technically safe to retry normally (due to potential data 

532 duplication or other side-effects) but become safe to retry if a condition such as 

533 if_metageneration_match is set. 

534 

535 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

536 information on retry types and how to configure them. 

537 

538 _target_object (Union[ \ 

539 :class:`~google.cloud.storage.bucket.Bucket`, \ 

540 :class:`~google.cloud.storage.bucket.blob`, \ 

541 ]): 

542 Object to which future data is to be applied -- only relevant 

543 in the context of a batch. 

544 

545 Returns: 

546 dict 

547 The JSON resource fetched 

548 

549 Raises: 

550 google.cloud.exceptions.NotFound 

551 If the bucket is not found. 

552 """ 

553 return self._connection.api_request( 

554 method="PUT", 

555 path=path, 

556 data=data, 

557 query_params=query_params, 

558 headers=headers, 

559 timeout=timeout, 

560 retry=retry, 

561 _target_object=_target_object, 

562 ) 

563 

564 def _post_resource( 

565 self, 

566 path, 

567 data, 

568 query_params=None, 

569 headers=None, 

570 timeout=_DEFAULT_TIMEOUT, 

571 retry=None, 

572 _target_object=None, 

573 ): 

574 """Helper for bucket / blob methods making API 'POST' calls. 

575 

576 Args: 

577 path str: 

578 The path of the resource to which to post. 

579 

580 data dict: 

581 The data to be posted. 

582 

583 query_params Optional[dict]: 

584 HTTP query parameters to be passed 

585 

586 headers Optional[dict]: 

587 HTTP headers to be passed 

588 

589 timeout (Optional[Union[float, Tuple[float, float]]]): 

590 The amount of time, in seconds, to wait for the server response. 

591 

592 Can also be passed as a tuple (connect_timeout, read_timeout). 

593 See :meth:`requests.Session.request` documentation for details. 

594 

595 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

596 How to retry the RPC. A None value will disable retries. 

597 A google.api_core.retry.Retry value will enable retries, and the object will 

598 define retriable response codes and errors and configure backoff and timeout options. 

599 

600 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

601 activates it only if certain conditions are met. This class exists to provide safe defaults 

602 for RPC calls that are not technically safe to retry normally (due to potential data 

603 duplication or other side-effects) but become safe to retry if a condition such as 

604 if_metageneration_match is set. 

605 

606 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

607 information on retry types and how to configure them. 

608 

609 _target_object (Union[ \ 

610 :class:`~google.cloud.storage.bucket.Bucket`, \ 

611 :class:`~google.cloud.storage.bucket.blob`, \ 

612 ]): 

613 Object to which future data is to be applied -- only relevant 

614 in the context of a batch. 

615 

616 Returns: 

617 dict 

618 The JSON resource returned from the post. 

619 

620 Raises: 

621 google.cloud.exceptions.NotFound 

622 If the bucket is not found. 

623 """ 

624 

625 return self._connection.api_request( 

626 method="POST", 

627 path=path, 

628 data=data, 

629 query_params=query_params, 

630 headers=headers, 

631 timeout=timeout, 

632 retry=retry, 

633 _target_object=_target_object, 

634 ) 

635 

636 def _delete_resource( 

637 self, 

638 path, 

639 query_params=None, 

640 headers=None, 

641 timeout=_DEFAULT_TIMEOUT, 

642 retry=DEFAULT_RETRY, 

643 _target_object=None, 

644 ): 

645 """Helper for bucket / blob methods making API 'DELETE' calls. 

646 

647 Args: 

648 path str: 

649 The path of the resource to delete. 

650 

651 query_params Optional[dict]: 

652 HTTP query parameters to be passed 

653 

654 headers Optional[dict]: 

655 HTTP headers to be passed 

656 

657 timeout (Optional[Union[float, Tuple[float, float]]]): 

658 The amount of time, in seconds, to wait for the server response. 

659 

660 Can also be passed as a tuple (connect_timeout, read_timeout). 

661 See :meth:`requests.Session.request` documentation for details. 

662 

663 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

664 How to retry the RPC. A None value will disable retries. 

665 A google.api_core.retry.Retry value will enable retries, and the object will 

666 define retriable response codes and errors and configure backoff and timeout options. 

667 

668 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

669 activates it only if certain conditions are met. This class exists to provide safe defaults 

670 for RPC calls that are not technically safe to retry normally (due to potential data 

671 duplication or other side-effects) but become safe to retry if a condition such as 

672 if_metageneration_match is set. 

673 

674 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

675 information on retry types and how to configure them. 

676 

677 _target_object (Union[ \ 

678 :class:`~google.cloud.storage.bucket.Bucket`, \ 

679 :class:`~google.cloud.storage.bucket.blob`, \ 

680 ]): 

681 Object to which future data is to be applied -- only relevant 

682 in the context of a batch. 

683 

684 Returns: 

685 dict 

686 The JSON resource fetched 

687 

688 Raises: 

689 google.cloud.exceptions.NotFound 

690 If the bucket is not found. 

691 """ 

692 return self._connection.api_request( 

693 method="DELETE", 

694 path=path, 

695 query_params=query_params, 

696 headers=headers, 

697 timeout=timeout, 

698 retry=retry, 

699 _target_object=_target_object, 

700 ) 

701 

702 def _bucket_arg_to_bucket(self, bucket_or_name): 

703 """Helper to return given bucket or create new by name. 

704 

705 Args: 

706 bucket_or_name (Union[ \ 

707 :class:`~google.cloud.storage.bucket.Bucket`, \ 

708 str, \ 

709 ]): 

710 The bucket resource to pass or name to create. 

711 

712 Returns: 

713 google.cloud.storage.bucket.Bucket 

714 The newly created bucket or the given one. 

715 """ 

716 if isinstance(bucket_or_name, Bucket): 

717 bucket = bucket_or_name 

718 if bucket.client is None: 

719 bucket._client = self 

720 else: 

721 bucket = Bucket(self, name=bucket_or_name) 

722 return bucket 

723 

724 def get_bucket( 

725 self, 

726 bucket_or_name, 

727 timeout=_DEFAULT_TIMEOUT, 

728 if_metageneration_match=None, 

729 if_metageneration_not_match=None, 

730 retry=DEFAULT_RETRY, 

731 ): 

732 """Retrieve a bucket via a GET request. 

733 

734 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python). 

735 

736 Args: 

737 bucket_or_name (Union[ \ 

738 :class:`~google.cloud.storage.bucket.Bucket`, \ 

739 str, \ 

740 ]): 

741 The bucket resource to pass or name to create. 

742 

743 timeout (Optional[Union[float, Tuple[float, float]]]): 

744 The amount of time, in seconds, to wait for the server response. 

745 

746 Can also be passed as a tuple (connect_timeout, read_timeout). 

747 See :meth:`requests.Session.request` documentation for details. 

748 

749 if_metageneration_match (Optional[long]): 

750 Make the operation conditional on whether the 

751 blob's current metageneration matches the given value. 

752 

753 if_metageneration_not_match (Optional[long]): 

754 Make the operation conditional on whether the blob's 

755 current metageneration does not match the given value. 

756 

757 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

758 How to retry the RPC. A None value will disable retries. 

759 A google.api_core.retry.Retry value will enable retries, and the object will 

760 define retriable response codes and errors and configure backoff and timeout options. 

761 

762 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

763 activates it only if certain conditions are met. This class exists to provide safe defaults 

764 for RPC calls that are not technically safe to retry normally (due to potential data 

765 duplication or other side-effects) but become safe to retry if a condition such as 

766 if_metageneration_match is set. 

767 

768 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

769 information on retry types and how to configure them. 

770 

771 Returns: 

772 google.cloud.storage.bucket.Bucket 

773 The bucket matching the name provided. 

774 

775 Raises: 

776 google.cloud.exceptions.NotFound 

777 If the bucket is not found. 

778 """ 

779 bucket = self._bucket_arg_to_bucket(bucket_or_name) 

780 bucket.reload( 

781 client=self, 

782 timeout=timeout, 

783 if_metageneration_match=if_metageneration_match, 

784 if_metageneration_not_match=if_metageneration_not_match, 

785 retry=retry, 

786 ) 

787 return bucket 

788 

789 def lookup_bucket( 

790 self, 

791 bucket_name, 

792 timeout=_DEFAULT_TIMEOUT, 

793 if_metageneration_match=None, 

794 if_metageneration_not_match=None, 

795 retry=DEFAULT_RETRY, 

796 ): 

797 """Get a bucket by name, returning None if not found. 

798 

799 You can use this if you would rather check for a None value 

800 than catching a NotFound exception. 

801 

802 :type bucket_name: str 

803 :param bucket_name: The name of the bucket to get. 

804 

805 :type timeout: float or tuple 

806 :param timeout: 

807 (Optional) The amount of time, in seconds, to wait 

808 for the server response. See: :ref:`configuring_timeouts` 

809 

810 :type if_metageneration_match: long 

811 :param if_metageneration_match: (Optional) Make the operation conditional on whether the 

812 blob's current metageneration matches the given value. 

813 

814 :type if_metageneration_not_match: long 

815 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the 

816 blob's current metageneration does not match the given value. 

817 

818 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

819 :param retry: 

820 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

821 

822 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType`` 

823 :returns: The bucket matching the name provided or None if not found. 

824 """ 

825 try: 

826 return self.get_bucket( 

827 bucket_name, 

828 timeout=timeout, 

829 if_metageneration_match=if_metageneration_match, 

830 if_metageneration_not_match=if_metageneration_not_match, 

831 retry=retry, 

832 ) 

833 except NotFound: 

834 return None 

835 

836 def create_bucket( 

837 self, 

838 bucket_or_name, 

839 requester_pays=None, 

840 project=None, 

841 user_project=None, 

842 location=None, 

843 data_locations=None, 

844 predefined_acl=None, 

845 predefined_default_object_acl=None, 

846 timeout=_DEFAULT_TIMEOUT, 

847 retry=DEFAULT_RETRY, 

848 ): 

849 """Create a new bucket via a POST request. 

850 

851 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python). 

852 

853 Args: 

854 bucket_or_name (Union[ \ 

855 :class:`~google.cloud.storage.bucket.Bucket`, \ 

856 str, \ 

857 ]): 

858 The bucket resource to pass or name to create. 

859 requester_pays (bool): 

860 DEPRECATED. Use Bucket().requester_pays instead. 

861 (Optional) Whether requester pays for API requests for 

862 this bucket and its blobs. 

863 project (str): 

864 (Optional) The project under which the bucket is to be created. 

865 If not passed, uses the project set on the client. 

866 user_project (str): 

867 (Optional) The project ID to be billed for API requests 

868 made via created bucket. 

869 location (str): 

870 (Optional) The location of the bucket. If not passed, 

871 the default location, US, will be used. If specifying a dual-region, 

872 `data_locations` should be set in conjunction. See: 

873 https://cloud.google.com/storage/docs/locations 

874 data_locations (list of str): 

875 (Optional) The list of regional locations of a custom dual-region bucket. 

876 Dual-regions require exactly 2 regional locations. See: 

877 https://cloud.google.com/storage/docs/locations 

878 predefined_acl (str): 

879 (Optional) Name of predefined ACL to apply to bucket. See: 

880 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl 

881 predefined_default_object_acl (str): 

882 (Optional) Name of predefined ACL to apply to bucket's objects. See: 

883 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl 

884 timeout (Optional[Union[float, Tuple[float, float]]]): 

885 The amount of time, in seconds, to wait for the server response. 

886 

887 Can also be passed as a tuple (connect_timeout, read_timeout). 

888 See :meth:`requests.Session.request` documentation for details. 

889 

890 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

891 How to retry the RPC. A None value will disable retries. 

892 A google.api_core.retry.Retry value will enable retries, and the object will 

893 define retriable response codes and errors and configure backoff and timeout options. 

894 

895 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

896 activates it only if certain conditions are met. This class exists to provide safe defaults 

897 for RPC calls that are not technically safe to retry normally (due to potential data 

898 duplication or other side-effects) but become safe to retry if a condition such as 

899 if_metageneration_match is set. 

900 

901 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

902 information on retry types and how to configure them. 

903 

904 Returns: 

905 google.cloud.storage.bucket.Bucket 

906 The newly created bucket. 

907 

908 Raises: 

909 google.cloud.exceptions.Conflict 

910 If the bucket already exists. 

911 """ 

912 bucket = self._bucket_arg_to_bucket(bucket_or_name) 

913 query_params = {} 

914 

915 if project is None: 

916 project = self.project 

917 

918 # Use no project if STORAGE_EMULATOR_HOST is set 

919 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST 

920 if _is_emulator_set: 

921 if project is None: 

922 project = _get_environ_project() 

923 if project is None: 

924 project = "<none>" 

925 

926 # Only include the project parameter if a project is set. 

927 # If a project is not set, falls back to API validation (BadRequest). 

928 if project is not None: 

929 query_params = {"project": project} 

930 

931 if requester_pays is not None: 

932 warnings.warn( 

933 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", 

934 PendingDeprecationWarning, 

935 stacklevel=1, 

936 ) 

937 bucket.requester_pays = requester_pays 

938 

939 if predefined_acl is not None: 

940 predefined_acl = BucketACL.validate_predefined(predefined_acl) 

941 query_params["predefinedAcl"] = predefined_acl 

942 

943 if predefined_default_object_acl is not None: 

944 predefined_default_object_acl = DefaultObjectACL.validate_predefined( 

945 predefined_default_object_acl 

946 ) 

947 query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl 

948 

949 if user_project is not None: 

950 query_params["userProject"] = user_project 

951 

952 properties = {key: bucket._properties[key] for key in bucket._changes} 

953 properties["name"] = bucket.name 

954 

955 if location is not None: 

956 properties["location"] = location 

957 

958 if data_locations is not None: 

959 properties["customPlacementConfig"] = {"dataLocations": data_locations} 

960 

961 api_response = self._post_resource( 

962 "/b", 

963 properties, 

964 query_params=query_params, 

965 timeout=timeout, 

966 retry=retry, 

967 _target_object=bucket, 

968 ) 

969 

970 bucket._set_properties(api_response) 

971 return bucket 

972 

973 def download_blob_to_file( 

974 self, 

975 blob_or_uri, 

976 file_obj, 

977 start=None, 

978 end=None, 

979 raw_download=False, 

980 if_etag_match=None, 

981 if_etag_not_match=None, 

982 if_generation_match=None, 

983 if_generation_not_match=None, 

984 if_metageneration_match=None, 

985 if_metageneration_not_match=None, 

986 timeout=_DEFAULT_TIMEOUT, 

987 checksum="md5", 

988 retry=DEFAULT_RETRY, 

989 ): 

990 """Download the contents of a blob object or blob URI into a file-like object. 

991 

992 See https://cloud.google.com/storage/docs/downloading-objects 

993 

994 Args: 

995 blob_or_uri (Union[ \ 

996 :class:`~google.cloud.storage.blob.Blob`, \ 

997 str, \ 

998 ]): 

999 The blob resource to pass or URI to download. 

1000 

1001 file_obj (file): 

1002 A file handle to which to write the blob's data. 

1003 

1004 start (int): 

1005 (Optional) The first byte in a range to be downloaded. 

1006 

1007 end (int): 

1008 (Optional) The last byte in a range to be downloaded. 

1009 

1010 raw_download (bool): 

1011 (Optional) If true, download the object without any expansion. 

1012 

1013 if_etag_match (Union[str, Set[str]]): 

1014 (Optional) See :ref:`using-if-etag-match` 

1015 

1016 if_etag_not_match (Union[str, Set[str]]): 

1017 (Optional) See :ref:`using-if-etag-not-match` 

1018 

1019 if_generation_match (long): 

1020 (Optional) See :ref:`using-if-generation-match` 

1021 

1022 if_generation_not_match (long): 

1023 (Optional) See :ref:`using-if-generation-not-match` 

1024 

1025 if_metageneration_match (long): 

1026 (Optional) See :ref:`using-if-metageneration-match` 

1027 

1028 if_metageneration_not_match (long): 

1029 (Optional) See :ref:`using-if-metageneration-not-match` 

1030 

1031 timeout ([Union[float, Tuple[float, float]]]): 

1032 (Optional) The amount of time, in seconds, to wait 

1033 for the server response. See: :ref:`configuring_timeouts` 

1034 

1035 checksum (str): 

1036 (Optional) The type of checksum to compute to verify the integrity 

1037 of the object. The response headers must contain a checksum of the 

1038 requested type. If the headers lack an appropriate checksum (for 

1039 instance in the case of transcoded or ranged downloads where the 

1040 remote service does not know the correct checksum, including 

1041 downloads where chunk_size is set) an INFO-level log will be 

1042 emitted. Supported values are "md5", "crc32c" and None. The default 

1043 is "md5". 

1044 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy) 

1045 (Optional) How to retry the RPC. A None value will disable 

1046 retries. A google.api_core.retry.Retry value will enable retries, 

1047 and the object will define retriable response codes and errors and 

1048 configure backoff and timeout options. 

1049 

1050 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 

1051 Retry object and activates it only if certain conditions are met. 

1052 This class exists to provide safe defaults for RPC calls that are 

1053 not technically safe to retry normally (due to potential data 

1054 duplication or other side-effects) but become safe to retry if a 

1055 condition such as if_metageneration_match is set. 

1056 

1057 See the retry.py source code and docstrings in this package 

1058 (google.cloud.storage.retry) for information on retry types and how 

1059 to configure them. 

1060 

1061 Media operations (downloads and uploads) do not support non-default 

1062 predicates in a Retry object. The default will always be used. Other 

1063 configuration changes for Retry objects such as delays and deadlines 

1064 are respected. 

1065 """ 

1066 

1067 # Handle ConditionalRetryPolicy. 

1068 if isinstance(retry, ConditionalRetryPolicy): 

1069 # Conditional retries are designed for non-media calls, which change 

1070 # arguments into query_params dictionaries. Media operations work 

1071 # differently, so here we make a "fake" query_params to feed to the 

1072 # ConditionalRetryPolicy. 

1073 query_params = { 

1074 "ifGenerationMatch": if_generation_match, 

1075 "ifMetagenerationMatch": if_metageneration_match, 

1076 } 

1077 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) 

1078 

1079 if not isinstance(blob_or_uri, Blob): 

1080 blob_or_uri = Blob.from_string(blob_or_uri) 

1081 download_url = blob_or_uri._get_download_url( 

1082 self, 

1083 if_generation_match=if_generation_match, 

1084 if_generation_not_match=if_generation_not_match, 

1085 if_metageneration_match=if_metageneration_match, 

1086 if_metageneration_not_match=if_metageneration_not_match, 

1087 ) 

1088 headers = _get_encryption_headers(blob_or_uri._encryption_key) 

1089 headers["accept-encoding"] = "gzip" 

1090 _add_etag_match_headers( 

1091 headers, 

1092 if_etag_match=if_etag_match, 

1093 if_etag_not_match=if_etag_not_match, 

1094 ) 

1095 headers = {**_get_default_headers(self._connection.user_agent), **headers} 

1096 

1097 transport = self._http 

1098 try: 

1099 blob_or_uri._do_download( 

1100 transport, 

1101 file_obj, 

1102 download_url, 

1103 headers, 

1104 start, 

1105 end, 

1106 raw_download, 

1107 timeout=timeout, 

1108 checksum=checksum, 

1109 retry=retry, 

1110 ) 

1111 except resumable_media.InvalidResponse as exc: 

1112 _raise_from_invalid_response(exc) 

1113 

1114 def list_blobs( 

1115 self, 

1116 bucket_or_name, 

1117 max_results=None, 

1118 page_token=None, 

1119 prefix=None, 

1120 delimiter=None, 

1121 start_offset=None, 

1122 end_offset=None, 

1123 include_trailing_delimiter=None, 

1124 versions=None, 

1125 projection="noAcl", 

1126 fields=None, 

1127 page_size=None, 

1128 timeout=_DEFAULT_TIMEOUT, 

1129 retry=DEFAULT_RETRY, 

1130 ): 

1131 """Return an iterator used to find blobs in the bucket. 

1132 

1133 If :attr:`user_project` is set, bills the API request to that project. 

1134 

1135 .. note:: 

1136 List prefixes (directories) in a bucket using a prefix and delimiter. 

1137 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python) 

1138 listing objects using a prefix filter. 

1139 

1140 Args: 

1141 bucket_or_name (Union[ \ 

1142 :class:`~google.cloud.storage.bucket.Bucket`, \ 

1143 str, \ 

1144 ]): 

1145 The bucket resource to pass or name to create. 

1146 

1147 max_results (int): 

1148 (Optional) The maximum number of blobs to return. 

1149 

1150 page_token (str): 

1151 (Optional) If present, return the next batch of blobs, using the 

1152 value, which must correspond to the ``nextPageToken`` value 

1153 returned in the previous response. Deprecated: use the ``pages`` 

1154 property of the returned iterator instead of manually passing the 

1155 token. 

1156 

1157 prefix (str): 

1158 (Optional) Prefix used to filter blobs. 

1159 

1160 delimiter (str): 

1161 (Optional) Delimiter, used with ``prefix`` to 

1162 emulate hierarchy. 

1163 

1164 start_offset (str): 

1165 (Optional) Filter results to objects whose names are 

1166 lexicographically equal to or after ``startOffset``. If 

1167 ``endOffset`` is also set, the objects listed will have names 

1168 between ``startOffset`` (inclusive) and ``endOffset`` 

1169 (exclusive). 

1170 

1171 end_offset (str): 

1172 (Optional) Filter results to objects whose names are 

1173 lexicographically before ``endOffset``. If ``startOffset`` is 

1174 also set, the objects listed will have names between 

1175 ``startOffset`` (inclusive) and ``endOffset`` (exclusive). 

1176 

1177 include_trailing_delimiter (boolean): 

1178 (Optional) If true, objects that end in exactly one instance of 

1179 ``delimiter`` will have their metadata included in ``items`` in 

1180 addition to ``prefixes``. 

1181 

1182 versions (bool): 

1183 (Optional) Whether object versions should be returned 

1184 as separate blobs. 

1185 

1186 projection (str): 

1187 (Optional) If used, must be 'full' or 'noAcl'. 

1188 Defaults to ``'noAcl'``. Specifies the set of 

1189 properties to return. 

1190 

1191 fields (str): 

1192 (Optional) Selector specifying which fields to include 

1193 in a partial response. Must be a list of fields. For 

1194 example to get a partial response with just the next 

1195 page token and the name and language of each blob returned: 

1196 ``'items(name,contentLanguage),nextPageToken'``. 

1197 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields 

1198 

1199 page_size (int): 

1200 (Optional) Maximum number of blobs to return in each page. 

1201 Defaults to a value set by the API. 

1202 

1203 timeout (Optional[Union[float, Tuple[float, float]]]): 

1204 The amount of time, in seconds, to wait for the server response. 

1205 

1206 Can also be passed as a tuple (connect_timeout, read_timeout). 

1207 See :meth:`requests.Session.request` documentation for details. 

1208 

1209 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 

1210 How to retry the RPC. A None value will disable retries. 

1211 A google.api_core.retry.Retry value will enable retries, and the object will 

1212 define retriable response codes and errors and configure backoff and timeout options. 

1213 

1214 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

1215 activates it only if certain conditions are met. This class exists to provide safe defaults 

1216 for RPC calls that are not technically safe to retry normally (due to potential data 

1217 duplication or other side-effects) but become safe to retry if a condition such as 

1218 if_metageneration_match is set. 

1219 

1220 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

1221 information on retry types and how to configure them. 

1222 

1223 Returns: 

1224 Iterator of all :class:`~google.cloud.storage.blob.Blob` 

1225 in this bucket matching the arguments. The RPC call 

1226 returns a response when the iterator is consumed. 

1227 

1228 As part of the response, you'll also get back an iterator.prefixes entity that lists object names 

1229 up to and including the requested delimiter. Duplicate entries are omitted from this list. 

1230 """ 

1231 bucket = self._bucket_arg_to_bucket(bucket_or_name) 

1232 

1233 extra_params = {"projection": projection} 

1234 

1235 if prefix is not None: 

1236 extra_params["prefix"] = prefix 

1237 

1238 if delimiter is not None: 

1239 extra_params["delimiter"] = delimiter 

1240 

1241 if start_offset is not None: 

1242 extra_params["startOffset"] = start_offset 

1243 

1244 if end_offset is not None: 

1245 extra_params["endOffset"] = end_offset 

1246 

1247 if include_trailing_delimiter is not None: 

1248 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter 

1249 

1250 if versions is not None: 

1251 extra_params["versions"] = versions 

1252 

1253 if fields is not None: 

1254 extra_params["fields"] = fields 

1255 

1256 if bucket.user_project is not None: 

1257 extra_params["userProject"] = bucket.user_project 

1258 

1259 path = bucket.path + "/o" 

1260 iterator = self._list_resource( 

1261 path, 

1262 _item_to_blob, 

1263 page_token=page_token, 

1264 max_results=max_results, 

1265 extra_params=extra_params, 

1266 page_start=_blobs_page_start, 

1267 page_size=page_size, 

1268 timeout=timeout, 

1269 retry=retry, 

1270 ) 

1271 iterator.bucket = bucket 

1272 iterator.prefixes = set() 

1273 return iterator 

1274 

1275 def list_buckets( 

1276 self, 

1277 max_results=None, 

1278 page_token=None, 

1279 prefix=None, 

1280 projection="noAcl", 

1281 fields=None, 

1282 project=None, 

1283 page_size=None, 

1284 timeout=_DEFAULT_TIMEOUT, 

1285 retry=DEFAULT_RETRY, 

1286 ): 

1287 """Get all buckets in the project associated to the client. 

1288 

1289 This will not populate the list of blobs available in each 

1290 bucket. 

1291 

1292 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python). 

1293 

1294 :type max_results: int 

1295 :param max_results: (Optional) The maximum number of buckets to return. 

1296 

1297 :type page_token: str 

1298 :param page_token: 

1299 (Optional) If present, return the next batch of buckets, using the 

1300 value, which must correspond to the ``nextPageToken`` value 

1301 returned in the previous response. Deprecated: use the ``pages`` 

1302 property of the returned iterator instead of manually passing the 

1303 token. 

1304 

1305 :type prefix: str 

1306 :param prefix: (Optional) Filter results to buckets whose names begin 

1307 with this prefix. 

1308 

1309 :type projection: str 

1310 :param projection: 

1311 (Optional) Specifies the set of properties to return. If used, must 

1312 be 'full' or 'noAcl'. Defaults to 'noAcl'. 

1313 

1314 :type fields: str 

1315 :param fields: 

1316 (Optional) Selector specifying which fields to include in a partial 

1317 response. Must be a list of fields. For example to get a partial 

1318 response with just the next page token and the language of each 

1319 bucket returned: 'items/id,nextPageToken' 

1320 

1321 :type project: str 

1322 :param project: (Optional) The project whose buckets are to be listed. 

1323 If not passed, uses the project set on the client. 

1324 

1325 :type page_size: int 

1326 :param page_size: (Optional) Maximum number of buckets to return in each page. 

1327 Defaults to a value set by the API. 

1328 

1329 :type timeout: float or tuple 

1330 :param timeout: 

1331 (Optional) The amount of time, in seconds, to wait 

1332 for the server response. See: :ref:`configuring_timeouts` 

1333 

1334 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1335 :param retry: 

1336 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

1337 

1338 :rtype: :class:`~google.api_core.page_iterator.Iterator` 

1339 :raises ValueError: if both ``project`` is ``None`` and the client's 

1340 project is also ``None``. 

1341 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket` 

1342 belonging to this project. 

1343 """ 

1344 extra_params = {} 

1345 

1346 if project is None: 

1347 project = self.project 

1348 

1349 # Use no project if STORAGE_EMULATOR_HOST is set 

1350 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST 

1351 if _is_emulator_set: 

1352 if project is None: 

1353 project = _get_environ_project() 

1354 if project is None: 

1355 project = "<none>" 

1356 

1357 # Only include the project parameter if a project is set. 

1358 # If a project is not set, falls back to API validation (BadRequest). 

1359 if project is not None: 

1360 extra_params = {"project": project} 

1361 

1362 if prefix is not None: 

1363 extra_params["prefix"] = prefix 

1364 

1365 extra_params["projection"] = projection 

1366 

1367 if fields is not None: 

1368 extra_params["fields"] = fields 

1369 

1370 return self._list_resource( 

1371 "/b", 

1372 _item_to_bucket, 

1373 page_token=page_token, 

1374 max_results=max_results, 

1375 extra_params=extra_params, 

1376 page_size=page_size, 

1377 timeout=timeout, 

1378 retry=retry, 

1379 ) 

1380 

1381 def create_hmac_key( 

1382 self, 

1383 service_account_email, 

1384 project_id=None, 

1385 user_project=None, 

1386 timeout=_DEFAULT_TIMEOUT, 

1387 retry=None, 

1388 ): 

1389 """Create an HMAC key for a service account. 

1390 

1391 :type service_account_email: str 

1392 :param service_account_email: e-mail address of the service account 

1393 

1394 :type project_id: str 

1395 :param project_id: (Optional) Explicit project ID for the key. 

1396 Defaults to the client's project. 

1397 

1398 :type user_project: str 

1399 :param user_project: (Optional) This parameter is currently ignored. 

1400 

1401 :type timeout: float or tuple 

1402 :param timeout: 

1403 (Optional) The amount of time, in seconds, to wait 

1404 for the server response. See: :ref:`configuring_timeouts` 

1405 

1406 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1407 :param retry: (Optional) How to retry the RPC. A None value will disable retries. 

1408 A google.api_core.retry.Retry value will enable retries, and the object will 

1409 define retriable response codes and errors and configure backoff and timeout options. 

1410 

1411 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 

1412 activates it only if certain conditions are met. This class exists to provide safe defaults 

1413 for RPC calls that are not technically safe to retry normally (due to potential data 

1414 duplication or other side-effects) but become safe to retry if a condition such as 

1415 if_metageneration_match is set. 

1416 

1417 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 

1418 information on retry types and how to configure them. 

1419 

1420 :rtype: 

1421 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] 

1422 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. 

1423 """ 

1424 if project_id is None: 

1425 project_id = self.project 

1426 

1427 path = f"/projects/{project_id}/hmacKeys" 

1428 qs_params = {"serviceAccountEmail": service_account_email} 

1429 

1430 if user_project is not None: 

1431 qs_params["userProject"] = user_project 

1432 

1433 api_response = self._post_resource( 

1434 path, 

1435 None, 

1436 query_params=qs_params, 

1437 timeout=timeout, 

1438 retry=retry, 

1439 ) 

1440 metadata = HMACKeyMetadata(self) 

1441 metadata._properties = api_response["metadata"] 

1442 secret = api_response["secret"] 

1443 return metadata, secret 

1444 

1445 def list_hmac_keys( 

1446 self, 

1447 max_results=None, 

1448 service_account_email=None, 

1449 show_deleted_keys=None, 

1450 project_id=None, 

1451 user_project=None, 

1452 timeout=_DEFAULT_TIMEOUT, 

1453 retry=DEFAULT_RETRY, 

1454 ): 

1455 """List HMAC keys for a project. 

1456 

1457 :type max_results: int 

1458 :param max_results: 

1459 (Optional) Max number of keys to return in a given page. 

1460 

1461 :type service_account_email: str 

1462 :param service_account_email: 

1463 (Optional) Limit keys to those created by the given service account. 

1464 

1465 :type show_deleted_keys: bool 

1466 :param show_deleted_keys: 

1467 (Optional) Included deleted keys in the list. Default is to 

1468 exclude them. 

1469 

1470 :type project_id: str 

1471 :param project_id: (Optional) Explicit project ID for the key. 

1472 Defaults to the client's project. 

1473 

1474 :type user_project: str 

1475 :param user_project: (Optional) This parameter is currently ignored. 

1476 

1477 :type timeout: float or tuple 

1478 :param timeout: 

1479 (Optional) The amount of time, in seconds, to wait 

1480 for the server response. See: :ref:`configuring_timeouts` 

1481 

1482 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 

1483 :param retry: 

1484 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 

1485 

1486 :rtype: 

1487 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] 

1488 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. 

1489 """ 

1490 if project_id is None: 

1491 project_id = self.project 

1492 

1493 path = f"/projects/{project_id}/hmacKeys" 

1494 extra_params = {} 

1495 

1496 if service_account_email is not None: 

1497 extra_params["serviceAccountEmail"] = service_account_email 

1498 

1499 if show_deleted_keys is not None: 

1500 extra_params["showDeletedKeys"] = show_deleted_keys 

1501 

1502 if user_project is not None: 

1503 extra_params["userProject"] = user_project 

1504 

1505 return self._list_resource( 

1506 path, 

1507 _item_to_hmac_key_metadata, 

1508 max_results=max_results, 

1509 extra_params=extra_params, 

1510 timeout=timeout, 

1511 retry=retry, 

1512 ) 

1513 

1514 def get_hmac_key_metadata( 

1515 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT 

1516 ): 

1517 """Return a metadata instance for the given HMAC key. 

1518 

1519 :type access_id: str 

1520 :param access_id: Unique ID of an existing key. 

1521 

1522 :type project_id: str 

1523 :param project_id: (Optional) Project ID of an existing key. 

1524 Defaults to client's project. 

1525 

1526 :type timeout: float or tuple 

1527 :param timeout: 

1528 (Optional) The amount of time, in seconds, to wait 

1529 for the server response. See: :ref:`configuring_timeouts` 

1530 

1531 :type user_project: str 

1532 :param user_project: (Optional) This parameter is currently ignored. 

1533 """ 

1534 metadata = HMACKeyMetadata(self, access_id, project_id, user_project) 

1535 metadata.reload(timeout=timeout) # raises NotFound for missing key 

1536 return metadata 

1537 

1538 def generate_signed_post_policy_v4( 

1539 self, 

1540 bucket_name, 

1541 blob_name, 

1542 expiration, 

1543 conditions=None, 

1544 fields=None, 

1545 credentials=None, 

1546 virtual_hosted_style=False, 

1547 bucket_bound_hostname=None, 

1548 scheme="http", 

1549 service_account_email=None, 

1550 access_token=None, 

1551 ): 

1552 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request. 

1553 

1554 .. note:: 

1555 

1556 Assumes ``credentials`` implements the 

1557 :class:`google.auth.credentials.Signing` interface. Also assumes 

1558 ``credentials`` has a ``service_account_email`` property which 

1559 identifies the credentials. 

1560 

1561 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py). 

1562 

1563 :type bucket_name: str 

1564 :param bucket_name: Bucket name. 

1565 

1566 :type blob_name: str 

1567 :param blob_name: Object name. 

1568 

1569 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] 

1570 :param expiration: Policy expiration time. If a ``datetime`` instance is 

1571 passed without an explicit ``tzinfo`` set, it will be 

1572 assumed to be ``UTC``. 

1573 

1574 :type conditions: list 

1575 :param conditions: (Optional) List of POST policy conditions, which are 

1576 used to restrict what is allowed in the request. 

1577 

1578 :type fields: dict 

1579 :param fields: (Optional) Additional elements to include into request. 

1580 

1581 :type credentials: :class:`google.auth.credentials.Signing` 

1582 :param credentials: (Optional) Credentials object with an associated private 

1583 key to sign text. 

1584 

1585 :type virtual_hosted_style: bool 

1586 :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket 

1587 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'. 

1588 

1589 :type bucket_bound_hostname: str 

1590 :param bucket_bound_hostname: 

1591 (Optional) If passed, construct the URL relative to the bucket-bound hostname. 

1592 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'. 

1593 See: https://cloud.google.com/storage/docs/request-endpoints#cname 

1594 

1595 :type scheme: str 

1596 :param scheme: 

1597 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use 

1598 this value as a scheme. ``https`` will work only when using a CDN. 

1599 Defaults to ``"http"``. 

1600 

1601 :type service_account_email: str 

1602 :param service_account_email: (Optional) E-mail address of the service account. 

1603 

1604 :type access_token: str 

1605 :param access_token: (Optional) Access token for a service account. 

1606 

1607 :rtype: dict 

1608 :returns: Signed POST policy. 

1609 """ 

1610 credentials = self._credentials if credentials is None else credentials 

1611 ensure_signed_credentials(credentials) 

1612 

1613 # prepare policy conditions and fields 

1614 timestamp, datestamp = get_v4_now_dtstamps() 

1615 

1616 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format( 

1617 email=credentials.signer_email, datestamp=datestamp 

1618 ) 

1619 required_conditions = [ 

1620 {"bucket": bucket_name}, 

1621 {"key": blob_name}, 

1622 {"x-goog-date": timestamp}, 

1623 {"x-goog-credential": x_goog_credential}, 

1624 {"x-goog-algorithm": "GOOG4-RSA-SHA256"}, 

1625 ] 

1626 

1627 conditions = conditions or [] 

1628 policy_fields = {} 

1629 for key, value in sorted((fields or {}).items()): 

1630 if not key.startswith("x-ignore-"): 

1631 policy_fields[key] = value 

1632 conditions.append({key: value}) 

1633 

1634 conditions += required_conditions 

1635 

1636 # calculate policy expiration time 

1637 now = _NOW() 

1638 if expiration is None: 

1639 expiration = now + datetime.timedelta(hours=1) 

1640 

1641 policy_expires = now + datetime.timedelta( 

1642 seconds=get_expiration_seconds_v4(expiration) 

1643 ) 

1644 

1645 # encode policy for signing 

1646 policy = json.dumps( 

1647 collections.OrderedDict( 

1648 sorted( 

1649 { 

1650 "conditions": conditions, 

1651 "expiration": policy_expires.isoformat() + "Z", 

1652 }.items() 

1653 ) 

1654 ), 

1655 separators=(",", ":"), 

1656 ) 

1657 str_to_sign = base64.b64encode(policy.encode("utf-8")) 

1658 

1659 # sign the policy and get its cryptographic signature 

1660 if access_token and service_account_email: 

1661 signature = _sign_message(str_to_sign, access_token, service_account_email) 

1662 signature_bytes = base64.b64decode(signature) 

1663 else: 

1664 signature_bytes = credentials.sign_bytes(str_to_sign) 

1665 

1666 # get hexadecimal representation of the signature 

1667 signature = binascii.hexlify(signature_bytes).decode("utf-8") 

1668 

1669 policy_fields.update( 

1670 { 

1671 "key": blob_name, 

1672 "x-goog-algorithm": "GOOG4-RSA-SHA256", 

1673 "x-goog-credential": x_goog_credential, 

1674 "x-goog-date": timestamp, 

1675 "x-goog-signature": signature, 

1676 "policy": str_to_sign.decode("utf-8"), 

1677 } 

1678 ) 

1679 # designate URL 

1680 if virtual_hosted_style: 

1681 url = f"https://{bucket_name}.storage.googleapis.com/" 

1682 elif bucket_bound_hostname: 

1683 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/" 

1684 else: 

1685 url = f"https://storage.googleapis.com/{bucket_name}/" 

1686 

1687 return {"url": url, "fields": policy_fields} 

1688 

1689 

1690def _item_to_bucket(iterator, item): 

1691 """Convert a JSON bucket to the native object. 

1692 

1693 :type iterator: :class:`~google.api_core.page_iterator.Iterator` 

1694 :param iterator: The iterator that has retrieved the item. 

1695 

1696 :type item: dict 

1697 :param item: An item to be converted to a bucket. 

1698 

1699 :rtype: :class:`.Bucket` 

1700 :returns: The next bucket in the page. 

1701 """ 

1702 name = item.get("name") 

1703 bucket = Bucket(iterator.client, name) 

1704 bucket._set_properties(item) 

1705 return bucket 

1706 

1707 

1708def _item_to_hmac_key_metadata(iterator, item): 

1709 """Convert a JSON key metadata resource to the native object. 

1710 

1711 :type iterator: :class:`~google.api_core.page_iterator.Iterator` 

1712 :param iterator: The iterator that has retrieved the item. 

1713 

1714 :type item: dict 

1715 :param item: An item to be converted to a key metadata instance. 

1716 

1717 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata` 

1718 :returns: The next key metadata instance in the page. 

1719 """ 

1720 metadata = HMACKeyMetadata(iterator.client) 

1721 metadata._properties = item 

1722 return metadata