Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/client.py: 23%
290 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 07:13 +0000
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Client for interacting with the Google Cloud Storage API."""
17import base64
18import binascii
19import collections
20import datetime
21import functools
22import json
23import warnings
24import google.api_core.client_options
26from google.auth.credentials import AnonymousCredentials
28from google import resumable_media
30from google.api_core import page_iterator
31from google.cloud._helpers import _LocalStack, _NOW
32from google.cloud.client import ClientWithProject
33from google.cloud.exceptions import NotFound
34from google.cloud.storage._helpers import _get_default_headers
35from google.cloud.storage._helpers import _get_environ_project
36from google.cloud.storage._helpers import _get_storage_host
37from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
38from google.cloud.storage._helpers import _bucket_bound_hostname_url
39from google.cloud.storage._helpers import _add_etag_match_headers
40from google.cloud.storage._http import Connection
41from google.cloud.storage._signing import (
42 get_expiration_seconds_v4,
43 get_v4_now_dtstamps,
44 ensure_signed_credentials,
45 _sign_message,
46)
47from google.cloud.storage.batch import Batch
48from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start
49from google.cloud.storage.blob import (
50 Blob,
51 _get_encryption_headers,
52 _raise_from_invalid_response,
53)
54from google.cloud.storage.hmac_key import HMACKeyMetadata
55from google.cloud.storage.acl import BucketACL
56from google.cloud.storage.acl import DefaultObjectACL
57from google.cloud.storage.constants import _DEFAULT_TIMEOUT
58from google.cloud.storage.retry import DEFAULT_RETRY
59from google.cloud.storage.retry import ConditionalRetryPolicy
62_marker = object()
65class Client(ClientWithProject):
66 """Client to bundle configuration needed for API requests.
68 :type project: str or None
69 :param project: the project which the client acts on behalf of. Will be
70 passed when creating a topic. If not passed,
71 falls back to the default inferred from the environment.
73 :type credentials: :class:`~google.auth.credentials.Credentials`
74 :param credentials: (Optional) The OAuth2 Credentials to use for this
75 client. If not passed (and if no ``_http`` object is
76 passed), falls back to the default inferred from the
77 environment.
79 :type _http: :class:`~requests.Session`
80 :param _http: (Optional) HTTP object to make requests. Can be any object
81 that defines ``request()`` with the same interface as
82 :meth:`requests.Session.request`. If not passed, an
83 ``_http`` object is created that is bound to the
84 ``credentials`` for the current object.
85 This parameter should be considered private, and could
86 change in the future.
88 :type client_info: :class:`~google.api_core.client_info.ClientInfo`
89 :param client_info:
90 The client info used to send a user-agent string along with API
91 requests. If ``None``, then default info will be used. Generally,
92 you only need to set this if you're developing your own library
93 or partner tool.
95 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
96 :param client_options: (Optional) Client options used to set user options on the client.
97 API Endpoint should be set through client_options.
99 :type use_auth_w_custom_endpoint: bool
100 :param use_auth_w_custom_endpoint:
101 (Optional) Whether authentication is required under custom endpoints.
102 If false, uses AnonymousCredentials and bypasses authentication.
103 Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
104 """
106 SCOPE = (
107 "https://www.googleapis.com/auth/devstorage.full_control",
108 "https://www.googleapis.com/auth/devstorage.read_only",
109 "https://www.googleapis.com/auth/devstorage.read_write",
110 )
111 """The scopes required for authenticating as a Cloud Storage consumer."""
113 def __init__(
114 self,
115 project=_marker,
116 credentials=None,
117 _http=None,
118 client_info=None,
119 client_options=None,
120 use_auth_w_custom_endpoint=True,
121 ):
122 self._base_connection = None
124 if project is None:
125 no_project = True
126 project = "<none>"
127 else:
128 no_project = False
130 if project is _marker:
131 project = None
133 # Save the initial value of constructor arguments before they
134 # are passed along, for use in __reduce__ defined elsewhere.
135 self._initial_client_info = client_info
136 self._initial_client_options = client_options
137 self._initial_credentials = credentials
139 kw_args = {"client_info": client_info}
141 # `api_endpoint` should be only set by the user via `client_options`,
142 # or if the _get_storage_host() returns a non-default value (_is_emulator_set).
143 # `api_endpoint` plays an important role for mTLS, if it is not set,
144 # then mTLS logic will be applied to decide which endpoint will be used.
145 storage_host = _get_storage_host()
146 _is_emulator_set = storage_host != _DEFAULT_STORAGE_HOST
147 kw_args["api_endpoint"] = storage_host if _is_emulator_set else None
149 if client_options:
150 if type(client_options) == dict:
151 client_options = google.api_core.client_options.from_dict(
152 client_options
153 )
154 if client_options.api_endpoint:
155 api_endpoint = client_options.api_endpoint
156 kw_args["api_endpoint"] = api_endpoint
158 # If a custom endpoint is set, the client checks for credentials
159 # or finds the default credentials based on the current environment.
160 # Authentication may be bypassed under certain conditions:
161 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
162 # (2) use_auth_w_custom_endpoint is set to False.
163 if kw_args["api_endpoint"] is not None:
164 if _is_emulator_set or not use_auth_w_custom_endpoint:
165 if credentials is None:
166 credentials = AnonymousCredentials()
167 if project is None:
168 project = _get_environ_project()
169 if project is None:
170 no_project = True
171 project = "<none>"
173 super(Client, self).__init__(
174 project=project,
175 credentials=credentials,
176 client_options=client_options,
177 _http=_http,
178 )
180 if no_project:
181 self.project = None
183 self._connection = Connection(self, **kw_args)
184 self._batch_stack = _LocalStack()
186 @classmethod
187 def create_anonymous_client(cls):
188 """Factory: return client with anonymous credentials.
190 .. note::
192 Such a client has only limited access to "public" buckets:
193 listing their contents and downloading their blobs.
195 :rtype: :class:`google.cloud.storage.client.Client`
196 :returns: Instance w/ anonymous credentials and no project.
197 """
198 client = cls(project="<none>", credentials=AnonymousCredentials())
199 client.project = None
200 return client
202 @property
203 def _connection(self):
204 """Get connection or batch on the client.
206 :rtype: :class:`google.cloud.storage._http.Connection`
207 :returns: The connection set on the client, or the batch
208 if one is set.
209 """
210 if self.current_batch is not None:
211 return self.current_batch
212 else:
213 return self._base_connection
215 @_connection.setter
216 def _connection(self, value):
217 """Set connection on the client.
219 Intended to be used by constructor (since the base class calls)
220 self._connection = connection
221 Will raise if the connection is set more than once.
223 :type value: :class:`google.cloud.storage._http.Connection`
224 :param value: The connection set on the client.
226 :raises: :class:`ValueError` if connection has already been set.
227 """
228 if self._base_connection is not None:
229 raise ValueError("Connection already set on client")
230 self._base_connection = value
232 def _push_batch(self, batch):
233 """Push a batch onto our stack.
235 "Protected", intended for use by batch context mgrs.
237 :type batch: :class:`google.cloud.storage.batch.Batch`
238 :param batch: newly-active batch
239 """
240 self._batch_stack.push(batch)
242 def _pop_batch(self):
243 """Pop a batch from our stack.
245 "Protected", intended for use by batch context mgrs.
247 :raises: IndexError if the stack is empty.
248 :rtype: :class:`google.cloud.storage.batch.Batch`
249 :returns: the top-most batch/transaction, after removing it.
250 """
251 return self._batch_stack.pop()
253 @property
254 def current_batch(self):
255 """Currently-active batch.
257 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if
258 no batch is active).
259 :returns: The batch at the top of the batch stack.
260 """
261 return self._batch_stack.top
263 def get_service_account_email(
264 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
265 ):
266 """Get the email address of the project's GCS service account
268 :type project: str
269 :param project:
270 (Optional) Project ID to use for retreiving GCS service account
271 email address. Defaults to the client's project.
272 :type timeout: float or tuple
273 :param timeout:
274 (Optional) The amount of time, in seconds, to wait
275 for the server response. See: :ref:`configuring_timeouts`
277 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
278 :param retry:
279 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
281 :rtype: str
282 :returns: service account email address
283 """
284 if project is None:
285 project = self.project
287 path = f"/projects/{project}/serviceAccount"
288 api_response = self._get_resource(path, timeout=timeout, retry=retry)
289 return api_response["email_address"]
291 def bucket(self, bucket_name, user_project=None):
292 """Factory constructor for bucket object.
294 .. note::
295 This will not make an HTTP request; it simply instantiates
296 a bucket object owned by this client.
298 :type bucket_name: str
299 :param bucket_name: The name of the bucket to be instantiated.
301 :type user_project: str
302 :param user_project: (Optional) The project ID to be billed for API
303 requests made via the bucket.
305 :rtype: :class:`google.cloud.storage.bucket.Bucket`
306 :returns: The bucket object created.
307 """
308 return Bucket(client=self, name=bucket_name, user_project=user_project)
310 def batch(self, raise_exception=True):
311 """Factory constructor for batch object.
313 .. note::
314 This will not make an HTTP request; it simply instantiates
315 a batch object owned by this client.
317 :type raise_exception: bool
318 :param raise_exception:
319 (Optional) Defaults to True. If True, instead of adding exceptions
320 to the list of return responses, the final exception will be raised.
321 Note that exceptions are unwrapped after all operations are complete
322 in success or failure, and only the last exception is raised.
324 :rtype: :class:`google.cloud.storage.batch.Batch`
325 :returns: The batch object created.
326 """
327 return Batch(client=self, raise_exception=raise_exception)
329 def _get_resource(
330 self,
331 path,
332 query_params=None,
333 headers=None,
334 timeout=_DEFAULT_TIMEOUT,
335 retry=DEFAULT_RETRY,
336 _target_object=None,
337 ):
338 """Helper for bucket / blob methods making API 'GET' calls.
340 Args:
341 path str:
342 The path of the resource to fetch.
344 query_params Optional[dict]:
345 HTTP query parameters to be passed
347 headers Optional[dict]:
348 HTTP headers to be passed
350 timeout (Optional[Union[float, Tuple[float, float]]]):
351 The amount of time, in seconds, to wait for the server response.
353 Can also be passed as a tuple (connect_timeout, read_timeout).
354 See :meth:`requests.Session.request` documentation for details.
356 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
357 How to retry the RPC. A None value will disable retries.
358 A google.api_core.retry.Retry value will enable retries, and the object will
359 define retriable response codes and errors and configure backoff and timeout options.
361 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
362 activates it only if certain conditions are met. This class exists to provide safe defaults
363 for RPC calls that are not technically safe to retry normally (due to potential data
364 duplication or other side-effects) but become safe to retry if a condition such as
365 if_metageneration_match is set.
367 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
368 information on retry types and how to configure them.
370 _target_object (Union[ \
371 :class:`~google.cloud.storage.bucket.Bucket`, \
372 :class:`~google.cloud.storage.bucket.blob`, \
373 ]):
374 Object to which future data is to be applied -- only relevant
375 in the context of a batch.
377 Returns:
378 dict
379 The JSON resource fetched
381 Raises:
382 google.cloud.exceptions.NotFound
383 If the bucket is not found.
384 """
385 return self._connection.api_request(
386 method="GET",
387 path=path,
388 query_params=query_params,
389 headers=headers,
390 timeout=timeout,
391 retry=retry,
392 _target_object=_target_object,
393 )
395 def _list_resource(
396 self,
397 path,
398 item_to_value,
399 page_token=None,
400 max_results=None,
401 extra_params=None,
402 page_start=page_iterator._do_nothing_page_start,
403 page_size=None,
404 timeout=_DEFAULT_TIMEOUT,
405 retry=DEFAULT_RETRY,
406 ):
407 api_request = functools.partial(
408 self._connection.api_request, timeout=timeout, retry=retry
409 )
410 return page_iterator.HTTPIterator(
411 client=self,
412 api_request=api_request,
413 path=path,
414 item_to_value=item_to_value,
415 page_token=page_token,
416 max_results=max_results,
417 extra_params=extra_params,
418 page_start=page_start,
419 page_size=page_size,
420 )
422 def _patch_resource(
423 self,
424 path,
425 data,
426 query_params=None,
427 headers=None,
428 timeout=_DEFAULT_TIMEOUT,
429 retry=None,
430 _target_object=None,
431 ):
432 """Helper for bucket / blob methods making API 'PATCH' calls.
434 Args:
435 path str:
436 The path of the resource to fetch.
438 data dict:
439 The data to be patched.
441 query_params Optional[dict]:
442 HTTP query parameters to be passed
444 headers Optional[dict]:
445 HTTP headers to be passed
447 timeout (Optional[Union[float, Tuple[float, float]]]):
448 The amount of time, in seconds, to wait for the server response.
450 Can also be passed as a tuple (connect_timeout, read_timeout).
451 See :meth:`requests.Session.request` documentation for details.
453 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
454 How to retry the RPC. A None value will disable retries.
455 A google.api_core.retry.Retry value will enable retries, and the object will
456 define retriable response codes and errors and configure backoff and timeout options.
458 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
459 activates it only if certain conditions are met. This class exists to provide safe defaults
460 for RPC calls that are not technically safe to retry normally (due to potential data
461 duplication or other side-effects) but become safe to retry if a condition such as
462 if_metageneration_match is set.
464 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
465 information on retry types and how to configure them.
467 _target_object (Union[ \
468 :class:`~google.cloud.storage.bucket.Bucket`, \
469 :class:`~google.cloud.storage.bucket.blob`, \
470 ]):
471 Object to which future data is to be applied -- only relevant
472 in the context of a batch.
474 Returns:
475 dict
476 The JSON resource fetched
478 Raises:
479 google.cloud.exceptions.NotFound
480 If the bucket is not found.
481 """
482 return self._connection.api_request(
483 method="PATCH",
484 path=path,
485 data=data,
486 query_params=query_params,
487 headers=headers,
488 timeout=timeout,
489 retry=retry,
490 _target_object=_target_object,
491 )
493 def _put_resource(
494 self,
495 path,
496 data,
497 query_params=None,
498 headers=None,
499 timeout=_DEFAULT_TIMEOUT,
500 retry=None,
501 _target_object=None,
502 ):
503 """Helper for bucket / blob methods making API 'PUT' calls.
505 Args:
506 path str:
507 The path of the resource to fetch.
509 data dict:
510 The data to be patched.
512 query_params Optional[dict]:
513 HTTP query parameters to be passed
515 headers Optional[dict]:
516 HTTP headers to be passed
518 timeout (Optional[Union[float, Tuple[float, float]]]):
519 The amount of time, in seconds, to wait for the server response.
521 Can also be passed as a tuple (connect_timeout, read_timeout).
522 See :meth:`requests.Session.request` documentation for details.
524 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
525 How to retry the RPC. A None value will disable retries.
526 A google.api_core.retry.Retry value will enable retries, and the object will
527 define retriable response codes and errors and configure backoff and timeout options.
529 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
530 activates it only if certain conditions are met. This class exists to provide safe defaults
531 for RPC calls that are not technically safe to retry normally (due to potential data
532 duplication or other side-effects) but become safe to retry if a condition such as
533 if_metageneration_match is set.
535 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
536 information on retry types and how to configure them.
538 _target_object (Union[ \
539 :class:`~google.cloud.storage.bucket.Bucket`, \
540 :class:`~google.cloud.storage.bucket.blob`, \
541 ]):
542 Object to which future data is to be applied -- only relevant
543 in the context of a batch.
545 Returns:
546 dict
547 The JSON resource fetched
549 Raises:
550 google.cloud.exceptions.NotFound
551 If the bucket is not found.
552 """
553 return self._connection.api_request(
554 method="PUT",
555 path=path,
556 data=data,
557 query_params=query_params,
558 headers=headers,
559 timeout=timeout,
560 retry=retry,
561 _target_object=_target_object,
562 )
564 def _post_resource(
565 self,
566 path,
567 data,
568 query_params=None,
569 headers=None,
570 timeout=_DEFAULT_TIMEOUT,
571 retry=None,
572 _target_object=None,
573 ):
574 """Helper for bucket / blob methods making API 'POST' calls.
576 Args:
577 path str:
578 The path of the resource to which to post.
580 data dict:
581 The data to be posted.
583 query_params Optional[dict]:
584 HTTP query parameters to be passed
586 headers Optional[dict]:
587 HTTP headers to be passed
589 timeout (Optional[Union[float, Tuple[float, float]]]):
590 The amount of time, in seconds, to wait for the server response.
592 Can also be passed as a tuple (connect_timeout, read_timeout).
593 See :meth:`requests.Session.request` documentation for details.
595 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
596 How to retry the RPC. A None value will disable retries.
597 A google.api_core.retry.Retry value will enable retries, and the object will
598 define retriable response codes and errors and configure backoff and timeout options.
600 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
601 activates it only if certain conditions are met. This class exists to provide safe defaults
602 for RPC calls that are not technically safe to retry normally (due to potential data
603 duplication or other side-effects) but become safe to retry if a condition such as
604 if_metageneration_match is set.
606 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
607 information on retry types and how to configure them.
609 _target_object (Union[ \
610 :class:`~google.cloud.storage.bucket.Bucket`, \
611 :class:`~google.cloud.storage.bucket.blob`, \
612 ]):
613 Object to which future data is to be applied -- only relevant
614 in the context of a batch.
616 Returns:
617 dict
618 The JSON resource returned from the post.
620 Raises:
621 google.cloud.exceptions.NotFound
622 If the bucket is not found.
623 """
625 return self._connection.api_request(
626 method="POST",
627 path=path,
628 data=data,
629 query_params=query_params,
630 headers=headers,
631 timeout=timeout,
632 retry=retry,
633 _target_object=_target_object,
634 )
636 def _delete_resource(
637 self,
638 path,
639 query_params=None,
640 headers=None,
641 timeout=_DEFAULT_TIMEOUT,
642 retry=DEFAULT_RETRY,
643 _target_object=None,
644 ):
645 """Helper for bucket / blob methods making API 'DELETE' calls.
647 Args:
648 path str:
649 The path of the resource to delete.
651 query_params Optional[dict]:
652 HTTP query parameters to be passed
654 headers Optional[dict]:
655 HTTP headers to be passed
657 timeout (Optional[Union[float, Tuple[float, float]]]):
658 The amount of time, in seconds, to wait for the server response.
660 Can also be passed as a tuple (connect_timeout, read_timeout).
661 See :meth:`requests.Session.request` documentation for details.
663 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
664 How to retry the RPC. A None value will disable retries.
665 A google.api_core.retry.Retry value will enable retries, and the object will
666 define retriable response codes and errors and configure backoff and timeout options.
668 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
669 activates it only if certain conditions are met. This class exists to provide safe defaults
670 for RPC calls that are not technically safe to retry normally (due to potential data
671 duplication or other side-effects) but become safe to retry if a condition such as
672 if_metageneration_match is set.
674 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
675 information on retry types and how to configure them.
677 _target_object (Union[ \
678 :class:`~google.cloud.storage.bucket.Bucket`, \
679 :class:`~google.cloud.storage.bucket.blob`, \
680 ]):
681 Object to which future data is to be applied -- only relevant
682 in the context of a batch.
684 Returns:
685 dict
686 The JSON resource fetched
688 Raises:
689 google.cloud.exceptions.NotFound
690 If the bucket is not found.
691 """
692 return self._connection.api_request(
693 method="DELETE",
694 path=path,
695 query_params=query_params,
696 headers=headers,
697 timeout=timeout,
698 retry=retry,
699 _target_object=_target_object,
700 )
702 def _bucket_arg_to_bucket(self, bucket_or_name):
703 """Helper to return given bucket or create new by name.
705 Args:
706 bucket_or_name (Union[ \
707 :class:`~google.cloud.storage.bucket.Bucket`, \
708 str, \
709 ]):
710 The bucket resource to pass or name to create.
712 Returns:
713 google.cloud.storage.bucket.Bucket
714 The newly created bucket or the given one.
715 """
716 if isinstance(bucket_or_name, Bucket):
717 bucket = bucket_or_name
718 if bucket.client is None:
719 bucket._client = self
720 else:
721 bucket = Bucket(self, name=bucket_or_name)
722 return bucket
724 def get_bucket(
725 self,
726 bucket_or_name,
727 timeout=_DEFAULT_TIMEOUT,
728 if_metageneration_match=None,
729 if_metageneration_not_match=None,
730 retry=DEFAULT_RETRY,
731 ):
732 """Retrieve a bucket via a GET request.
734 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python).
736 Args:
737 bucket_or_name (Union[ \
738 :class:`~google.cloud.storage.bucket.Bucket`, \
739 str, \
740 ]):
741 The bucket resource to pass or name to create.
743 timeout (Optional[Union[float, Tuple[float, float]]]):
744 The amount of time, in seconds, to wait for the server response.
746 Can also be passed as a tuple (connect_timeout, read_timeout).
747 See :meth:`requests.Session.request` documentation for details.
749 if_metageneration_match (Optional[long]):
750 Make the operation conditional on whether the
751 blob's current metageneration matches the given value.
753 if_metageneration_not_match (Optional[long]):
754 Make the operation conditional on whether the blob's
755 current metageneration does not match the given value.
757 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
758 How to retry the RPC. A None value will disable retries.
759 A google.api_core.retry.Retry value will enable retries, and the object will
760 define retriable response codes and errors and configure backoff and timeout options.
762 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
763 activates it only if certain conditions are met. This class exists to provide safe defaults
764 for RPC calls that are not technically safe to retry normally (due to potential data
765 duplication or other side-effects) but become safe to retry if a condition such as
766 if_metageneration_match is set.
768 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
769 information on retry types and how to configure them.
771 Returns:
772 google.cloud.storage.bucket.Bucket
773 The bucket matching the name provided.
775 Raises:
776 google.cloud.exceptions.NotFound
777 If the bucket is not found.
778 """
779 bucket = self._bucket_arg_to_bucket(bucket_or_name)
780 bucket.reload(
781 client=self,
782 timeout=timeout,
783 if_metageneration_match=if_metageneration_match,
784 if_metageneration_not_match=if_metageneration_not_match,
785 retry=retry,
786 )
787 return bucket
789 def lookup_bucket(
790 self,
791 bucket_name,
792 timeout=_DEFAULT_TIMEOUT,
793 if_metageneration_match=None,
794 if_metageneration_not_match=None,
795 retry=DEFAULT_RETRY,
796 ):
797 """Get a bucket by name, returning None if not found.
799 You can use this if you would rather check for a None value
800 than catching a NotFound exception.
802 :type bucket_name: str
803 :param bucket_name: The name of the bucket to get.
805 :type timeout: float or tuple
806 :param timeout:
807 (Optional) The amount of time, in seconds, to wait
808 for the server response. See: :ref:`configuring_timeouts`
810 :type if_metageneration_match: long
811 :param if_metageneration_match: (Optional) Make the operation conditional on whether the
812 blob's current metageneration matches the given value.
814 :type if_metageneration_not_match: long
815 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
816 blob's current metageneration does not match the given value.
818 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
819 :param retry:
820 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
822 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType``
823 :returns: The bucket matching the name provided or None if not found.
824 """
825 try:
826 return self.get_bucket(
827 bucket_name,
828 timeout=timeout,
829 if_metageneration_match=if_metageneration_match,
830 if_metageneration_not_match=if_metageneration_not_match,
831 retry=retry,
832 )
833 except NotFound:
834 return None
836 def create_bucket(
837 self,
838 bucket_or_name,
839 requester_pays=None,
840 project=None,
841 user_project=None,
842 location=None,
843 data_locations=None,
844 predefined_acl=None,
845 predefined_default_object_acl=None,
846 timeout=_DEFAULT_TIMEOUT,
847 retry=DEFAULT_RETRY,
848 ):
849 """Create a new bucket via a POST request.
851 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python).
853 Args:
854 bucket_or_name (Union[ \
855 :class:`~google.cloud.storage.bucket.Bucket`, \
856 str, \
857 ]):
858 The bucket resource to pass or name to create.
859 requester_pays (bool):
860 DEPRECATED. Use Bucket().requester_pays instead.
861 (Optional) Whether requester pays for API requests for
862 this bucket and its blobs.
863 project (str):
864 (Optional) The project under which the bucket is to be created.
865 If not passed, uses the project set on the client.
866 user_project (str):
867 (Optional) The project ID to be billed for API requests
868 made via created bucket.
869 location (str):
870 (Optional) The location of the bucket. If not passed,
871 the default location, US, will be used. If specifying a dual-region,
872 `data_locations` should be set in conjunction. See:
873 https://cloud.google.com/storage/docs/locations
874 data_locations (list of str):
875 (Optional) The list of regional locations of a custom dual-region bucket.
876 Dual-regions require exactly 2 regional locations. See:
877 https://cloud.google.com/storage/docs/locations
878 predefined_acl (str):
879 (Optional) Name of predefined ACL to apply to bucket. See:
880 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
881 predefined_default_object_acl (str):
882 (Optional) Name of predefined ACL to apply to bucket's objects. See:
883 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
884 timeout (Optional[Union[float, Tuple[float, float]]]):
885 The amount of time, in seconds, to wait for the server response.
887 Can also be passed as a tuple (connect_timeout, read_timeout).
888 See :meth:`requests.Session.request` documentation for details.
890 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
891 How to retry the RPC. A None value will disable retries.
892 A google.api_core.retry.Retry value will enable retries, and the object will
893 define retriable response codes and errors and configure backoff and timeout options.
895 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
896 activates it only if certain conditions are met. This class exists to provide safe defaults
897 for RPC calls that are not technically safe to retry normally (due to potential data
898 duplication or other side-effects) but become safe to retry if a condition such as
899 if_metageneration_match is set.
901 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
902 information on retry types and how to configure them.
904 Returns:
905 google.cloud.storage.bucket.Bucket
906 The newly created bucket.
908 Raises:
909 google.cloud.exceptions.Conflict
910 If the bucket already exists.
911 """
912 bucket = self._bucket_arg_to_bucket(bucket_or_name)
913 query_params = {}
915 if project is None:
916 project = self.project
918 # Use no project if STORAGE_EMULATOR_HOST is set
919 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
920 if _is_emulator_set:
921 if project is None:
922 project = _get_environ_project()
923 if project is None:
924 project = "<none>"
926 # Only include the project parameter if a project is set.
927 # If a project is not set, falls back to API validation (BadRequest).
928 if project is not None:
929 query_params = {"project": project}
931 if requester_pays is not None:
932 warnings.warn(
933 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.",
934 PendingDeprecationWarning,
935 stacklevel=1,
936 )
937 bucket.requester_pays = requester_pays
939 if predefined_acl is not None:
940 predefined_acl = BucketACL.validate_predefined(predefined_acl)
941 query_params["predefinedAcl"] = predefined_acl
943 if predefined_default_object_acl is not None:
944 predefined_default_object_acl = DefaultObjectACL.validate_predefined(
945 predefined_default_object_acl
946 )
947 query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl
949 if user_project is not None:
950 query_params["userProject"] = user_project
952 properties = {key: bucket._properties[key] for key in bucket._changes}
953 properties["name"] = bucket.name
955 if location is not None:
956 properties["location"] = location
958 if data_locations is not None:
959 properties["customPlacementConfig"] = {"dataLocations": data_locations}
961 api_response = self._post_resource(
962 "/b",
963 properties,
964 query_params=query_params,
965 timeout=timeout,
966 retry=retry,
967 _target_object=bucket,
968 )
970 bucket._set_properties(api_response)
971 return bucket
973 def download_blob_to_file(
974 self,
975 blob_or_uri,
976 file_obj,
977 start=None,
978 end=None,
979 raw_download=False,
980 if_etag_match=None,
981 if_etag_not_match=None,
982 if_generation_match=None,
983 if_generation_not_match=None,
984 if_metageneration_match=None,
985 if_metageneration_not_match=None,
986 timeout=_DEFAULT_TIMEOUT,
987 checksum="md5",
988 retry=DEFAULT_RETRY,
989 ):
990 """Download the contents of a blob object or blob URI into a file-like object.
992 See https://cloud.google.com/storage/docs/downloading-objects
994 Args:
995 blob_or_uri (Union[ \
996 :class:`~google.cloud.storage.blob.Blob`, \
997 str, \
998 ]):
999 The blob resource to pass or URI to download.
1001 file_obj (file):
1002 A file handle to which to write the blob's data.
1004 start (int):
1005 (Optional) The first byte in a range to be downloaded.
1007 end (int):
1008 (Optional) The last byte in a range to be downloaded.
1010 raw_download (bool):
1011 (Optional) If true, download the object without any expansion.
1013 if_etag_match (Union[str, Set[str]]):
1014 (Optional) See :ref:`using-if-etag-match`
1016 if_etag_not_match (Union[str, Set[str]]):
1017 (Optional) See :ref:`using-if-etag-not-match`
1019 if_generation_match (long):
1020 (Optional) See :ref:`using-if-generation-match`
1022 if_generation_not_match (long):
1023 (Optional) See :ref:`using-if-generation-not-match`
1025 if_metageneration_match (long):
1026 (Optional) See :ref:`using-if-metageneration-match`
1028 if_metageneration_not_match (long):
1029 (Optional) See :ref:`using-if-metageneration-not-match`
1031 timeout ([Union[float, Tuple[float, float]]]):
1032 (Optional) The amount of time, in seconds, to wait
1033 for the server response. See: :ref:`configuring_timeouts`
1035 checksum (str):
1036 (Optional) The type of checksum to compute to verify the integrity
1037 of the object. The response headers must contain a checksum of the
1038 requested type. If the headers lack an appropriate checksum (for
1039 instance in the case of transcoded or ranged downloads where the
1040 remote service does not know the correct checksum, including
1041 downloads where chunk_size is set) an INFO-level log will be
1042 emitted. Supported values are "md5", "crc32c" and None. The default
1043 is "md5".
1044 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy)
1045 (Optional) How to retry the RPC. A None value will disable
1046 retries. A google.api_core.retry.Retry value will enable retries,
1047 and the object will define retriable response codes and errors and
1048 configure backoff and timeout options.
1050 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1051 Retry object and activates it only if certain conditions are met.
1052 This class exists to provide safe defaults for RPC calls that are
1053 not technically safe to retry normally (due to potential data
1054 duplication or other side-effects) but become safe to retry if a
1055 condition such as if_metageneration_match is set.
1057 See the retry.py source code and docstrings in this package
1058 (google.cloud.storage.retry) for information on retry types and how
1059 to configure them.
1061 Media operations (downloads and uploads) do not support non-default
1062 predicates in a Retry object. The default will always be used. Other
1063 configuration changes for Retry objects such as delays and deadlines
1064 are respected.
1065 """
1067 # Handle ConditionalRetryPolicy.
1068 if isinstance(retry, ConditionalRetryPolicy):
1069 # Conditional retries are designed for non-media calls, which change
1070 # arguments into query_params dictionaries. Media operations work
1071 # differently, so here we make a "fake" query_params to feed to the
1072 # ConditionalRetryPolicy.
1073 query_params = {
1074 "ifGenerationMatch": if_generation_match,
1075 "ifMetagenerationMatch": if_metageneration_match,
1076 }
1077 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)
1079 if not isinstance(blob_or_uri, Blob):
1080 blob_or_uri = Blob.from_string(blob_or_uri)
1081 download_url = blob_or_uri._get_download_url(
1082 self,
1083 if_generation_match=if_generation_match,
1084 if_generation_not_match=if_generation_not_match,
1085 if_metageneration_match=if_metageneration_match,
1086 if_metageneration_not_match=if_metageneration_not_match,
1087 )
1088 headers = _get_encryption_headers(blob_or_uri._encryption_key)
1089 headers["accept-encoding"] = "gzip"
1090 _add_etag_match_headers(
1091 headers,
1092 if_etag_match=if_etag_match,
1093 if_etag_not_match=if_etag_not_match,
1094 )
1095 headers = {**_get_default_headers(self._connection.user_agent), **headers}
1097 transport = self._http
1098 try:
1099 blob_or_uri._do_download(
1100 transport,
1101 file_obj,
1102 download_url,
1103 headers,
1104 start,
1105 end,
1106 raw_download,
1107 timeout=timeout,
1108 checksum=checksum,
1109 retry=retry,
1110 )
1111 except resumable_media.InvalidResponse as exc:
1112 _raise_from_invalid_response(exc)
1114 def list_blobs(
1115 self,
1116 bucket_or_name,
1117 max_results=None,
1118 page_token=None,
1119 prefix=None,
1120 delimiter=None,
1121 start_offset=None,
1122 end_offset=None,
1123 include_trailing_delimiter=None,
1124 versions=None,
1125 projection="noAcl",
1126 fields=None,
1127 page_size=None,
1128 timeout=_DEFAULT_TIMEOUT,
1129 retry=DEFAULT_RETRY,
1130 ):
1131 """Return an iterator used to find blobs in the bucket.
1133 If :attr:`user_project` is set, bills the API request to that project.
1135 .. note::
1136 List prefixes (directories) in a bucket using a prefix and delimiter.
1137 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python)
1138 listing objects using a prefix filter.
1140 Args:
1141 bucket_or_name (Union[ \
1142 :class:`~google.cloud.storage.bucket.Bucket`, \
1143 str, \
1144 ]):
1145 The bucket resource to pass or name to create.
1147 max_results (int):
1148 (Optional) The maximum number of blobs to return.
1150 page_token (str):
1151 (Optional) If present, return the next batch of blobs, using the
1152 value, which must correspond to the ``nextPageToken`` value
1153 returned in the previous response. Deprecated: use the ``pages``
1154 property of the returned iterator instead of manually passing the
1155 token.
1157 prefix (str):
1158 (Optional) Prefix used to filter blobs.
1160 delimiter (str):
1161 (Optional) Delimiter, used with ``prefix`` to
1162 emulate hierarchy.
1164 start_offset (str):
1165 (Optional) Filter results to objects whose names are
1166 lexicographically equal to or after ``startOffset``. If
1167 ``endOffset`` is also set, the objects listed will have names
1168 between ``startOffset`` (inclusive) and ``endOffset``
1169 (exclusive).
1171 end_offset (str):
1172 (Optional) Filter results to objects whose names are
1173 lexicographically before ``endOffset``. If ``startOffset`` is
1174 also set, the objects listed will have names between
1175 ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
1177 include_trailing_delimiter (boolean):
1178 (Optional) If true, objects that end in exactly one instance of
1179 ``delimiter`` will have their metadata included in ``items`` in
1180 addition to ``prefixes``.
1182 versions (bool):
1183 (Optional) Whether object versions should be returned
1184 as separate blobs.
1186 projection (str):
1187 (Optional) If used, must be 'full' or 'noAcl'.
1188 Defaults to ``'noAcl'``. Specifies the set of
1189 properties to return.
1191 fields (str):
1192 (Optional) Selector specifying which fields to include
1193 in a partial response. Must be a list of fields. For
1194 example to get a partial response with just the next
1195 page token and the name and language of each blob returned:
1196 ``'items(name,contentLanguage),nextPageToken'``.
1197 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
1199 page_size (int):
1200 (Optional) Maximum number of blobs to return in each page.
1201 Defaults to a value set by the API.
1203 timeout (Optional[Union[float, Tuple[float, float]]]):
1204 The amount of time, in seconds, to wait for the server response.
1206 Can also be passed as a tuple (connect_timeout, read_timeout).
1207 See :meth:`requests.Session.request` documentation for details.
1209 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1210 How to retry the RPC. A None value will disable retries.
1211 A google.api_core.retry.Retry value will enable retries, and the object will
1212 define retriable response codes and errors and configure backoff and timeout options.
1214 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1215 activates it only if certain conditions are met. This class exists to provide safe defaults
1216 for RPC calls that are not technically safe to retry normally (due to potential data
1217 duplication or other side-effects) but become safe to retry if a condition such as
1218 if_metageneration_match is set.
1220 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1221 information on retry types and how to configure them.
1223 Returns:
1224 Iterator of all :class:`~google.cloud.storage.blob.Blob`
1225 in this bucket matching the arguments. The RPC call
1226 returns a response when the iterator is consumed.
1228 As part of the response, you'll also get back an iterator.prefixes entity that lists object names
1229 up to and including the requested delimiter. Duplicate entries are omitted from this list.
1230 """
1231 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1233 extra_params = {"projection": projection}
1235 if prefix is not None:
1236 extra_params["prefix"] = prefix
1238 if delimiter is not None:
1239 extra_params["delimiter"] = delimiter
1241 if start_offset is not None:
1242 extra_params["startOffset"] = start_offset
1244 if end_offset is not None:
1245 extra_params["endOffset"] = end_offset
1247 if include_trailing_delimiter is not None:
1248 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1250 if versions is not None:
1251 extra_params["versions"] = versions
1253 if fields is not None:
1254 extra_params["fields"] = fields
1256 if bucket.user_project is not None:
1257 extra_params["userProject"] = bucket.user_project
1259 path = bucket.path + "/o"
1260 iterator = self._list_resource(
1261 path,
1262 _item_to_blob,
1263 page_token=page_token,
1264 max_results=max_results,
1265 extra_params=extra_params,
1266 page_start=_blobs_page_start,
1267 page_size=page_size,
1268 timeout=timeout,
1269 retry=retry,
1270 )
1271 iterator.bucket = bucket
1272 iterator.prefixes = set()
1273 return iterator
1275 def list_buckets(
1276 self,
1277 max_results=None,
1278 page_token=None,
1279 prefix=None,
1280 projection="noAcl",
1281 fields=None,
1282 project=None,
1283 page_size=None,
1284 timeout=_DEFAULT_TIMEOUT,
1285 retry=DEFAULT_RETRY,
1286 ):
1287 """Get all buckets in the project associated to the client.
1289 This will not populate the list of blobs available in each
1290 bucket.
1292 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python).
1294 :type max_results: int
1295 :param max_results: (Optional) The maximum number of buckets to return.
1297 :type page_token: str
1298 :param page_token:
1299 (Optional) If present, return the next batch of buckets, using the
1300 value, which must correspond to the ``nextPageToken`` value
1301 returned in the previous response. Deprecated: use the ``pages``
1302 property of the returned iterator instead of manually passing the
1303 token.
1305 :type prefix: str
1306 :param prefix: (Optional) Filter results to buckets whose names begin
1307 with this prefix.
1309 :type projection: str
1310 :param projection:
1311 (Optional) Specifies the set of properties to return. If used, must
1312 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1314 :type fields: str
1315 :param fields:
1316 (Optional) Selector specifying which fields to include in a partial
1317 response. Must be a list of fields. For example to get a partial
1318 response with just the next page token and the language of each
1319 bucket returned: 'items/id,nextPageToken'
1321 :type project: str
1322 :param project: (Optional) The project whose buckets are to be listed.
1323 If not passed, uses the project set on the client.
1325 :type page_size: int
1326 :param page_size: (Optional) Maximum number of buckets to return in each page.
1327 Defaults to a value set by the API.
1329 :type timeout: float or tuple
1330 :param timeout:
1331 (Optional) The amount of time, in seconds, to wait
1332 for the server response. See: :ref:`configuring_timeouts`
1334 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1335 :param retry:
1336 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1338 :rtype: :class:`~google.api_core.page_iterator.Iterator`
1339 :raises ValueError: if both ``project`` is ``None`` and the client's
1340 project is also ``None``.
1341 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
1342 belonging to this project.
1343 """
1344 extra_params = {}
1346 if project is None:
1347 project = self.project
1349 # Use no project if STORAGE_EMULATOR_HOST is set
1350 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
1351 if _is_emulator_set:
1352 if project is None:
1353 project = _get_environ_project()
1354 if project is None:
1355 project = "<none>"
1357 # Only include the project parameter if a project is set.
1358 # If a project is not set, falls back to API validation (BadRequest).
1359 if project is not None:
1360 extra_params = {"project": project}
1362 if prefix is not None:
1363 extra_params["prefix"] = prefix
1365 extra_params["projection"] = projection
1367 if fields is not None:
1368 extra_params["fields"] = fields
1370 return self._list_resource(
1371 "/b",
1372 _item_to_bucket,
1373 page_token=page_token,
1374 max_results=max_results,
1375 extra_params=extra_params,
1376 page_size=page_size,
1377 timeout=timeout,
1378 retry=retry,
1379 )
1381 def create_hmac_key(
1382 self,
1383 service_account_email,
1384 project_id=None,
1385 user_project=None,
1386 timeout=_DEFAULT_TIMEOUT,
1387 retry=None,
1388 ):
1389 """Create an HMAC key for a service account.
1391 :type service_account_email: str
1392 :param service_account_email: e-mail address of the service account
1394 :type project_id: str
1395 :param project_id: (Optional) Explicit project ID for the key.
1396 Defaults to the client's project.
1398 :type user_project: str
1399 :param user_project: (Optional) This parameter is currently ignored.
1401 :type timeout: float or tuple
1402 :param timeout:
1403 (Optional) The amount of time, in seconds, to wait
1404 for the server response. See: :ref:`configuring_timeouts`
1406 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1407 :param retry: (Optional) How to retry the RPC. A None value will disable retries.
1408 A google.api_core.retry.Retry value will enable retries, and the object will
1409 define retriable response codes and errors and configure backoff and timeout options.
1411 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1412 activates it only if certain conditions are met. This class exists to provide safe defaults
1413 for RPC calls that are not technically safe to retry normally (due to potential data
1414 duplication or other side-effects) but become safe to retry if a condition such as
1415 if_metageneration_match is set.
1417 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1418 information on retry types and how to configure them.
1420 :rtype:
1421 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1422 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1423 """
1424 if project_id is None:
1425 project_id = self.project
1427 path = f"/projects/{project_id}/hmacKeys"
1428 qs_params = {"serviceAccountEmail": service_account_email}
1430 if user_project is not None:
1431 qs_params["userProject"] = user_project
1433 api_response = self._post_resource(
1434 path,
1435 None,
1436 query_params=qs_params,
1437 timeout=timeout,
1438 retry=retry,
1439 )
1440 metadata = HMACKeyMetadata(self)
1441 metadata._properties = api_response["metadata"]
1442 secret = api_response["secret"]
1443 return metadata, secret
1445 def list_hmac_keys(
1446 self,
1447 max_results=None,
1448 service_account_email=None,
1449 show_deleted_keys=None,
1450 project_id=None,
1451 user_project=None,
1452 timeout=_DEFAULT_TIMEOUT,
1453 retry=DEFAULT_RETRY,
1454 ):
1455 """List HMAC keys for a project.
1457 :type max_results: int
1458 :param max_results:
1459 (Optional) Max number of keys to return in a given page.
1461 :type service_account_email: str
1462 :param service_account_email:
1463 (Optional) Limit keys to those created by the given service account.
1465 :type show_deleted_keys: bool
1466 :param show_deleted_keys:
1467 (Optional) Included deleted keys in the list. Default is to
1468 exclude them.
1470 :type project_id: str
1471 :param project_id: (Optional) Explicit project ID for the key.
1472 Defaults to the client's project.
1474 :type user_project: str
1475 :param user_project: (Optional) This parameter is currently ignored.
1477 :type timeout: float or tuple
1478 :param timeout:
1479 (Optional) The amount of time, in seconds, to wait
1480 for the server response. See: :ref:`configuring_timeouts`
1482 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1483 :param retry:
1484 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1486 :rtype:
1487 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1488 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1489 """
1490 if project_id is None:
1491 project_id = self.project
1493 path = f"/projects/{project_id}/hmacKeys"
1494 extra_params = {}
1496 if service_account_email is not None:
1497 extra_params["serviceAccountEmail"] = service_account_email
1499 if show_deleted_keys is not None:
1500 extra_params["showDeletedKeys"] = show_deleted_keys
1502 if user_project is not None:
1503 extra_params["userProject"] = user_project
1505 return self._list_resource(
1506 path,
1507 _item_to_hmac_key_metadata,
1508 max_results=max_results,
1509 extra_params=extra_params,
1510 timeout=timeout,
1511 retry=retry,
1512 )
1514 def get_hmac_key_metadata(
1515 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT
1516 ):
1517 """Return a metadata instance for the given HMAC key.
1519 :type access_id: str
1520 :param access_id: Unique ID of an existing key.
1522 :type project_id: str
1523 :param project_id: (Optional) Project ID of an existing key.
1524 Defaults to client's project.
1526 :type timeout: float or tuple
1527 :param timeout:
1528 (Optional) The amount of time, in seconds, to wait
1529 for the server response. See: :ref:`configuring_timeouts`
1531 :type user_project: str
1532 :param user_project: (Optional) This parameter is currently ignored.
1533 """
1534 metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
1535 metadata.reload(timeout=timeout) # raises NotFound for missing key
1536 return metadata
1538 def generate_signed_post_policy_v4(
1539 self,
1540 bucket_name,
1541 blob_name,
1542 expiration,
1543 conditions=None,
1544 fields=None,
1545 credentials=None,
1546 virtual_hosted_style=False,
1547 bucket_bound_hostname=None,
1548 scheme="http",
1549 service_account_email=None,
1550 access_token=None,
1551 ):
1552 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request.
1554 .. note::
1556 Assumes ``credentials`` implements the
1557 :class:`google.auth.credentials.Signing` interface. Also assumes
1558 ``credentials`` has a ``service_account_email`` property which
1559 identifies the credentials.
1561 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py).
1563 :type bucket_name: str
1564 :param bucket_name: Bucket name.
1566 :type blob_name: str
1567 :param blob_name: Object name.
1569 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
1570 :param expiration: Policy expiration time. If a ``datetime`` instance is
1571 passed without an explicit ``tzinfo`` set, it will be
1572 assumed to be ``UTC``.
1574 :type conditions: list
1575 :param conditions: (Optional) List of POST policy conditions, which are
1576 used to restrict what is allowed in the request.
1578 :type fields: dict
1579 :param fields: (Optional) Additional elements to include into request.
1581 :type credentials: :class:`google.auth.credentials.Signing`
1582 :param credentials: (Optional) Credentials object with an associated private
1583 key to sign text.
1585 :type virtual_hosted_style: bool
1586 :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket
1587 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
1589 :type bucket_bound_hostname: str
1590 :param bucket_bound_hostname:
1591 (Optional) If passed, construct the URL relative to the bucket-bound hostname.
1592 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
1593 See: https://cloud.google.com/storage/docs/request-endpoints#cname
1595 :type scheme: str
1596 :param scheme:
1597 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
1598 this value as a scheme. ``https`` will work only when using a CDN.
1599 Defaults to ``"http"``.
1601 :type service_account_email: str
1602 :param service_account_email: (Optional) E-mail address of the service account.
1604 :type access_token: str
1605 :param access_token: (Optional) Access token for a service account.
1607 :rtype: dict
1608 :returns: Signed POST policy.
1609 """
1610 credentials = self._credentials if credentials is None else credentials
1611 ensure_signed_credentials(credentials)
1613 # prepare policy conditions and fields
1614 timestamp, datestamp = get_v4_now_dtstamps()
1616 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
1617 email=credentials.signer_email, datestamp=datestamp
1618 )
1619 required_conditions = [
1620 {"bucket": bucket_name},
1621 {"key": blob_name},
1622 {"x-goog-date": timestamp},
1623 {"x-goog-credential": x_goog_credential},
1624 {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
1625 ]
1627 conditions = conditions or []
1628 policy_fields = {}
1629 for key, value in sorted((fields or {}).items()):
1630 if not key.startswith("x-ignore-"):
1631 policy_fields[key] = value
1632 conditions.append({key: value})
1634 conditions += required_conditions
1636 # calculate policy expiration time
1637 now = _NOW()
1638 if expiration is None:
1639 expiration = now + datetime.timedelta(hours=1)
1641 policy_expires = now + datetime.timedelta(
1642 seconds=get_expiration_seconds_v4(expiration)
1643 )
1645 # encode policy for signing
1646 policy = json.dumps(
1647 collections.OrderedDict(
1648 sorted(
1649 {
1650 "conditions": conditions,
1651 "expiration": policy_expires.isoformat() + "Z",
1652 }.items()
1653 )
1654 ),
1655 separators=(",", ":"),
1656 )
1657 str_to_sign = base64.b64encode(policy.encode("utf-8"))
1659 # sign the policy and get its cryptographic signature
1660 if access_token and service_account_email:
1661 signature = _sign_message(str_to_sign, access_token, service_account_email)
1662 signature_bytes = base64.b64decode(signature)
1663 else:
1664 signature_bytes = credentials.sign_bytes(str_to_sign)
1666 # get hexadecimal representation of the signature
1667 signature = binascii.hexlify(signature_bytes).decode("utf-8")
1669 policy_fields.update(
1670 {
1671 "key": blob_name,
1672 "x-goog-algorithm": "GOOG4-RSA-SHA256",
1673 "x-goog-credential": x_goog_credential,
1674 "x-goog-date": timestamp,
1675 "x-goog-signature": signature,
1676 "policy": str_to_sign.decode("utf-8"),
1677 }
1678 )
1679 # designate URL
1680 if virtual_hosted_style:
1681 url = f"https://{bucket_name}.storage.googleapis.com/"
1682 elif bucket_bound_hostname:
1683 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/"
1684 else:
1685 url = f"https://storage.googleapis.com/{bucket_name}/"
1687 return {"url": url, "fields": policy_fields}
1690def _item_to_bucket(iterator, item):
1691 """Convert a JSON bucket to the native object.
1693 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1694 :param iterator: The iterator that has retrieved the item.
1696 :type item: dict
1697 :param item: An item to be converted to a bucket.
1699 :rtype: :class:`.Bucket`
1700 :returns: The next bucket in the page.
1701 """
1702 name = item.get("name")
1703 bucket = Bucket(iterator.client, name)
1704 bucket._set_properties(item)
1705 return bucket
1708def _item_to_hmac_key_metadata(iterator, item):
1709 """Convert a JSON key metadata resource to the native object.
1711 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1712 :param iterator: The iterator that has retrieved the item.
1714 :type item: dict
1715 :param item: An item to be converted to a key metadata instance.
1717 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`
1718 :returns: The next key metadata instance in the page.
1719 """
1720 metadata = HMACKeyMetadata(iterator.client)
1721 metadata._properties = item
1722 return metadata