Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/client.py: 23%
276 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:17 +0000
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Client for interacting with the Google Cloud Storage API."""
17import base64
18import binascii
19import collections
20import datetime
21import functools
22import json
23import warnings
24import google.api_core.client_options
26from google.auth.credentials import AnonymousCredentials
28from google.api_core import page_iterator
29from google.cloud._helpers import _LocalStack, _NOW
30from google.cloud.client import ClientWithProject
31from google.cloud.exceptions import NotFound
33from google.cloud.storage._helpers import _get_environ_project
34from google.cloud.storage._helpers import _get_storage_host
35from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
36from google.cloud.storage._helpers import _bucket_bound_hostname_url
38from google.cloud.storage._http import Connection
39from google.cloud.storage._signing import (
40 get_expiration_seconds_v4,
41 get_v4_now_dtstamps,
42 ensure_signed_credentials,
43 _sign_message,
44)
45from google.cloud.storage.batch import Batch
46from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start
47from google.cloud.storage.blob import Blob
48from google.cloud.storage.hmac_key import HMACKeyMetadata
49from google.cloud.storage.acl import BucketACL
50from google.cloud.storage.acl import DefaultObjectACL
51from google.cloud.storage.constants import _DEFAULT_TIMEOUT
52from google.cloud.storage.retry import DEFAULT_RETRY
55_marker = object()
58class Client(ClientWithProject):
59 """Client to bundle configuration needed for API requests.
61 :type project: str or None
62 :param project: the project which the client acts on behalf of. Will be
63 passed when creating a topic. If not passed,
64 falls back to the default inferred from the environment.
66 :type credentials: :class:`~google.auth.credentials.Credentials`
67 :param credentials: (Optional) The OAuth2 Credentials to use for this
68 client. If not passed (and if no ``_http`` object is
69 passed), falls back to the default inferred from the
70 environment.
72 :type _http: :class:`~requests.Session`
73 :param _http: (Optional) HTTP object to make requests. Can be any object
74 that defines ``request()`` with the same interface as
75 :meth:`requests.Session.request`. If not passed, an
76 ``_http`` object is created that is bound to the
77 ``credentials`` for the current object.
78 This parameter should be considered private, and could
79 change in the future.
81 :type client_info: :class:`~google.api_core.client_info.ClientInfo`
82 :param client_info:
83 The client info used to send a user-agent string along with API
84 requests. If ``None``, then default info will be used. Generally,
85 you only need to set this if you're developing your own library
86 or partner tool.
88 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
89 :param client_options: (Optional) Client options used to set user options on the client.
90 API Endpoint should be set through client_options.
92 :type use_auth_w_custom_endpoint: bool
93 :param use_auth_w_custom_endpoint:
94 (Optional) Whether authentication is required under custom endpoints.
95 If false, uses AnonymousCredentials and bypasses authentication.
96 Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
97 """
99 SCOPE = (
100 "https://www.googleapis.com/auth/devstorage.full_control",
101 "https://www.googleapis.com/auth/devstorage.read_only",
102 "https://www.googleapis.com/auth/devstorage.read_write",
103 )
104 """The scopes required for authenticating as a Cloud Storage consumer."""
106 def __init__(
107 self,
108 project=_marker,
109 credentials=None,
110 _http=None,
111 client_info=None,
112 client_options=None,
113 use_auth_w_custom_endpoint=True,
114 ):
115 self._base_connection = None
117 if project is None:
118 no_project = True
119 project = "<none>"
120 else:
121 no_project = False
123 if project is _marker:
124 project = None
126 # Save the initial value of constructor arguments before they
127 # are passed along, for use in __reduce__ defined elsewhere.
128 self._initial_client_info = client_info
129 self._initial_client_options = client_options
130 self._initial_credentials = credentials
132 kw_args = {"client_info": client_info}
134 # `api_endpoint` should be only set by the user via `client_options`,
135 # or if the _get_storage_host() returns a non-default value (_is_emulator_set).
136 # `api_endpoint` plays an important role for mTLS, if it is not set,
137 # then mTLS logic will be applied to decide which endpoint will be used.
138 storage_host = _get_storage_host()
139 _is_emulator_set = storage_host != _DEFAULT_STORAGE_HOST
140 kw_args["api_endpoint"] = storage_host if _is_emulator_set else None
142 if client_options:
143 if isinstance(client_options, dict):
144 client_options = google.api_core.client_options.from_dict(
145 client_options
146 )
147 if client_options.api_endpoint:
148 api_endpoint = client_options.api_endpoint
149 kw_args["api_endpoint"] = api_endpoint
151 # If a custom endpoint is set, the client checks for credentials
152 # or finds the default credentials based on the current environment.
153 # Authentication may be bypassed under certain conditions:
154 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
155 # (2) use_auth_w_custom_endpoint is set to False.
156 if kw_args["api_endpoint"] is not None:
157 if _is_emulator_set or not use_auth_w_custom_endpoint:
158 if credentials is None:
159 credentials = AnonymousCredentials()
160 if project is None:
161 project = _get_environ_project()
162 if project is None:
163 no_project = True
164 project = "<none>"
166 super(Client, self).__init__(
167 project=project,
168 credentials=credentials,
169 client_options=client_options,
170 _http=_http,
171 )
173 if no_project:
174 self.project = None
176 self._connection = Connection(self, **kw_args)
177 self._batch_stack = _LocalStack()
179 @classmethod
180 def create_anonymous_client(cls):
181 """Factory: return client with anonymous credentials.
183 .. note::
185 Such a client has only limited access to "public" buckets:
186 listing their contents and downloading their blobs.
188 :rtype: :class:`google.cloud.storage.client.Client`
189 :returns: Instance w/ anonymous credentials and no project.
190 """
191 client = cls(project="<none>", credentials=AnonymousCredentials())
192 client.project = None
193 return client
195 @property
196 def _connection(self):
197 """Get connection or batch on the client.
199 :rtype: :class:`google.cloud.storage._http.Connection`
200 :returns: The connection set on the client, or the batch
201 if one is set.
202 """
203 if self.current_batch is not None:
204 return self.current_batch
205 else:
206 return self._base_connection
208 @_connection.setter
209 def _connection(self, value):
210 """Set connection on the client.
212 Intended to be used by constructor (since the base class calls)
213 self._connection = connection
214 Will raise if the connection is set more than once.
216 :type value: :class:`google.cloud.storage._http.Connection`
217 :param value: The connection set on the client.
219 :raises: :class:`ValueError` if connection has already been set.
220 """
221 if self._base_connection is not None:
222 raise ValueError("Connection already set on client")
223 self._base_connection = value
225 def _push_batch(self, batch):
226 """Push a batch onto our stack.
228 "Protected", intended for use by batch context mgrs.
230 :type batch: :class:`google.cloud.storage.batch.Batch`
231 :param batch: newly-active batch
232 """
233 self._batch_stack.push(batch)
235 def _pop_batch(self):
236 """Pop a batch from our stack.
238 "Protected", intended for use by batch context mgrs.
240 :raises: IndexError if the stack is empty.
241 :rtype: :class:`google.cloud.storage.batch.Batch`
242 :returns: the top-most batch/transaction, after removing it.
243 """
244 return self._batch_stack.pop()
246 @property
247 def current_batch(self):
248 """Currently-active batch.
250 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if
251 no batch is active).
252 :returns: The batch at the top of the batch stack.
253 """
254 return self._batch_stack.top
256 def get_service_account_email(
257 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
258 ):
259 """Get the email address of the project's GCS service account
261 :type project: str
262 :param project:
263 (Optional) Project ID to use for retreiving GCS service account
264 email address. Defaults to the client's project.
265 :type timeout: float or tuple
266 :param timeout:
267 (Optional) The amount of time, in seconds, to wait
268 for the server response. See: :ref:`configuring_timeouts`
270 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
271 :param retry:
272 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
274 :rtype: str
275 :returns: service account email address
276 """
277 if project is None:
278 project = self.project
280 path = f"/projects/{project}/serviceAccount"
281 api_response = self._get_resource(path, timeout=timeout, retry=retry)
282 return api_response["email_address"]
284 def bucket(self, bucket_name, user_project=None):
285 """Factory constructor for bucket object.
287 .. note::
288 This will not make an HTTP request; it simply instantiates
289 a bucket object owned by this client.
291 :type bucket_name: str
292 :param bucket_name: The name of the bucket to be instantiated.
294 :type user_project: str
295 :param user_project: (Optional) The project ID to be billed for API
296 requests made via the bucket.
298 :rtype: :class:`google.cloud.storage.bucket.Bucket`
299 :returns: The bucket object created.
300 """
301 return Bucket(client=self, name=bucket_name, user_project=user_project)
303 def batch(self, raise_exception=True):
304 """Factory constructor for batch object.
306 .. note::
307 This will not make an HTTP request; it simply instantiates
308 a batch object owned by this client.
310 :type raise_exception: bool
311 :param raise_exception:
312 (Optional) Defaults to True. If True, instead of adding exceptions
313 to the list of return responses, the final exception will be raised.
314 Note that exceptions are unwrapped after all operations are complete
315 in success or failure, and only the last exception is raised.
317 :rtype: :class:`google.cloud.storage.batch.Batch`
318 :returns: The batch object created.
319 """
320 return Batch(client=self, raise_exception=raise_exception)
322 def _get_resource(
323 self,
324 path,
325 query_params=None,
326 headers=None,
327 timeout=_DEFAULT_TIMEOUT,
328 retry=DEFAULT_RETRY,
329 _target_object=None,
330 ):
331 """Helper for bucket / blob methods making API 'GET' calls.
333 Args:
334 path str:
335 The path of the resource to fetch.
337 query_params Optional[dict]:
338 HTTP query parameters to be passed
340 headers Optional[dict]:
341 HTTP headers to be passed
343 timeout (Optional[Union[float, Tuple[float, float]]]):
344 The amount of time, in seconds, to wait for the server response.
346 Can also be passed as a tuple (connect_timeout, read_timeout).
347 See :meth:`requests.Session.request` documentation for details.
349 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
350 How to retry the RPC. A None value will disable retries.
351 A google.api_core.retry.Retry value will enable retries, and the object will
352 define retriable response codes and errors and configure backoff and timeout options.
354 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
355 activates it only if certain conditions are met. This class exists to provide safe defaults
356 for RPC calls that are not technically safe to retry normally (due to potential data
357 duplication or other side-effects) but become safe to retry if a condition such as
358 if_metageneration_match is set.
360 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
361 information on retry types and how to configure them.
363 _target_object (Union[ \
364 :class:`~google.cloud.storage.bucket.Bucket`, \
365 :class:`~google.cloud.storage.bucket.blob`, \
366 ]):
367 Object to which future data is to be applied -- only relevant
368 in the context of a batch.
370 Returns:
371 dict
372 The JSON resource fetched
374 Raises:
375 google.cloud.exceptions.NotFound
376 If the bucket is not found.
377 """
378 return self._connection.api_request(
379 method="GET",
380 path=path,
381 query_params=query_params,
382 headers=headers,
383 timeout=timeout,
384 retry=retry,
385 _target_object=_target_object,
386 )
388 def _list_resource(
389 self,
390 path,
391 item_to_value,
392 page_token=None,
393 max_results=None,
394 extra_params=None,
395 page_start=page_iterator._do_nothing_page_start,
396 page_size=None,
397 timeout=_DEFAULT_TIMEOUT,
398 retry=DEFAULT_RETRY,
399 ):
400 api_request = functools.partial(
401 self._connection.api_request, timeout=timeout, retry=retry
402 )
403 return page_iterator.HTTPIterator(
404 client=self,
405 api_request=api_request,
406 path=path,
407 item_to_value=item_to_value,
408 page_token=page_token,
409 max_results=max_results,
410 extra_params=extra_params,
411 page_start=page_start,
412 page_size=page_size,
413 )
415 def _patch_resource(
416 self,
417 path,
418 data,
419 query_params=None,
420 headers=None,
421 timeout=_DEFAULT_TIMEOUT,
422 retry=None,
423 _target_object=None,
424 ):
425 """Helper for bucket / blob methods making API 'PATCH' calls.
427 Args:
428 path str:
429 The path of the resource to fetch.
431 data dict:
432 The data to be patched.
434 query_params Optional[dict]:
435 HTTP query parameters to be passed
437 headers Optional[dict]:
438 HTTP headers to be passed
440 timeout (Optional[Union[float, Tuple[float, float]]]):
441 The amount of time, in seconds, to wait for the server response.
443 Can also be passed as a tuple (connect_timeout, read_timeout).
444 See :meth:`requests.Session.request` documentation for details.
446 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
447 How to retry the RPC. A None value will disable retries.
448 A google.api_core.retry.Retry value will enable retries, and the object will
449 define retriable response codes and errors and configure backoff and timeout options.
451 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
452 activates it only if certain conditions are met. This class exists to provide safe defaults
453 for RPC calls that are not technically safe to retry normally (due to potential data
454 duplication or other side-effects) but become safe to retry if a condition such as
455 if_metageneration_match is set.
457 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
458 information on retry types and how to configure them.
460 _target_object (Union[ \
461 :class:`~google.cloud.storage.bucket.Bucket`, \
462 :class:`~google.cloud.storage.bucket.blob`, \
463 ]):
464 Object to which future data is to be applied -- only relevant
465 in the context of a batch.
467 Returns:
468 dict
469 The JSON resource fetched
471 Raises:
472 google.cloud.exceptions.NotFound
473 If the bucket is not found.
474 """
475 return self._connection.api_request(
476 method="PATCH",
477 path=path,
478 data=data,
479 query_params=query_params,
480 headers=headers,
481 timeout=timeout,
482 retry=retry,
483 _target_object=_target_object,
484 )
486 def _put_resource(
487 self,
488 path,
489 data,
490 query_params=None,
491 headers=None,
492 timeout=_DEFAULT_TIMEOUT,
493 retry=None,
494 _target_object=None,
495 ):
496 """Helper for bucket / blob methods making API 'PUT' calls.
498 Args:
499 path str:
500 The path of the resource to fetch.
502 data dict:
503 The data to be patched.
505 query_params Optional[dict]:
506 HTTP query parameters to be passed
508 headers Optional[dict]:
509 HTTP headers to be passed
511 timeout (Optional[Union[float, Tuple[float, float]]]):
512 The amount of time, in seconds, to wait for the server response.
514 Can also be passed as a tuple (connect_timeout, read_timeout).
515 See :meth:`requests.Session.request` documentation for details.
517 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
518 How to retry the RPC. A None value will disable retries.
519 A google.api_core.retry.Retry value will enable retries, and the object will
520 define retriable response codes and errors and configure backoff and timeout options.
522 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
523 activates it only if certain conditions are met. This class exists to provide safe defaults
524 for RPC calls that are not technically safe to retry normally (due to potential data
525 duplication or other side-effects) but become safe to retry if a condition such as
526 if_metageneration_match is set.
528 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
529 information on retry types and how to configure them.
531 _target_object (Union[ \
532 :class:`~google.cloud.storage.bucket.Bucket`, \
533 :class:`~google.cloud.storage.bucket.blob`, \
534 ]):
535 Object to which future data is to be applied -- only relevant
536 in the context of a batch.
538 Returns:
539 dict
540 The JSON resource fetched
542 Raises:
543 google.cloud.exceptions.NotFound
544 If the bucket is not found.
545 """
546 return self._connection.api_request(
547 method="PUT",
548 path=path,
549 data=data,
550 query_params=query_params,
551 headers=headers,
552 timeout=timeout,
553 retry=retry,
554 _target_object=_target_object,
555 )
557 def _post_resource(
558 self,
559 path,
560 data,
561 query_params=None,
562 headers=None,
563 timeout=_DEFAULT_TIMEOUT,
564 retry=None,
565 _target_object=None,
566 ):
567 """Helper for bucket / blob methods making API 'POST' calls.
569 Args:
570 path str:
571 The path of the resource to which to post.
573 data dict:
574 The data to be posted.
576 query_params Optional[dict]:
577 HTTP query parameters to be passed
579 headers Optional[dict]:
580 HTTP headers to be passed
582 timeout (Optional[Union[float, Tuple[float, float]]]):
583 The amount of time, in seconds, to wait for the server response.
585 Can also be passed as a tuple (connect_timeout, read_timeout).
586 See :meth:`requests.Session.request` documentation for details.
588 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
589 How to retry the RPC. A None value will disable retries.
590 A google.api_core.retry.Retry value will enable retries, and the object will
591 define retriable response codes and errors and configure backoff and timeout options.
593 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
594 activates it only if certain conditions are met. This class exists to provide safe defaults
595 for RPC calls that are not technically safe to retry normally (due to potential data
596 duplication or other side-effects) but become safe to retry if a condition such as
597 if_metageneration_match is set.
599 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
600 information on retry types and how to configure them.
602 _target_object (Union[ \
603 :class:`~google.cloud.storage.bucket.Bucket`, \
604 :class:`~google.cloud.storage.bucket.blob`, \
605 ]):
606 Object to which future data is to be applied -- only relevant
607 in the context of a batch.
609 Returns:
610 dict
611 The JSON resource returned from the post.
613 Raises:
614 google.cloud.exceptions.NotFound
615 If the bucket is not found.
616 """
618 return self._connection.api_request(
619 method="POST",
620 path=path,
621 data=data,
622 query_params=query_params,
623 headers=headers,
624 timeout=timeout,
625 retry=retry,
626 _target_object=_target_object,
627 )
629 def _delete_resource(
630 self,
631 path,
632 query_params=None,
633 headers=None,
634 timeout=_DEFAULT_TIMEOUT,
635 retry=DEFAULT_RETRY,
636 _target_object=None,
637 ):
638 """Helper for bucket / blob methods making API 'DELETE' calls.
640 Args:
641 path str:
642 The path of the resource to delete.
644 query_params Optional[dict]:
645 HTTP query parameters to be passed
647 headers Optional[dict]:
648 HTTP headers to be passed
650 timeout (Optional[Union[float, Tuple[float, float]]]):
651 The amount of time, in seconds, to wait for the server response.
653 Can also be passed as a tuple (connect_timeout, read_timeout).
654 See :meth:`requests.Session.request` documentation for details.
656 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
657 How to retry the RPC. A None value will disable retries.
658 A google.api_core.retry.Retry value will enable retries, and the object will
659 define retriable response codes and errors and configure backoff and timeout options.
661 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
662 activates it only if certain conditions are met. This class exists to provide safe defaults
663 for RPC calls that are not technically safe to retry normally (due to potential data
664 duplication or other side-effects) but become safe to retry if a condition such as
665 if_metageneration_match is set.
667 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
668 information on retry types and how to configure them.
670 _target_object (Union[ \
671 :class:`~google.cloud.storage.bucket.Bucket`, \
672 :class:`~google.cloud.storage.bucket.blob`, \
673 ]):
674 Object to which future data is to be applied -- only relevant
675 in the context of a batch.
677 Returns:
678 dict
679 The JSON resource fetched
681 Raises:
682 google.cloud.exceptions.NotFound
683 If the bucket is not found.
684 """
685 return self._connection.api_request(
686 method="DELETE",
687 path=path,
688 query_params=query_params,
689 headers=headers,
690 timeout=timeout,
691 retry=retry,
692 _target_object=_target_object,
693 )
695 def _bucket_arg_to_bucket(self, bucket_or_name):
696 """Helper to return given bucket or create new by name.
698 Args:
699 bucket_or_name (Union[ \
700 :class:`~google.cloud.storage.bucket.Bucket`, \
701 str, \
702 ]):
703 The bucket resource to pass or name to create.
705 Returns:
706 google.cloud.storage.bucket.Bucket
707 The newly created bucket or the given one.
708 """
709 if isinstance(bucket_or_name, Bucket):
710 bucket = bucket_or_name
711 if bucket.client is None:
712 bucket._client = self
713 else:
714 bucket = Bucket(self, name=bucket_or_name)
715 return bucket
717 def get_bucket(
718 self,
719 bucket_or_name,
720 timeout=_DEFAULT_TIMEOUT,
721 if_metageneration_match=None,
722 if_metageneration_not_match=None,
723 retry=DEFAULT_RETRY,
724 ):
725 """Retrieve a bucket via a GET request.
727 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python).
729 Args:
730 bucket_or_name (Union[ \
731 :class:`~google.cloud.storage.bucket.Bucket`, \
732 str, \
733 ]):
734 The bucket resource to pass or name to create.
736 timeout (Optional[Union[float, Tuple[float, float]]]):
737 The amount of time, in seconds, to wait for the server response.
739 Can also be passed as a tuple (connect_timeout, read_timeout).
740 See :meth:`requests.Session.request` documentation for details.
742 if_metageneration_match (Optional[long]):
743 Make the operation conditional on whether the
744 blob's current metageneration matches the given value.
746 if_metageneration_not_match (Optional[long]):
747 Make the operation conditional on whether the blob's
748 current metageneration does not match the given value.
750 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
751 How to retry the RPC. A None value will disable retries.
752 A google.api_core.retry.Retry value will enable retries, and the object will
753 define retriable response codes and errors and configure backoff and timeout options.
755 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
756 activates it only if certain conditions are met. This class exists to provide safe defaults
757 for RPC calls that are not technically safe to retry normally (due to potential data
758 duplication or other side-effects) but become safe to retry if a condition such as
759 if_metageneration_match is set.
761 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
762 information on retry types and how to configure them.
764 Returns:
765 google.cloud.storage.bucket.Bucket
766 The bucket matching the name provided.
768 Raises:
769 google.cloud.exceptions.NotFound
770 If the bucket is not found.
771 """
772 bucket = self._bucket_arg_to_bucket(bucket_or_name)
773 bucket.reload(
774 client=self,
775 timeout=timeout,
776 if_metageneration_match=if_metageneration_match,
777 if_metageneration_not_match=if_metageneration_not_match,
778 retry=retry,
779 )
780 return bucket
782 def lookup_bucket(
783 self,
784 bucket_name,
785 timeout=_DEFAULT_TIMEOUT,
786 if_metageneration_match=None,
787 if_metageneration_not_match=None,
788 retry=DEFAULT_RETRY,
789 ):
790 """Get a bucket by name, returning None if not found.
792 You can use this if you would rather check for a None value
793 than catching a NotFound exception.
795 :type bucket_name: str
796 :param bucket_name: The name of the bucket to get.
798 :type timeout: float or tuple
799 :param timeout:
800 (Optional) The amount of time, in seconds, to wait
801 for the server response. See: :ref:`configuring_timeouts`
803 :type if_metageneration_match: long
804 :param if_metageneration_match: (Optional) Make the operation conditional on whether the
805 blob's current metageneration matches the given value.
807 :type if_metageneration_not_match: long
808 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
809 blob's current metageneration does not match the given value.
811 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
812 :param retry:
813 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
815 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType``
816 :returns: The bucket matching the name provided or None if not found.
817 """
818 try:
819 return self.get_bucket(
820 bucket_name,
821 timeout=timeout,
822 if_metageneration_match=if_metageneration_match,
823 if_metageneration_not_match=if_metageneration_not_match,
824 retry=retry,
825 )
826 except NotFound:
827 return None
829 def create_bucket(
830 self,
831 bucket_or_name,
832 requester_pays=None,
833 project=None,
834 user_project=None,
835 location=None,
836 data_locations=None,
837 predefined_acl=None,
838 predefined_default_object_acl=None,
839 timeout=_DEFAULT_TIMEOUT,
840 retry=DEFAULT_RETRY,
841 ):
842 """Create a new bucket via a POST request.
844 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python).
846 Args:
847 bucket_or_name (Union[ \
848 :class:`~google.cloud.storage.bucket.Bucket`, \
849 str, \
850 ]):
851 The bucket resource to pass or name to create.
852 requester_pays (bool):
853 DEPRECATED. Use Bucket().requester_pays instead.
854 (Optional) Whether requester pays for API requests for
855 this bucket and its blobs.
856 project (str):
857 (Optional) The project under which the bucket is to be created.
858 If not passed, uses the project set on the client.
859 user_project (str):
860 (Optional) The project ID to be billed for API requests
861 made via created bucket.
862 location (str):
863 (Optional) The location of the bucket. If not passed,
864 the default location, US, will be used. If specifying a dual-region,
865 `data_locations` should be set in conjunction. See:
866 https://cloud.google.com/storage/docs/locations
867 data_locations (list of str):
868 (Optional) The list of regional locations of a custom dual-region bucket.
869 Dual-regions require exactly 2 regional locations. See:
870 https://cloud.google.com/storage/docs/locations
871 predefined_acl (str):
872 (Optional) Name of predefined ACL to apply to bucket. See:
873 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
874 predefined_default_object_acl (str):
875 (Optional) Name of predefined ACL to apply to bucket's objects. See:
876 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
877 timeout (Optional[Union[float, Tuple[float, float]]]):
878 The amount of time, in seconds, to wait for the server response.
880 Can also be passed as a tuple (connect_timeout, read_timeout).
881 See :meth:`requests.Session.request` documentation for details.
883 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
884 How to retry the RPC. A None value will disable retries.
885 A google.api_core.retry.Retry value will enable retries, and the object will
886 define retriable response codes and errors and configure backoff and timeout options.
888 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
889 activates it only if certain conditions are met. This class exists to provide safe defaults
890 for RPC calls that are not technically safe to retry normally (due to potential data
891 duplication or other side-effects) but become safe to retry if a condition such as
892 if_metageneration_match is set.
894 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
895 information on retry types and how to configure them.
897 Returns:
898 google.cloud.storage.bucket.Bucket
899 The newly created bucket.
901 Raises:
902 google.cloud.exceptions.Conflict
903 If the bucket already exists.
904 """
905 bucket = self._bucket_arg_to_bucket(bucket_or_name)
906 query_params = {}
908 if project is None:
909 project = self.project
911 # Use no project if STORAGE_EMULATOR_HOST is set
912 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
913 if _is_emulator_set:
914 if project is None:
915 project = _get_environ_project()
916 if project is None:
917 project = "<none>"
919 # Only include the project parameter if a project is set.
920 # If a project is not set, falls back to API validation (BadRequest).
921 if project is not None:
922 query_params = {"project": project}
924 if requester_pays is not None:
925 warnings.warn(
926 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.",
927 PendingDeprecationWarning,
928 stacklevel=1,
929 )
930 bucket.requester_pays = requester_pays
932 if predefined_acl is not None:
933 predefined_acl = BucketACL.validate_predefined(predefined_acl)
934 query_params["predefinedAcl"] = predefined_acl
936 if predefined_default_object_acl is not None:
937 predefined_default_object_acl = DefaultObjectACL.validate_predefined(
938 predefined_default_object_acl
939 )
940 query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl
942 if user_project is not None:
943 query_params["userProject"] = user_project
945 properties = {key: bucket._properties[key] for key in bucket._changes}
946 properties["name"] = bucket.name
948 if location is not None:
949 properties["location"] = location
951 if data_locations is not None:
952 properties["customPlacementConfig"] = {"dataLocations": data_locations}
954 api_response = self._post_resource(
955 "/b",
956 properties,
957 query_params=query_params,
958 timeout=timeout,
959 retry=retry,
960 _target_object=bucket,
961 )
963 bucket._set_properties(api_response)
964 return bucket
966 def download_blob_to_file(
967 self,
968 blob_or_uri,
969 file_obj,
970 start=None,
971 end=None,
972 raw_download=False,
973 if_etag_match=None,
974 if_etag_not_match=None,
975 if_generation_match=None,
976 if_generation_not_match=None,
977 if_metageneration_match=None,
978 if_metageneration_not_match=None,
979 timeout=_DEFAULT_TIMEOUT,
980 checksum="md5",
981 retry=DEFAULT_RETRY,
982 ):
983 """Download the contents of a blob object or blob URI into a file-like object.
985 See https://cloud.google.com/storage/docs/downloading-objects
987 Args:
988 blob_or_uri (Union[ \
989 :class:`~google.cloud.storage.blob.Blob`, \
990 str, \
991 ]):
992 The blob resource to pass or URI to download.
994 file_obj (file):
995 A file handle to which to write the blob's data.
997 start (int):
998 (Optional) The first byte in a range to be downloaded.
1000 end (int):
1001 (Optional) The last byte in a range to be downloaded.
1003 raw_download (bool):
1004 (Optional) If true, download the object without any expansion.
1006 if_etag_match (Union[str, Set[str]]):
1007 (Optional) See :ref:`using-if-etag-match`
1009 if_etag_not_match (Union[str, Set[str]]):
1010 (Optional) See :ref:`using-if-etag-not-match`
1012 if_generation_match (long):
1013 (Optional) See :ref:`using-if-generation-match`
1015 if_generation_not_match (long):
1016 (Optional) See :ref:`using-if-generation-not-match`
1018 if_metageneration_match (long):
1019 (Optional) See :ref:`using-if-metageneration-match`
1021 if_metageneration_not_match (long):
1022 (Optional) See :ref:`using-if-metageneration-not-match`
1024 timeout ([Union[float, Tuple[float, float]]]):
1025 (Optional) The amount of time, in seconds, to wait
1026 for the server response. See: :ref:`configuring_timeouts`
1028 checksum (str):
1029 (Optional) The type of checksum to compute to verify the integrity
1030 of the object. The response headers must contain a checksum of the
1031 requested type. If the headers lack an appropriate checksum (for
1032 instance in the case of transcoded or ranged downloads where the
1033 remote service does not know the correct checksum, including
1034 downloads where chunk_size is set) an INFO-level log will be
1035 emitted. Supported values are "md5", "crc32c" and None. The default
1036 is "md5".
1037 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy)
1038 (Optional) How to retry the RPC. A None value will disable
1039 retries. A google.api_core.retry.Retry value will enable retries,
1040 and the object will define retriable response codes and errors and
1041 configure backoff and timeout options.
1043 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1044 Retry object and activates it only if certain conditions are met.
1045 This class exists to provide safe defaults for RPC calls that are
1046 not technically safe to retry normally (due to potential data
1047 duplication or other side-effects) but become safe to retry if a
1048 condition such as if_metageneration_match is set.
1050 See the retry.py source code and docstrings in this package
1051 (google.cloud.storage.retry) for information on retry types and how
1052 to configure them.
1054 Media operations (downloads and uploads) do not support non-default
1055 predicates in a Retry object. The default will always be used. Other
1056 configuration changes for Retry objects such as delays and deadlines
1057 are respected.
1058 """
1060 if not isinstance(blob_or_uri, Blob):
1061 blob_or_uri = Blob.from_string(blob_or_uri)
1063 blob_or_uri._prep_and_do_download(
1064 file_obj,
1065 client=self,
1066 start=start,
1067 end=end,
1068 raw_download=raw_download,
1069 if_etag_match=if_etag_match,
1070 if_etag_not_match=if_etag_not_match,
1071 if_generation_match=if_generation_match,
1072 if_generation_not_match=if_generation_not_match,
1073 if_metageneration_match=if_metageneration_match,
1074 if_metageneration_not_match=if_metageneration_not_match,
1075 timeout=timeout,
1076 checksum=checksum,
1077 retry=retry,
1078 )
1080 def list_blobs(
1081 self,
1082 bucket_or_name,
1083 max_results=None,
1084 page_token=None,
1085 prefix=None,
1086 delimiter=None,
1087 start_offset=None,
1088 end_offset=None,
1089 include_trailing_delimiter=None,
1090 versions=None,
1091 projection="noAcl",
1092 fields=None,
1093 page_size=None,
1094 timeout=_DEFAULT_TIMEOUT,
1095 retry=DEFAULT_RETRY,
1096 match_glob=None,
1097 ):
1098 """Return an iterator used to find blobs in the bucket.
1100 If :attr:`user_project` is set, bills the API request to that project.
1102 .. note::
1103 List prefixes (directories) in a bucket using a prefix and delimiter.
1104 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python)
1105 listing objects using a prefix filter.
1107 Args:
1108 bucket_or_name (Union[ \
1109 :class:`~google.cloud.storage.bucket.Bucket`, \
1110 str, \
1111 ]):
1112 The bucket resource to pass or name to create.
1114 max_results (int):
1115 (Optional) The maximum number of blobs to return.
1117 page_token (str):
1118 (Optional) If present, return the next batch of blobs, using the
1119 value, which must correspond to the ``nextPageToken`` value
1120 returned in the previous response. Deprecated: use the ``pages``
1121 property of the returned iterator instead of manually passing the
1122 token.
1124 prefix (str):
1125 (Optional) Prefix used to filter blobs.
1127 delimiter (str):
1128 (Optional) Delimiter, used with ``prefix`` to
1129 emulate hierarchy.
1131 start_offset (str):
1132 (Optional) Filter results to objects whose names are
1133 lexicographically equal to or after ``startOffset``. If
1134 ``endOffset`` is also set, the objects listed will have names
1135 between ``startOffset`` (inclusive) and ``endOffset``
1136 (exclusive).
1138 end_offset (str):
1139 (Optional) Filter results to objects whose names are
1140 lexicographically before ``endOffset``. If ``startOffset`` is
1141 also set, the objects listed will have names between
1142 ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
1144 include_trailing_delimiter (boolean):
1145 (Optional) If true, objects that end in exactly one instance of
1146 ``delimiter`` will have their metadata included in ``items`` in
1147 addition to ``prefixes``.
1149 versions (bool):
1150 (Optional) Whether object versions should be returned
1151 as separate blobs.
1153 projection (str):
1154 (Optional) If used, must be 'full' or 'noAcl'.
1155 Defaults to ``'noAcl'``. Specifies the set of
1156 properties to return.
1158 fields (str):
1159 (Optional) Selector specifying which fields to include
1160 in a partial response. Must be a list of fields. For
1161 example to get a partial response with just the next
1162 page token and the name and language of each blob returned:
1163 ``'items(name,contentLanguage),nextPageToken'``.
1164 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
1166 page_size (int):
1167 (Optional) Maximum number of blobs to return in each page.
1168 Defaults to a value set by the API.
1170 timeout (Optional[Union[float, Tuple[float, float]]]):
1171 The amount of time, in seconds, to wait for the server response.
1173 Can also be passed as a tuple (connect_timeout, read_timeout).
1174 See :meth:`requests.Session.request` documentation for details.
1176 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1177 How to retry the RPC. A None value will disable retries.
1178 A google.api_core.retry.Retry value will enable retries, and the object will
1179 define retriable response codes and errors and configure backoff and timeout options.
1181 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1182 activates it only if certain conditions are met. This class exists to provide safe defaults
1183 for RPC calls that are not technically safe to retry normally (due to potential data
1184 duplication or other side-effects) but become safe to retry if a condition such as
1185 if_metageneration_match is set.
1187 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1188 information on retry types and how to configure them.
1190 match_glob (str):
1191 (Optional) A glob pattern used to filter results (for example, foo*bar).
1192 The string value must be UTF-8 encoded. See:
1193 https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
1195 Returns:
1196 Iterator of all :class:`~google.cloud.storage.blob.Blob`
1197 in this bucket matching the arguments. The RPC call
1198 returns a response when the iterator is consumed.
1200 As part of the response, you'll also get back an iterator.prefixes entity that lists object names
1201 up to and including the requested delimiter. Duplicate entries are omitted from this list.
1202 """
1203 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1205 extra_params = {"projection": projection}
1207 if prefix is not None:
1208 extra_params["prefix"] = prefix
1210 if delimiter is not None:
1211 extra_params["delimiter"] = delimiter
1213 if match_glob is not None:
1214 extra_params["matchGlob"] = match_glob
1216 if start_offset is not None:
1217 extra_params["startOffset"] = start_offset
1219 if end_offset is not None:
1220 extra_params["endOffset"] = end_offset
1222 if include_trailing_delimiter is not None:
1223 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1225 if versions is not None:
1226 extra_params["versions"] = versions
1228 if fields is not None:
1229 extra_params["fields"] = fields
1231 if bucket.user_project is not None:
1232 extra_params["userProject"] = bucket.user_project
1234 path = bucket.path + "/o"
1235 iterator = self._list_resource(
1236 path,
1237 _item_to_blob,
1238 page_token=page_token,
1239 max_results=max_results,
1240 extra_params=extra_params,
1241 page_start=_blobs_page_start,
1242 page_size=page_size,
1243 timeout=timeout,
1244 retry=retry,
1245 )
1246 iterator.bucket = bucket
1247 iterator.prefixes = set()
1248 return iterator
1250 def list_buckets(
1251 self,
1252 max_results=None,
1253 page_token=None,
1254 prefix=None,
1255 projection="noAcl",
1256 fields=None,
1257 project=None,
1258 page_size=None,
1259 timeout=_DEFAULT_TIMEOUT,
1260 retry=DEFAULT_RETRY,
1261 ):
1262 """Get all buckets in the project associated to the client.
1264 This will not populate the list of blobs available in each
1265 bucket.
1267 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python).
1269 :type max_results: int
1270 :param max_results: (Optional) The maximum number of buckets to return.
1272 :type page_token: str
1273 :param page_token:
1274 (Optional) If present, return the next batch of buckets, using the
1275 value, which must correspond to the ``nextPageToken`` value
1276 returned in the previous response. Deprecated: use the ``pages``
1277 property of the returned iterator instead of manually passing the
1278 token.
1280 :type prefix: str
1281 :param prefix: (Optional) Filter results to buckets whose names begin
1282 with this prefix.
1284 :type projection: str
1285 :param projection:
1286 (Optional) Specifies the set of properties to return. If used, must
1287 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1289 :type fields: str
1290 :param fields:
1291 (Optional) Selector specifying which fields to include in a partial
1292 response. Must be a list of fields. For example to get a partial
1293 response with just the next page token and the language of each
1294 bucket returned: 'items/id,nextPageToken'
1296 :type project: str
1297 :param project: (Optional) The project whose buckets are to be listed.
1298 If not passed, uses the project set on the client.
1300 :type page_size: int
1301 :param page_size: (Optional) Maximum number of buckets to return in each page.
1302 Defaults to a value set by the API.
1304 :type timeout: float or tuple
1305 :param timeout:
1306 (Optional) The amount of time, in seconds, to wait
1307 for the server response. See: :ref:`configuring_timeouts`
1309 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1310 :param retry:
1311 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1313 :rtype: :class:`~google.api_core.page_iterator.Iterator`
1314 :raises ValueError: if both ``project`` is ``None`` and the client's
1315 project is also ``None``.
1316 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
1317 belonging to this project.
1318 """
1319 extra_params = {}
1321 if project is None:
1322 project = self.project
1324 # Use no project if STORAGE_EMULATOR_HOST is set
1325 _is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
1326 if _is_emulator_set:
1327 if project is None:
1328 project = _get_environ_project()
1329 if project is None:
1330 project = "<none>"
1332 # Only include the project parameter if a project is set.
1333 # If a project is not set, falls back to API validation (BadRequest).
1334 if project is not None:
1335 extra_params = {"project": project}
1337 if prefix is not None:
1338 extra_params["prefix"] = prefix
1340 extra_params["projection"] = projection
1342 if fields is not None:
1343 extra_params["fields"] = fields
1345 return self._list_resource(
1346 "/b",
1347 _item_to_bucket,
1348 page_token=page_token,
1349 max_results=max_results,
1350 extra_params=extra_params,
1351 page_size=page_size,
1352 timeout=timeout,
1353 retry=retry,
1354 )
1356 def create_hmac_key(
1357 self,
1358 service_account_email,
1359 project_id=None,
1360 user_project=None,
1361 timeout=_DEFAULT_TIMEOUT,
1362 retry=None,
1363 ):
1364 """Create an HMAC key for a service account.
1366 :type service_account_email: str
1367 :param service_account_email: e-mail address of the service account
1369 :type project_id: str
1370 :param project_id: (Optional) Explicit project ID for the key.
1371 Defaults to the client's project.
1373 :type user_project: str
1374 :param user_project: (Optional) This parameter is currently ignored.
1376 :type timeout: float or tuple
1377 :param timeout:
1378 (Optional) The amount of time, in seconds, to wait
1379 for the server response. See: :ref:`configuring_timeouts`
1381 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1382 :param retry: (Optional) How to retry the RPC. A None value will disable retries.
1383 A google.api_core.retry.Retry value will enable retries, and the object will
1384 define retriable response codes and errors and configure backoff and timeout options.
1386 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1387 activates it only if certain conditions are met. This class exists to provide safe defaults
1388 for RPC calls that are not technically safe to retry normally (due to potential data
1389 duplication or other side-effects) but become safe to retry if a condition such as
1390 if_metageneration_match is set.
1392 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1393 information on retry types and how to configure them.
1395 :rtype:
1396 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1397 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1398 """
1399 if project_id is None:
1400 project_id = self.project
1402 path = f"/projects/{project_id}/hmacKeys"
1403 qs_params = {"serviceAccountEmail": service_account_email}
1405 if user_project is not None:
1406 qs_params["userProject"] = user_project
1408 api_response = self._post_resource(
1409 path,
1410 None,
1411 query_params=qs_params,
1412 timeout=timeout,
1413 retry=retry,
1414 )
1415 metadata = HMACKeyMetadata(self)
1416 metadata._properties = api_response["metadata"]
1417 secret = api_response["secret"]
1418 return metadata, secret
1420 def list_hmac_keys(
1421 self,
1422 max_results=None,
1423 service_account_email=None,
1424 show_deleted_keys=None,
1425 project_id=None,
1426 user_project=None,
1427 timeout=_DEFAULT_TIMEOUT,
1428 retry=DEFAULT_RETRY,
1429 ):
1430 """List HMAC keys for a project.
1432 :type max_results: int
1433 :param max_results:
1434 (Optional) Max number of keys to return in a given page.
1436 :type service_account_email: str
1437 :param service_account_email:
1438 (Optional) Limit keys to those created by the given service account.
1440 :type show_deleted_keys: bool
1441 :param show_deleted_keys:
1442 (Optional) Included deleted keys in the list. Default is to
1443 exclude them.
1445 :type project_id: str
1446 :param project_id: (Optional) Explicit project ID for the key.
1447 Defaults to the client's project.
1449 :type user_project: str
1450 :param user_project: (Optional) This parameter is currently ignored.
1452 :type timeout: float or tuple
1453 :param timeout:
1454 (Optional) The amount of time, in seconds, to wait
1455 for the server response. See: :ref:`configuring_timeouts`
1457 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1458 :param retry:
1459 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1461 :rtype:
1462 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1463 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1464 """
1465 if project_id is None:
1466 project_id = self.project
1468 path = f"/projects/{project_id}/hmacKeys"
1469 extra_params = {}
1471 if service_account_email is not None:
1472 extra_params["serviceAccountEmail"] = service_account_email
1474 if show_deleted_keys is not None:
1475 extra_params["showDeletedKeys"] = show_deleted_keys
1477 if user_project is not None:
1478 extra_params["userProject"] = user_project
1480 return self._list_resource(
1481 path,
1482 _item_to_hmac_key_metadata,
1483 max_results=max_results,
1484 extra_params=extra_params,
1485 timeout=timeout,
1486 retry=retry,
1487 )
1489 def get_hmac_key_metadata(
1490 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT
1491 ):
1492 """Return a metadata instance for the given HMAC key.
1494 :type access_id: str
1495 :param access_id: Unique ID of an existing key.
1497 :type project_id: str
1498 :param project_id: (Optional) Project ID of an existing key.
1499 Defaults to client's project.
1501 :type timeout: float or tuple
1502 :param timeout:
1503 (Optional) The amount of time, in seconds, to wait
1504 for the server response. See: :ref:`configuring_timeouts`
1506 :type user_project: str
1507 :param user_project: (Optional) This parameter is currently ignored.
1508 """
1509 metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
1510 metadata.reload(timeout=timeout) # raises NotFound for missing key
1511 return metadata
1513 def generate_signed_post_policy_v4(
1514 self,
1515 bucket_name,
1516 blob_name,
1517 expiration,
1518 conditions=None,
1519 fields=None,
1520 credentials=None,
1521 virtual_hosted_style=False,
1522 bucket_bound_hostname=None,
1523 scheme="http",
1524 service_account_email=None,
1525 access_token=None,
1526 ):
1527 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request.
1529 .. note::
1531 Assumes ``credentials`` implements the
1532 :class:`google.auth.credentials.Signing` interface. Also assumes
1533 ``credentials`` has a ``service_account_email`` property which
1534 identifies the credentials.
1536 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py).
1538 :type bucket_name: str
1539 :param bucket_name: Bucket name.
1541 :type blob_name: str
1542 :param blob_name: Object name.
1544 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
1545 :param expiration: Policy expiration time. If a ``datetime`` instance is
1546 passed without an explicit ``tzinfo`` set, it will be
1547 assumed to be ``UTC``.
1549 :type conditions: list
1550 :param conditions: (Optional) List of POST policy conditions, which are
1551 used to restrict what is allowed in the request.
1553 :type fields: dict
1554 :param fields: (Optional) Additional elements to include into request.
1556 :type credentials: :class:`google.auth.credentials.Signing`
1557 :param credentials: (Optional) Credentials object with an associated private
1558 key to sign text.
1560 :type virtual_hosted_style: bool
1561 :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket
1562 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
1564 :type bucket_bound_hostname: str
1565 :param bucket_bound_hostname:
1566 (Optional) If passed, construct the URL relative to the bucket-bound hostname.
1567 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
1568 See: https://cloud.google.com/storage/docs/request-endpoints#cname
1570 :type scheme: str
1571 :param scheme:
1572 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
1573 this value as a scheme. ``https`` will work only when using a CDN.
1574 Defaults to ``"http"``.
1576 :type service_account_email: str
1577 :param service_account_email: (Optional) E-mail address of the service account.
1579 :type access_token: str
1580 :param access_token: (Optional) Access token for a service account.
1582 :rtype: dict
1583 :returns: Signed POST policy.
1584 """
1585 credentials = self._credentials if credentials is None else credentials
1586 ensure_signed_credentials(credentials)
1588 # prepare policy conditions and fields
1589 timestamp, datestamp = get_v4_now_dtstamps()
1591 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
1592 email=credentials.signer_email, datestamp=datestamp
1593 )
1594 required_conditions = [
1595 {"bucket": bucket_name},
1596 {"key": blob_name},
1597 {"x-goog-date": timestamp},
1598 {"x-goog-credential": x_goog_credential},
1599 {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
1600 ]
1602 conditions = conditions or []
1603 policy_fields = {}
1604 for key, value in sorted((fields or {}).items()):
1605 if not key.startswith("x-ignore-"):
1606 policy_fields[key] = value
1607 conditions.append({key: value})
1609 conditions += required_conditions
1611 # calculate policy expiration time
1612 now = _NOW()
1613 if expiration is None:
1614 expiration = now + datetime.timedelta(hours=1)
1616 policy_expires = now + datetime.timedelta(
1617 seconds=get_expiration_seconds_v4(expiration)
1618 )
1620 # encode policy for signing
1621 policy = json.dumps(
1622 collections.OrderedDict(
1623 sorted(
1624 {
1625 "conditions": conditions,
1626 "expiration": policy_expires.isoformat() + "Z",
1627 }.items()
1628 )
1629 ),
1630 separators=(",", ":"),
1631 )
1632 str_to_sign = base64.b64encode(policy.encode("utf-8"))
1634 # sign the policy and get its cryptographic signature
1635 if access_token and service_account_email:
1636 signature = _sign_message(str_to_sign, access_token, service_account_email)
1637 signature_bytes = base64.b64decode(signature)
1638 else:
1639 signature_bytes = credentials.sign_bytes(str_to_sign)
1641 # get hexadecimal representation of the signature
1642 signature = binascii.hexlify(signature_bytes).decode("utf-8")
1644 policy_fields.update(
1645 {
1646 "key": blob_name,
1647 "x-goog-algorithm": "GOOG4-RSA-SHA256",
1648 "x-goog-credential": x_goog_credential,
1649 "x-goog-date": timestamp,
1650 "x-goog-signature": signature,
1651 "policy": str_to_sign.decode("utf-8"),
1652 }
1653 )
1654 # designate URL
1655 if virtual_hosted_style:
1656 url = f"https://{bucket_name}.storage.googleapis.com/"
1657 elif bucket_bound_hostname:
1658 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/"
1659 else:
1660 url = f"https://storage.googleapis.com/{bucket_name}/"
1662 return {"url": url, "fields": policy_fields}
1665def _item_to_bucket(iterator, item):
1666 """Convert a JSON bucket to the native object.
1668 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1669 :param iterator: The iterator that has retrieved the item.
1671 :type item: dict
1672 :param item: An item to be converted to a bucket.
1674 :rtype: :class:`.Bucket`
1675 :returns: The next bucket in the page.
1676 """
1677 name = item.get("name")
1678 bucket = Bucket(iterator.client, name)
1679 bucket._set_properties(item)
1680 return bucket
1683def _item_to_hmac_key_metadata(iterator, item):
1684 """Convert a JSON key metadata resource to the native object.
1686 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1687 :param iterator: The iterator that has retrieved the item.
1689 :type item: dict
1690 :param item: An item to be converted to a key metadata instance.
1692 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`
1693 :returns: The next key metadata instance in the page.
1694 """
1695 metadata = HMACKeyMetadata(iterator.client)
1696 metadata._properties = item
1697 return metadata