Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Client for interacting with the Google Cloud Storage API."""
17import base64
18import binascii
19import collections
20import datetime
21import functools
22import json
23import warnings
24import google.api_core.client_options
26from google.auth.credentials import AnonymousCredentials
28from google.api_core import page_iterator
29from google.cloud._helpers import _LocalStack
30from google.cloud.client import ClientWithProject
31from google.cloud.exceptions import NotFound
33from google.cloud.storage._helpers import _add_generation_match_parameters
34from google.cloud.storage._helpers import _bucket_bound_hostname_url
35from google.cloud.storage._helpers import _get_api_endpoint_override
36from google.cloud.storage._helpers import _get_environ_project
37from google.cloud.storage._helpers import _get_storage_emulator_override
38from google.cloud.storage._helpers import _use_client_cert
39from google.cloud.storage._helpers import _virtual_hosted_style_base_url
40from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
41from google.cloud.storage._helpers import _DEFAULT_SCHEME
42from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
43from google.cloud.storage._helpers import _NOW
44from google.cloud.storage._helpers import _UTC
45from google.cloud.storage._opentelemetry_tracing import create_trace_span
47from google.cloud.storage._http import Connection
48from google.cloud.storage._signing import (
49 get_expiration_seconds_v4,
50 get_v4_now_dtstamps,
51 ensure_signed_credentials,
52 _sign_message,
53)
54from google.cloud.storage.batch import Batch
55from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start
56from google.cloud.storage.blob import Blob
57from google.cloud.storage.hmac_key import HMACKeyMetadata
58from google.cloud.storage.acl import BucketACL
59from google.cloud.storage.acl import DefaultObjectACL
60from google.cloud.storage.constants import _DEFAULT_TIMEOUT
61from google.cloud.storage.retry import DEFAULT_RETRY
64_marker = object()
67class Client(ClientWithProject):
68 """Client to bundle configuration needed for API requests.
70 :type project: str or None
71 :param project: the project which the client acts on behalf of. Will be
72 passed when creating a topic. If not passed,
73 falls back to the default inferred from the environment.
75 :type credentials: :class:`~google.auth.credentials.Credentials`
76 :param credentials: (Optional) The OAuth2 Credentials to use for this
77 client. If not passed (and if no ``_http`` object is
78 passed), falls back to the default inferred from the
79 environment.
81 :type _http: :class:`~requests.Session`
82 :param _http: (Optional) HTTP object to make requests. Can be any object
83 that defines ``request()`` with the same interface as
84 :meth:`requests.Session.request`. If not passed, an
85 ``_http`` object is created that is bound to the
86 ``credentials`` for the current object.
87 This parameter should be considered private, and could
88 change in the future.
90 :type client_info: :class:`~google.api_core.client_info.ClientInfo`
91 :param client_info:
92 The client info used to send a user-agent string along with API
93 requests. If ``None``, then default info will be used. Generally,
94 you only need to set this if you're developing your own library
95 or partner tool.
97 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
98 :param client_options: (Optional) Client options used to set user options on the client.
99 A non-default universe domain or api endpoint should be set through client_options.
101 :type use_auth_w_custom_endpoint: bool
102 :param use_auth_w_custom_endpoint:
103 (Optional) Whether authentication is required under custom endpoints.
104 If false, uses AnonymousCredentials and bypasses authentication.
105 Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
107 :type extra_headers: dict
108 :param extra_headers:
109 (Optional) Custom headers to be sent with the requests attached to the client.
110 For example, you can add custom audit logging headers.
112 :type api_key: string
113 :param api_key:
114 (Optional) An API key. Mutually exclusive with any other credentials.
115 This parameter is an alias for setting `client_options.api_key` and
116 will supercede any api key set in the `client_options` parameter.
117 """
119 SCOPE = (
120 "https://www.googleapis.com/auth/devstorage.full_control",
121 "https://www.googleapis.com/auth/devstorage.read_only",
122 "https://www.googleapis.com/auth/devstorage.read_write",
123 )
124 """The scopes required for authenticating as a Cloud Storage consumer."""
126 def __init__(
127 self,
128 project=_marker,
129 credentials=None,
130 _http=None,
131 client_info=None,
132 client_options=None,
133 use_auth_w_custom_endpoint=True,
134 extra_headers={},
135 *,
136 api_key=None,
137 ):
138 self._base_connection = None
140 if project is None:
141 no_project = True
142 project = "<none>"
143 else:
144 no_project = False
146 if project is _marker:
147 project = None
149 # Save the initial value of constructor arguments before they
150 # are passed along, for use in __reduce__ defined elsewhere.
151 self._initial_client_info = client_info
152 self._initial_client_options = client_options
153 self._extra_headers = extra_headers
155 connection_kw_args = {"client_info": client_info}
157 # api_key should set client_options.api_key. Set it here whether
158 # client_options was specified as a dict, as a ClientOptions object, or
159 # None.
160 if api_key:
161 if client_options and not isinstance(client_options, dict):
162 client_options.api_key = api_key
163 else:
164 if not client_options:
165 client_options = {}
166 client_options["api_key"] = api_key
168 if client_options:
169 if isinstance(client_options, dict):
170 client_options = google.api_core.client_options.from_dict(
171 client_options
172 )
174 if client_options and client_options.universe_domain:
175 self._universe_domain = client_options.universe_domain
176 else:
177 self._universe_domain = None
179 storage_emulator_override = _get_storage_emulator_override()
180 api_endpoint_override = _get_api_endpoint_override()
182 # Determine the api endpoint. The rules are as follows:
184 # 1. If the `api_endpoint` is set in `client_options`, use that as the
185 # endpoint.
186 if client_options and client_options.api_endpoint:
187 api_endpoint = client_options.api_endpoint
189 # 2. Elif the "STORAGE_EMULATOR_HOST" env var is set, then use that as the
190 # endpoint.
191 elif storage_emulator_override:
192 api_endpoint = storage_emulator_override
194 # 3. Elif the "API_ENDPOINT_OVERRIDE" env var is set, then use that as the
195 # endpoint.
196 elif api_endpoint_override:
197 api_endpoint = api_endpoint_override
199 # 4. Elif the `universe_domain` is set in `client_options`,
200 # create the endpoint using that as the default.
201 #
202 # Mutual TLS is not compatible with a non-default universe domain
203 # at this time. If such settings are enabled along with the
204 # "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable, a ValueError will
205 # be raised.
207 elif self._universe_domain:
208 # The final decision of whether to use mTLS takes place in
209 # google-auth-library-python. We peek at the environment variable
210 # here only to issue an exception in case of a conflict.
211 if _use_client_cert():
212 raise ValueError(
213 'The "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable is '
214 'set to "true" and a non-default universe domain is '
215 "configured. mTLS is not supported in any universe other than"
216 "googleapis.com."
217 )
218 api_endpoint = _DEFAULT_SCHEME + _STORAGE_HOST_TEMPLATE.format(
219 universe_domain=self._universe_domain
220 )
222 # 5. Else, use the default, which is to use the default
223 # universe domain of "googleapis.com" and create the endpoint
224 # "storage.googleapis.com" from that.
225 else:
226 api_endpoint = None
228 connection_kw_args["api_endpoint"] = api_endpoint
230 self._is_emulator_set = True if storage_emulator_override else False
232 # If a custom endpoint is set, the client checks for credentials
233 # or finds the default credentials based on the current environment.
234 # Authentication may be bypassed under certain conditions:
235 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
236 # (2) use_auth_w_custom_endpoint is set to False.
237 if connection_kw_args["api_endpoint"] is not None:
238 if self._is_emulator_set or not use_auth_w_custom_endpoint:
239 if credentials is None:
240 credentials = AnonymousCredentials()
241 if project is None:
242 project = _get_environ_project()
243 if project is None:
244 no_project = True
245 project = "<none>"
247 super(Client, self).__init__(
248 project=project,
249 credentials=credentials,
250 client_options=client_options,
251 _http=_http,
252 )
254 # Validate that the universe domain of the credentials matches the
255 # universe domain of the client.
256 if self._credentials.universe_domain != self.universe_domain:
257 raise ValueError(
258 "The configured universe domain ({client_ud}) does not match "
259 "the universe domain found in the credentials ({cred_ud}). If "
260 "you haven't configured the universe domain explicitly, "
261 "`googleapis.com` is the default.".format(
262 client_ud=self.universe_domain,
263 cred_ud=self._credentials.universe_domain,
264 )
265 )
267 if no_project:
268 self.project = None
270 # Pass extra_headers to Connection
271 connection = Connection(self, **connection_kw_args)
272 connection.extra_headers = extra_headers
273 self._connection = connection
274 self._batch_stack = _LocalStack()
276 @classmethod
277 def create_anonymous_client(cls):
278 """Factory: return client with anonymous credentials.
280 .. note::
282 Such a client has only limited access to "public" buckets:
283 listing their contents and downloading their blobs.
285 :rtype: :class:`google.cloud.storage.client.Client`
286 :returns: Instance w/ anonymous credentials and no project.
287 """
288 client = cls(project="<none>", credentials=AnonymousCredentials())
289 client.project = None
290 return client
292 @property
293 def universe_domain(self):
294 return self._universe_domain or _DEFAULT_UNIVERSE_DOMAIN
296 @property
297 def api_endpoint(self):
298 return self._connection.API_BASE_URL
300 @property
301 def _connection(self):
302 """Get connection or batch on the client.
304 :rtype: :class:`google.cloud.storage._http.Connection`
305 :returns: The connection set on the client, or the batch
306 if one is set.
307 """
308 if self.current_batch is not None:
309 return self.current_batch
310 else:
311 return self._base_connection
313 @_connection.setter
314 def _connection(self, value):
315 """Set connection on the client.
317 Intended to be used by constructor (since the base class calls)
318 self._connection = connection
319 Will raise if the connection is set more than once.
321 :type value: :class:`google.cloud.storage._http.Connection`
322 :param value: The connection set on the client.
324 :raises: :class:`ValueError` if connection has already been set.
325 """
326 if self._base_connection is not None:
327 raise ValueError("Connection already set on client")
328 self._base_connection = value
330 def _push_batch(self, batch):
331 """Push a batch onto our stack.
333 "Protected", intended for use by batch context mgrs.
335 :type batch: :class:`google.cloud.storage.batch.Batch`
336 :param batch: newly-active batch
337 """
338 self._batch_stack.push(batch)
340 def _pop_batch(self):
341 """Pop a batch from our stack.
343 "Protected", intended for use by batch context mgrs.
345 :raises: IndexError if the stack is empty.
346 :rtype: :class:`google.cloud.storage.batch.Batch`
347 :returns: the top-most batch/transaction, after removing it.
348 """
349 return self._batch_stack.pop()
351 @property
352 def current_batch(self):
353 """Currently-active batch.
355 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if
356 no batch is active).
357 :returns: The batch at the top of the batch stack.
358 """
359 return self._batch_stack.top
361 def get_service_account_email(
362 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
363 ):
364 """Get the email address of the project's GCS service account
366 :type project: str
367 :param project:
368 (Optional) Project ID to use for retreiving GCS service account
369 email address. Defaults to the client's project.
370 :type timeout: float or tuple
371 :param timeout:
372 (Optional) The amount of time, in seconds, to wait
373 for the server response. See: :ref:`configuring_timeouts`
375 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
376 :param retry:
377 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
379 :rtype: str
380 :returns: service account email address
381 """
382 with create_trace_span(name="Storage.Client.getServiceAccountEmail"):
383 if project is None:
384 project = self.project
386 path = f"/projects/{project}/serviceAccount"
387 api_response = self._get_resource(path, timeout=timeout, retry=retry)
388 return api_response["email_address"]
390 def bucket(self, bucket_name, user_project=None, generation=None):
391 """Factory constructor for bucket object.
393 .. note::
394 This will not make an HTTP request; it simply instantiates
395 a bucket object owned by this client.
397 :type bucket_name: str
398 :param bucket_name: The name of the bucket to be instantiated.
400 :type user_project: str
401 :param user_project: (Optional) The project ID to be billed for API
402 requests made via the bucket.
404 :type generation: int
405 :param generation: (Optional) If present, selects a specific revision of
406 this bucket.
408 :rtype: :class:`google.cloud.storage.bucket.Bucket`
409 :returns: The bucket object created.
410 """
411 return Bucket(
412 client=self,
413 name=bucket_name,
414 user_project=user_project,
415 generation=generation,
416 )
418 def batch(self, raise_exception=True):
419 """Factory constructor for batch object.
421 .. note::
422 This will not make an HTTP request; it simply instantiates
423 a batch object owned by this client.
425 :type raise_exception: bool
426 :param raise_exception:
427 (Optional) Defaults to True. If True, instead of adding exceptions
428 to the list of return responses, the final exception will be raised.
429 Note that exceptions are unwrapped after all operations are complete
430 in success or failure, and only the last exception is raised.
432 :rtype: :class:`google.cloud.storage.batch.Batch`
433 :returns: The batch object created.
434 """
435 return Batch(client=self, raise_exception=raise_exception)
437 def _get_resource(
438 self,
439 path,
440 query_params=None,
441 headers=None,
442 timeout=_DEFAULT_TIMEOUT,
443 retry=DEFAULT_RETRY,
444 _target_object=None,
445 ):
446 """Helper for bucket / blob methods making API 'GET' calls.
448 Args:
449 path str:
450 The path of the resource to fetch.
452 query_params Optional[dict]:
453 HTTP query parameters to be passed
455 headers Optional[dict]:
456 HTTP headers to be passed
458 timeout (Optional[Union[float, Tuple[float, float]]]):
459 The amount of time, in seconds, to wait for the server response.
461 Can also be passed as a tuple (connect_timeout, read_timeout).
462 See :meth:`requests.Session.request` documentation for details.
464 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
465 How to retry the RPC. A None value will disable retries.
466 A google.api_core.retry.Retry value will enable retries, and the object will
467 define retriable response codes and errors and configure backoff and timeout options.
469 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
470 activates it only if certain conditions are met. This class exists to provide safe defaults
471 for RPC calls that are not technically safe to retry normally (due to potential data
472 duplication or other side-effects) but become safe to retry if a condition such as
473 if_metageneration_match is set.
475 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
476 information on retry types and how to configure them.
478 _target_object (Union[ \
479 :class:`~google.cloud.storage.bucket.Bucket`, \
480 :class:`~google.cloud.storage.bucket.blob`, \
481 ]):
482 Object to which future data is to be applied -- only relevant
483 in the context of a batch.
485 Returns:
486 dict
487 The JSON resource fetched
489 Raises:
490 google.cloud.exceptions.NotFound
491 If the bucket is not found.
492 """
493 return self._connection.api_request(
494 method="GET",
495 path=path,
496 query_params=query_params,
497 headers=headers,
498 timeout=timeout,
499 retry=retry,
500 _target_object=_target_object,
501 )
503 def _list_resource(
504 self,
505 path,
506 item_to_value,
507 page_token=None,
508 max_results=None,
509 extra_params=None,
510 page_start=page_iterator._do_nothing_page_start,
511 page_size=None,
512 timeout=_DEFAULT_TIMEOUT,
513 retry=DEFAULT_RETRY,
514 ):
515 kwargs = {
516 "method": "GET",
517 "path": path,
518 "timeout": timeout,
519 }
520 with create_trace_span(
521 name="Storage.Client._list_resource_returns_iterator",
522 client=self,
523 api_request=kwargs,
524 retry=retry,
525 ):
526 api_request = functools.partial(
527 self._connection.api_request, timeout=timeout, retry=retry
528 )
529 return page_iterator.HTTPIterator(
530 client=self,
531 api_request=api_request,
532 path=path,
533 item_to_value=item_to_value,
534 page_token=page_token,
535 max_results=max_results,
536 extra_params=extra_params,
537 page_start=page_start,
538 page_size=page_size,
539 )
541 def _patch_resource(
542 self,
543 path,
544 data,
545 query_params=None,
546 headers=None,
547 timeout=_DEFAULT_TIMEOUT,
548 retry=None,
549 _target_object=None,
550 ):
551 """Helper for bucket / blob methods making API 'PATCH' calls.
553 Args:
554 path str:
555 The path of the resource to fetch.
557 data dict:
558 The data to be patched.
560 query_params Optional[dict]:
561 HTTP query parameters to be passed
563 headers Optional[dict]:
564 HTTP headers to be passed
566 timeout (Optional[Union[float, Tuple[float, float]]]):
567 The amount of time, in seconds, to wait for the server response.
569 Can also be passed as a tuple (connect_timeout, read_timeout).
570 See :meth:`requests.Session.request` documentation for details.
572 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
573 How to retry the RPC. A None value will disable retries.
574 A google.api_core.retry.Retry value will enable retries, and the object will
575 define retriable response codes and errors and configure backoff and timeout options.
577 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
578 activates it only if certain conditions are met. This class exists to provide safe defaults
579 for RPC calls that are not technically safe to retry normally (due to potential data
580 duplication or other side-effects) but become safe to retry if a condition such as
581 if_metageneration_match is set.
583 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
584 information on retry types and how to configure them.
586 _target_object (Union[ \
587 :class:`~google.cloud.storage.bucket.Bucket`, \
588 :class:`~google.cloud.storage.bucket.blob`, \
589 ]):
590 Object to which future data is to be applied -- only relevant
591 in the context of a batch.
593 Returns:
594 dict
595 The JSON resource fetched
597 Raises:
598 google.cloud.exceptions.NotFound
599 If the bucket is not found.
600 """
601 return self._connection.api_request(
602 method="PATCH",
603 path=path,
604 data=data,
605 query_params=query_params,
606 headers=headers,
607 timeout=timeout,
608 retry=retry,
609 _target_object=_target_object,
610 )
612 def _put_resource(
613 self,
614 path,
615 data,
616 query_params=None,
617 headers=None,
618 timeout=_DEFAULT_TIMEOUT,
619 retry=None,
620 _target_object=None,
621 ):
622 """Helper for bucket / blob methods making API 'PUT' calls.
624 Args:
625 path str:
626 The path of the resource to fetch.
628 data dict:
629 The data to be patched.
631 query_params Optional[dict]:
632 HTTP query parameters to be passed
634 headers Optional[dict]:
635 HTTP headers to be passed
637 timeout (Optional[Union[float, Tuple[float, float]]]):
638 The amount of time, in seconds, to wait for the server response.
640 Can also be passed as a tuple (connect_timeout, read_timeout).
641 See :meth:`requests.Session.request` documentation for details.
643 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
644 How to retry the RPC. A None value will disable retries.
645 A google.api_core.retry.Retry value will enable retries, and the object will
646 define retriable response codes and errors and configure backoff and timeout options.
648 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
649 activates it only if certain conditions are met. This class exists to provide safe defaults
650 for RPC calls that are not technically safe to retry normally (due to potential data
651 duplication or other side-effects) but become safe to retry if a condition such as
652 if_metageneration_match is set.
654 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
655 information on retry types and how to configure them.
657 _target_object (Union[ \
658 :class:`~google.cloud.storage.bucket.Bucket`, \
659 :class:`~google.cloud.storage.bucket.blob`, \
660 ]):
661 Object to which future data is to be applied -- only relevant
662 in the context of a batch.
664 Returns:
665 dict
666 The JSON resource fetched
668 Raises:
669 google.cloud.exceptions.NotFound
670 If the bucket is not found.
671 """
672 return self._connection.api_request(
673 method="PUT",
674 path=path,
675 data=data,
676 query_params=query_params,
677 headers=headers,
678 timeout=timeout,
679 retry=retry,
680 _target_object=_target_object,
681 )
683 def _post_resource(
684 self,
685 path,
686 data,
687 query_params=None,
688 headers=None,
689 timeout=_DEFAULT_TIMEOUT,
690 retry=None,
691 _target_object=None,
692 ):
693 """Helper for bucket / blob methods making API 'POST' calls.
695 Args:
696 path str:
697 The path of the resource to which to post.
699 data dict:
700 The data to be posted.
702 query_params Optional[dict]:
703 HTTP query parameters to be passed
705 headers Optional[dict]:
706 HTTP headers to be passed
708 timeout (Optional[Union[float, Tuple[float, float]]]):
709 The amount of time, in seconds, to wait for the server response.
711 Can also be passed as a tuple (connect_timeout, read_timeout).
712 See :meth:`requests.Session.request` documentation for details.
714 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
715 How to retry the RPC. A None value will disable retries.
716 A google.api_core.retry.Retry value will enable retries, and the object will
717 define retriable response codes and errors and configure backoff and timeout options.
719 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
720 activates it only if certain conditions are met. This class exists to provide safe defaults
721 for RPC calls that are not technically safe to retry normally (due to potential data
722 duplication or other side-effects) but become safe to retry if a condition such as
723 if_metageneration_match is set.
725 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
726 information on retry types and how to configure them.
728 _target_object (Union[ \
729 :class:`~google.cloud.storage.bucket.Bucket`, \
730 :class:`~google.cloud.storage.bucket.blob`, \
731 ]):
732 Object to which future data is to be applied -- only relevant
733 in the context of a batch.
735 Returns:
736 dict
737 The JSON resource returned from the post.
739 Raises:
740 google.cloud.exceptions.NotFound
741 If the bucket is not found.
742 """
744 return self._connection.api_request(
745 method="POST",
746 path=path,
747 data=data,
748 query_params=query_params,
749 headers=headers,
750 timeout=timeout,
751 retry=retry,
752 _target_object=_target_object,
753 )
755 def _delete_resource(
756 self,
757 path,
758 query_params=None,
759 headers=None,
760 timeout=_DEFAULT_TIMEOUT,
761 retry=DEFAULT_RETRY,
762 _target_object=None,
763 ):
764 """Helper for bucket / blob methods making API 'DELETE' calls.
766 Args:
767 path str:
768 The path of the resource to delete.
770 query_params Optional[dict]:
771 HTTP query parameters to be passed
773 headers Optional[dict]:
774 HTTP headers to be passed
776 timeout (Optional[Union[float, Tuple[float, float]]]):
777 The amount of time, in seconds, to wait for the server response.
779 Can also be passed as a tuple (connect_timeout, read_timeout).
780 See :meth:`requests.Session.request` documentation for details.
782 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
783 How to retry the RPC. A None value will disable retries.
784 A google.api_core.retry.Retry value will enable retries, and the object will
785 define retriable response codes and errors and configure backoff and timeout options.
787 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
788 activates it only if certain conditions are met. This class exists to provide safe defaults
789 for RPC calls that are not technically safe to retry normally (due to potential data
790 duplication or other side-effects) but become safe to retry if a condition such as
791 if_metageneration_match is set.
793 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
794 information on retry types and how to configure them.
796 _target_object (Union[ \
797 :class:`~google.cloud.storage.bucket.Bucket`, \
798 :class:`~google.cloud.storage.bucket.blob`, \
799 ]):
800 Object to which future data is to be applied -- only relevant
801 in the context of a batch.
803 Returns:
804 dict
805 The JSON resource fetched
807 Raises:
808 google.cloud.exceptions.NotFound
809 If the bucket is not found.
810 """
811 return self._connection.api_request(
812 method="DELETE",
813 path=path,
814 query_params=query_params,
815 headers=headers,
816 timeout=timeout,
817 retry=retry,
818 _target_object=_target_object,
819 )
821 def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
822 """Helper to return given bucket or create new by name.
824 Args:
825 bucket_or_name (Union[ \
826 :class:`~google.cloud.storage.bucket.Bucket`, \
827 str, \
828 ]):
829 The bucket resource to pass or name to create.
830 generation (Optional[int]):
831 The bucket generation. If generation is specified,
832 bucket_or_name must be a name (str).
834 Returns:
835 google.cloud.storage.bucket.Bucket
836 The newly created bucket or the given one.
837 """
838 if isinstance(bucket_or_name, Bucket):
839 if generation:
840 raise ValueError(
841 "The generation can only be specified if a "
842 "name is used to specify a bucket, not a Bucket object. "
843 "Create a new Bucket object with the correct generation "
844 "instead."
845 )
846 bucket = bucket_or_name
847 if bucket.client is None:
848 bucket._client = self
849 else:
850 bucket = Bucket(self, name=bucket_or_name, generation=generation)
851 return bucket
853 def get_bucket(
854 self,
855 bucket_or_name,
856 timeout=_DEFAULT_TIMEOUT,
857 if_metageneration_match=None,
858 if_metageneration_not_match=None,
859 retry=DEFAULT_RETRY,
860 *,
861 generation=None,
862 soft_deleted=None,
863 ):
864 """Retrieve a bucket via a GET request.
866 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python).
868 Args:
869 bucket_or_name (Union[ \
870 :class:`~google.cloud.storage.bucket.Bucket`, \
871 str, \
872 ]):
873 The bucket resource to pass or name to create.
875 timeout (Optional[Union[float, Tuple[float, float]]]):
876 The amount of time, in seconds, to wait for the server response.
878 Can also be passed as a tuple (connect_timeout, read_timeout).
879 See :meth:`requests.Session.request` documentation for details.
881 if_metageneration_match (Optional[int]):
882 Make the operation conditional on whether the
883 bucket's current metageneration matches the given value.
885 if_metageneration_not_match (Optional[int]):
886 Make the operation conditional on whether the bucket's
887 current metageneration does not match the given value.
889 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
890 How to retry the RPC. A None value will disable retries.
891 A google.api_core.retry.Retry value will enable retries, and the object will
892 define retriable response codes and errors and configure backoff and timeout options.
894 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
895 activates it only if certain conditions are met. This class exists to provide safe defaults
896 for RPC calls that are not technically safe to retry normally (due to potential data
897 duplication or other side-effects) but become safe to retry if a condition such as
898 if_metageneration_match is set.
900 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
901 information on retry types and how to configure them.
903 generation (Optional[int]):
904 The generation of the bucket. The generation can be used to
905 specify a specific soft-deleted version of the bucket, in
906 conjunction with the ``soft_deleted`` argument below. If
907 ``soft_deleted`` is not True, the generation is unused.
909 soft_deleted (Optional[bool]):
910 If True, looks for a soft-deleted bucket. Will only return
911 the bucket metadata if the bucket exists and is in a
912 soft-deleted state. The bucket ``generation`` is required if
913 ``soft_deleted`` is set to True.
914 See: https://cloud.google.com/storage/docs/soft-delete
916 Returns:
917 google.cloud.storage.bucket.Bucket
918 The bucket matching the name provided.
920 Raises:
921 google.cloud.exceptions.NotFound
922 If the bucket is not found.
923 """
924 with create_trace_span(name="Storage.Client.getBucket"):
925 bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
926 bucket.reload(
927 client=self,
928 timeout=timeout,
929 if_metageneration_match=if_metageneration_match,
930 if_metageneration_not_match=if_metageneration_not_match,
931 retry=retry,
932 soft_deleted=soft_deleted,
933 )
934 return bucket
936 def lookup_bucket(
937 self,
938 bucket_name,
939 timeout=_DEFAULT_TIMEOUT,
940 if_metageneration_match=None,
941 if_metageneration_not_match=None,
942 retry=DEFAULT_RETRY,
943 ):
944 """Get a bucket by name, returning None if not found.
946 You can use this if you would rather check for a None value
947 than catching a NotFound exception.
949 :type bucket_name: str
950 :param bucket_name: The name of the bucket to get.
952 :type timeout: float or tuple
953 :param timeout:
954 (Optional) The amount of time, in seconds, to wait
955 for the server response. See: :ref:`configuring_timeouts`
957 :type if_metageneration_match: long
958 :param if_metageneration_match: (Optional) Make the operation conditional on whether the
959 blob's current metageneration matches the given value.
961 :type if_metageneration_not_match: long
962 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
963 blob's current metageneration does not match the given value.
965 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
966 :param retry:
967 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
969 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType``
970 :returns: The bucket matching the name provided or None if not found.
971 """
972 with create_trace_span(name="Storage.Client.lookupBucket"):
973 try:
974 return self.get_bucket(
975 bucket_name,
976 timeout=timeout,
977 if_metageneration_match=if_metageneration_match,
978 if_metageneration_not_match=if_metageneration_not_match,
979 retry=retry,
980 )
981 except NotFound:
982 return None
984 def create_bucket(
985 self,
986 bucket_or_name,
987 requester_pays=None,
988 project=None,
989 user_project=None,
990 location=None,
991 data_locations=None,
992 predefined_acl=None,
993 predefined_default_object_acl=None,
994 enable_object_retention=False,
995 timeout=_DEFAULT_TIMEOUT,
996 retry=DEFAULT_RETRY,
997 ):
998 """Create a new bucket via a POST request.
1000 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python).
1002 Args:
1003 bucket_or_name (Union[ \
1004 :class:`~google.cloud.storage.bucket.Bucket`, \
1005 str, \
1006 ]):
1007 The bucket resource to pass or name to create.
1008 requester_pays (bool):
1009 DEPRECATED. Use Bucket().requester_pays instead.
1010 (Optional) Whether requester pays for API requests for
1011 this bucket and its blobs.
1012 project (str):
1013 (Optional) The project under which the bucket is to be created.
1014 If not passed, uses the project set on the client.
1015 user_project (str):
1016 (Optional) The project ID to be billed for API requests
1017 made via created bucket.
1018 location (str):
1019 (Optional) The location of the bucket. If not passed,
1020 the default location, US, will be used. If specifying a dual-region,
1021 `data_locations` should be set in conjunction. See:
1022 https://cloud.google.com/storage/docs/locations
1023 data_locations (list of str):
1024 (Optional) The list of regional locations of a custom dual-region bucket.
1025 Dual-regions require exactly 2 regional locations. See:
1026 https://cloud.google.com/storage/docs/locations
1027 predefined_acl (str):
1028 (Optional) Name of predefined ACL to apply to bucket. See:
1029 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1030 predefined_default_object_acl (str):
1031 (Optional) Name of predefined ACL to apply to bucket's objects. See:
1032 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1033 enable_object_retention (bool):
1034 (Optional) Whether object retention should be enabled on this bucket. See:
1035 https://cloud.google.com/storage/docs/object-lock
1036 timeout (Optional[Union[float, Tuple[float, float]]]):
1037 The amount of time, in seconds, to wait for the server response.
1039 Can also be passed as a tuple (connect_timeout, read_timeout).
1040 See :meth:`requests.Session.request` documentation for details.
1042 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1043 How to retry the RPC. A None value will disable retries.
1044 A google.api_core.retry.Retry value will enable retries, and the object will
1045 define retriable response codes and errors and configure backoff and timeout options.
1047 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1048 activates it only if certain conditions are met. This class exists to provide safe defaults
1049 for RPC calls that are not technically safe to retry normally (due to potential data
1050 duplication or other side-effects) but become safe to retry if a condition such as
1051 if_metageneration_match is set.
1053 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1054 information on retry types and how to configure them.
1056 Returns:
1057 google.cloud.storage.bucket.Bucket
1058 The newly created bucket.
1060 Raises:
1061 google.cloud.exceptions.Conflict
1062 If the bucket already exists.
1063 """
1064 with create_trace_span(name="Storage.Client.createBucket"):
1065 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1066 query_params = {}
1068 if project is None:
1069 project = self.project
1071 # Use no project if STORAGE_EMULATOR_HOST is set
1072 if self._is_emulator_set:
1073 if project is None:
1074 project = _get_environ_project()
1075 if project is None:
1076 project = "<none>"
1078 # Only include the project parameter if a project is set.
1079 # If a project is not set, falls back to API validation (BadRequest).
1080 if project is not None:
1081 query_params = {"project": project}
1083 if requester_pays is not None:
1084 warnings.warn(
1085 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.",
1086 PendingDeprecationWarning,
1087 stacklevel=1,
1088 )
1089 bucket.requester_pays = requester_pays
1091 if predefined_acl is not None:
1092 predefined_acl = BucketACL.validate_predefined(predefined_acl)
1093 query_params["predefinedAcl"] = predefined_acl
1095 if predefined_default_object_acl is not None:
1096 predefined_default_object_acl = DefaultObjectACL.validate_predefined(
1097 predefined_default_object_acl
1098 )
1099 query_params[
1100 "predefinedDefaultObjectAcl"
1101 ] = predefined_default_object_acl
1103 if user_project is not None:
1104 query_params["userProject"] = user_project
1106 if enable_object_retention:
1107 query_params["enableObjectRetention"] = enable_object_retention
1109 properties = {key: bucket._properties[key] for key in bucket._changes}
1110 properties["name"] = bucket.name
1112 if location is not None:
1113 properties["location"] = location
1115 if data_locations is not None:
1116 properties["customPlacementConfig"] = {"dataLocations": data_locations}
1118 api_response = self._post_resource(
1119 "/b",
1120 properties,
1121 query_params=query_params,
1122 timeout=timeout,
1123 retry=retry,
1124 _target_object=bucket,
1125 )
1127 bucket._set_properties(api_response)
1128 return bucket
1130 def download_blob_to_file(
1131 self,
1132 blob_or_uri,
1133 file_obj,
1134 start=None,
1135 end=None,
1136 raw_download=False,
1137 if_etag_match=None,
1138 if_etag_not_match=None,
1139 if_generation_match=None,
1140 if_generation_not_match=None,
1141 if_metageneration_match=None,
1142 if_metageneration_not_match=None,
1143 timeout=_DEFAULT_TIMEOUT,
1144 checksum="auto",
1145 retry=DEFAULT_RETRY,
1146 single_shot_download=False,
1147 ):
1148 """Download the contents of a blob object or blob URI into a file-like object.
1150 See https://cloud.google.com/storage/docs/downloading-objects
1152 Args:
1153 blob_or_uri (Union[ \
1154 :class:`~google.cloud.storage.blob.Blob`, \
1155 str, \
1156 ]):
1157 The blob resource to pass or URI to download.
1159 file_obj (file):
1160 A file handle to which to write the blob's data.
1162 start (int):
1163 (Optional) The first byte in a range to be downloaded.
1165 end (int):
1166 (Optional) The last byte in a range to be downloaded.
1168 raw_download (bool):
1169 (Optional) If true, download the object without any expansion.
1171 if_etag_match (Union[str, Set[str]]):
1172 (Optional) See :ref:`using-if-etag-match`
1174 if_etag_not_match (Union[str, Set[str]]):
1175 (Optional) See :ref:`using-if-etag-not-match`
1177 if_generation_match (long):
1178 (Optional) See :ref:`using-if-generation-match`
1180 if_generation_not_match (long):
1181 (Optional) See :ref:`using-if-generation-not-match`
1183 if_metageneration_match (long):
1184 (Optional) See :ref:`using-if-metageneration-match`
1186 if_metageneration_not_match (long):
1187 (Optional) See :ref:`using-if-metageneration-not-match`
1189 timeout ([Union[float, Tuple[float, float]]]):
1190 (Optional) The amount of time, in seconds, to wait
1191 for the server response. See: :ref:`configuring_timeouts`
1193 checksum (str):
1194 (Optional) The type of checksum to compute to verify the integrity
1195 of the object. The response headers must contain a checksum of the
1196 requested type. If the headers lack an appropriate checksum (for
1197 instance in the case of transcoded or ranged downloads where the
1198 remote service does not know the correct checksum, including
1199 downloads where chunk_size is set) an INFO-level log will be
1200 emitted. Supported values are "md5", "crc32c", "auto" and None.
1201 The default is "auto", which will try to detect if the C
1202 extension for crc32c is installed and fall back to md5 otherwise.
1204 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy)
1205 (Optional) How to retry the RPC. A None value will disable
1206 retries. A google.api_core.retry.Retry value will enable retries,
1207 and the object will define retriable response codes and errors and
1208 configure backoff and timeout options.
1210 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1211 Retry object and activates it only if certain conditions are met.
1212 This class exists to provide safe defaults for RPC calls that are
1213 not technically safe to retry normally (due to potential data
1214 duplication or other side-effects) but become safe to retry if a
1215 condition such as if_metageneration_match is set.
1217 See the retry.py source code and docstrings in this package
1218 (google.cloud.storage.retry) for information on retry types and how
1219 to configure them.
1221 single_shot_download (bool):
1222 (Optional) If true, download the object in a single request.
1223 """
1224 with create_trace_span(name="Storage.Client.downloadBlobToFile"):
1225 if not isinstance(blob_or_uri, Blob):
1226 blob_or_uri = Blob.from_uri(blob_or_uri)
1228 blob_or_uri._prep_and_do_download(
1229 file_obj,
1230 client=self,
1231 start=start,
1232 end=end,
1233 raw_download=raw_download,
1234 if_etag_match=if_etag_match,
1235 if_etag_not_match=if_etag_not_match,
1236 if_generation_match=if_generation_match,
1237 if_generation_not_match=if_generation_not_match,
1238 if_metageneration_match=if_metageneration_match,
1239 if_metageneration_not_match=if_metageneration_not_match,
1240 timeout=timeout,
1241 checksum=checksum,
1242 retry=retry,
1243 single_shot_download=single_shot_download,
1244 )
1246 def list_blobs(
1247 self,
1248 bucket_or_name,
1249 max_results=None,
1250 page_token=None,
1251 prefix=None,
1252 delimiter=None,
1253 start_offset=None,
1254 end_offset=None,
1255 include_trailing_delimiter=None,
1256 versions=None,
1257 projection="noAcl",
1258 fields=None,
1259 page_size=None,
1260 timeout=_DEFAULT_TIMEOUT,
1261 retry=DEFAULT_RETRY,
1262 match_glob=None,
1263 include_folders_as_prefixes=None,
1264 soft_deleted=None,
1265 ):
1266 """Return an iterator used to find blobs in the bucket.
1268 If :attr:`user_project` is set, bills the API request to that project.
1270 .. note::
1271 List prefixes (directories) in a bucket using a prefix and delimiter.
1272 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python)
1273 listing objects using a prefix filter.
1275 Args:
1276 bucket_or_name (Union[ \
1277 :class:`~google.cloud.storage.bucket.Bucket`, \
1278 str, \
1279 ]):
1280 The bucket resource to pass or name to create.
1282 max_results (int):
1283 (Optional) The maximum number of blobs to return.
1285 page_token (str):
1286 (Optional) If present, return the next batch of blobs, using the
1287 value, which must correspond to the ``nextPageToken`` value
1288 returned in the previous response. Deprecated: use the ``pages``
1289 property of the returned iterator instead of manually passing the
1290 token.
1292 prefix (str):
1293 (Optional) Prefix used to filter blobs.
1295 delimiter (str):
1296 (Optional) Delimiter, used with ``prefix`` to
1297 emulate hierarchy.
1299 start_offset (str):
1300 (Optional) Filter results to objects whose names are
1301 lexicographically equal to or after ``startOffset``. If
1302 ``endOffset`` is also set, the objects listed will have names
1303 between ``startOffset`` (inclusive) and ``endOffset``
1304 (exclusive).
1306 end_offset (str):
1307 (Optional) Filter results to objects whose names are
1308 lexicographically before ``endOffset``. If ``startOffset`` is
1309 also set, the objects listed will have names between
1310 ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
1312 include_trailing_delimiter (boolean):
1313 (Optional) If true, objects that end in exactly one instance of
1314 ``delimiter`` will have their metadata included in ``items`` in
1315 addition to ``prefixes``.
1317 versions (bool):
1318 (Optional) Whether object versions should be returned
1319 as separate blobs.
1321 projection (str):
1322 (Optional) If used, must be 'full' or 'noAcl'.
1323 Defaults to ``'noAcl'``. Specifies the set of
1324 properties to return.
1326 fields (str):
1327 (Optional) Selector specifying which fields to include
1328 in a partial response. Must be a list of fields. For
1329 example to get a partial response with just the next
1330 page token and the name and language of each blob returned:
1331 ``'items(name,contentLanguage),nextPageToken'``.
1332 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
1334 page_size (int):
1335 (Optional) Maximum number of blobs to return in each page.
1336 Defaults to a value set by the API.
1338 timeout (Optional[Union[float, Tuple[float, float]]]):
1339 The amount of time, in seconds, to wait for the server response.
1341 Can also be passed as a tuple (connect_timeout, read_timeout).
1342 See :meth:`requests.Session.request` documentation for details.
1344 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1345 How to retry the RPC. A None value will disable retries.
1346 A google.api_core.retry.Retry value will enable retries, and the object will
1347 define retriable response codes and errors and configure backoff and timeout options.
1349 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1350 activates it only if certain conditions are met. This class exists to provide safe defaults
1351 for RPC calls that are not technically safe to retry normally (due to potential data
1352 duplication or other side-effects) but become safe to retry if a condition such as
1353 if_metageneration_match is set.
1355 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1356 information on retry types and how to configure them.
1358 match_glob (str):
1359 (Optional) A glob pattern used to filter results (for example, foo*bar).
1360 The string value must be UTF-8 encoded. See:
1361 https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
1363 include_folders_as_prefixes (bool):
1364 (Optional) If true, includes Folders and Managed Folders in the set of
1365 ``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /.
1366 See: https://cloud.google.com/storage/docs/managed-folders
1368 soft_deleted (bool):
1369 (Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
1370 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1371 Note ``soft_deleted`` and ``versions`` cannot be set to True simultaneously. See:
1372 https://cloud.google.com/storage/docs/soft-delete
1374 Returns:
1375 Iterator of all :class:`~google.cloud.storage.blob.Blob`
1376 in this bucket matching the arguments. The RPC call
1377 returns a response when the iterator is consumed.
1379 As part of the response, you'll also get back an iterator.prefixes entity that lists object names
1380 up to and including the requested delimiter. Duplicate entries are omitted from this list.
1381 """
1382 with create_trace_span(name="Storage.Client.listBlobs"):
1383 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1385 extra_params = {"projection": projection}
1387 if prefix is not None:
1388 extra_params["prefix"] = prefix
1390 if delimiter is not None:
1391 extra_params["delimiter"] = delimiter
1393 if match_glob is not None:
1394 extra_params["matchGlob"] = match_glob
1396 if start_offset is not None:
1397 extra_params["startOffset"] = start_offset
1399 if end_offset is not None:
1400 extra_params["endOffset"] = end_offset
1402 if include_trailing_delimiter is not None:
1403 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1405 if versions is not None:
1406 extra_params["versions"] = versions
1408 if fields is not None:
1409 extra_params["fields"] = fields
1411 if include_folders_as_prefixes is not None:
1412 extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes
1414 if soft_deleted is not None:
1415 extra_params["softDeleted"] = soft_deleted
1417 if bucket.user_project is not None:
1418 extra_params["userProject"] = bucket.user_project
1420 path = bucket.path + "/o"
1421 iterator = self._list_resource(
1422 path,
1423 _item_to_blob,
1424 page_token=page_token,
1425 max_results=max_results,
1426 extra_params=extra_params,
1427 page_start=_blobs_page_start,
1428 page_size=page_size,
1429 timeout=timeout,
1430 retry=retry,
1431 )
1432 iterator.bucket = bucket
1433 iterator.prefixes = set()
1434 return iterator
1436 def list_buckets(
1437 self,
1438 max_results=None,
1439 page_token=None,
1440 prefix=None,
1441 projection="noAcl",
1442 fields=None,
1443 project=None,
1444 page_size=None,
1445 timeout=_DEFAULT_TIMEOUT,
1446 retry=DEFAULT_RETRY,
1447 *,
1448 soft_deleted=None,
1449 ):
1450 """Get all buckets in the project associated to the client.
1452 This will not populate the list of blobs available in each
1453 bucket.
1455 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python).
1457 :type max_results: int
1458 :param max_results: (Optional) The maximum number of buckets to return.
1460 :type page_token: str
1461 :param page_token:
1462 (Optional) If present, return the next batch of buckets, using the
1463 value, which must correspond to the ``nextPageToken`` value
1464 returned in the previous response. Deprecated: use the ``pages``
1465 property of the returned iterator instead of manually passing the
1466 token.
1468 :type prefix: str
1469 :param prefix: (Optional) Filter results to buckets whose names begin
1470 with this prefix.
1472 :type projection: str
1473 :param projection:
1474 (Optional) Specifies the set of properties to return. If used, must
1475 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1477 :type fields: str
1478 :param fields:
1479 (Optional) Selector specifying which fields to include in a partial
1480 response. Must be a list of fields. For example to get a partial
1481 response with just the next page token and the language of each
1482 bucket returned: 'items/id,nextPageToken'
1484 :type project: str
1485 :param project: (Optional) The project whose buckets are to be listed.
1486 If not passed, uses the project set on the client.
1488 :type page_size: int
1489 :param page_size: (Optional) Maximum number of buckets to return in each page.
1490 Defaults to a value set by the API.
1492 :type timeout: float or tuple
1493 :param timeout:
1494 (Optional) The amount of time, in seconds, to wait
1495 for the server response. See: :ref:`configuring_timeouts`
1497 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1498 :param retry:
1499 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1501 :type soft_deleted: bool
1502 :param soft_deleted:
1503 (Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
1504 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1505 See: https://cloud.google.com/storage/docs/soft-delete
1507 :rtype: :class:`~google.api_core.page_iterator.Iterator`
1508 :raises ValueError: if both ``project`` is ``None`` and the client's
1509 project is also ``None``.
1510 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
1511 belonging to this project.
1512 """
1513 with create_trace_span(name="Storage.Client.listBuckets"):
1514 extra_params = {}
1516 if project is None:
1517 project = self.project
1519 # Use no project if STORAGE_EMULATOR_HOST is set
1520 if self._is_emulator_set:
1521 if project is None:
1522 project = _get_environ_project()
1523 if project is None:
1524 project = "<none>"
1526 # Only include the project parameter if a project is set.
1527 # If a project is not set, falls back to API validation (BadRequest).
1528 if project is not None:
1529 extra_params = {"project": project}
1531 if prefix is not None:
1532 extra_params["prefix"] = prefix
1534 extra_params["projection"] = projection
1536 if fields is not None:
1537 extra_params["fields"] = fields
1539 if soft_deleted is not None:
1540 extra_params["softDeleted"] = soft_deleted
1542 return self._list_resource(
1543 "/b",
1544 _item_to_bucket,
1545 page_token=page_token,
1546 max_results=max_results,
1547 extra_params=extra_params,
1548 page_size=page_size,
1549 timeout=timeout,
1550 retry=retry,
1551 )
1553 def restore_bucket(
1554 self,
1555 bucket_name,
1556 generation,
1557 projection="noAcl",
1558 if_metageneration_match=None,
1559 if_metageneration_not_match=None,
1560 timeout=_DEFAULT_TIMEOUT,
1561 retry=DEFAULT_RETRY,
1562 ):
1563 """Restores a soft-deleted bucket.
1565 :type bucket_name: str
1566 :param bucket_name: The name of the bucket to be restored.
1568 :type generation: int
1569 :param generation: Selects the specific revision of the bucket.
1571 :type projection: str
1572 :param projection:
1573 (Optional) Specifies the set of properties to return. If used, must
1574 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1576 if_metageneration_match (Optional[int]):
1577 Make the operation conditional on whether the
1578 blob's current metageneration matches the given value.
1580 if_metageneration_not_match (Optional[int]):
1581 Make the operation conditional on whether the blob's
1582 current metageneration does not match the given value.
1584 :type timeout: float or tuple
1585 :param timeout:
1586 (Optional) The amount of time, in seconds, to wait
1587 for the server response. See: :ref:`configuring_timeouts`
1589 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1590 :param retry:
1591 (Optional) How to retry the RPC.
1593 Users can configure non-default retry behavior. A ``None`` value will
1594 disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
1596 :rtype: :class:`google.cloud.storage.bucket.Bucket`
1597 :returns: The restored Bucket.
1598 """
1599 query_params = {"generation": generation, "projection": projection}
1601 _add_generation_match_parameters(
1602 query_params,
1603 if_metageneration_match=if_metageneration_match,
1604 if_metageneration_not_match=if_metageneration_not_match,
1605 )
1607 bucket = self.bucket(bucket_name)
1608 api_response = self._post_resource(
1609 f"{bucket.path}/restore",
1610 None,
1611 query_params=query_params,
1612 timeout=timeout,
1613 retry=retry,
1614 )
1615 bucket._set_properties(api_response)
1616 return bucket
1618 def create_hmac_key(
1619 self,
1620 service_account_email,
1621 project_id=None,
1622 user_project=None,
1623 timeout=_DEFAULT_TIMEOUT,
1624 retry=None,
1625 ):
1626 """Create an HMAC key for a service account.
1628 :type service_account_email: str
1629 :param service_account_email: e-mail address of the service account
1631 :type project_id: str
1632 :param project_id: (Optional) Explicit project ID for the key.
1633 Defaults to the client's project.
1635 :type user_project: str
1636 :param user_project: (Optional) This parameter is currently ignored.
1638 :type timeout: float or tuple
1639 :param timeout:
1640 (Optional) The amount of time, in seconds, to wait
1641 for the server response. See: :ref:`configuring_timeouts`
1643 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1644 :param retry: (Optional) How to retry the RPC. A None value will disable retries.
1645 A google.api_core.retry.Retry value will enable retries, and the object will
1646 define retriable response codes and errors and configure backoff and timeout options.
1648 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1649 activates it only if certain conditions are met. This class exists to provide safe defaults
1650 for RPC calls that are not technically safe to retry normally (due to potential data
1651 duplication or other side-effects) but become safe to retry if a condition such as
1652 if_metageneration_match is set.
1654 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1655 information on retry types and how to configure them.
1657 :rtype:
1658 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1659 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1660 """
1661 with create_trace_span(name="Storage.Client.createHmacKey"):
1662 if project_id is None:
1663 project_id = self.project
1665 path = f"/projects/{project_id}/hmacKeys"
1666 qs_params = {"serviceAccountEmail": service_account_email}
1668 if user_project is not None:
1669 qs_params["userProject"] = user_project
1671 api_response = self._post_resource(
1672 path,
1673 None,
1674 query_params=qs_params,
1675 timeout=timeout,
1676 retry=retry,
1677 )
1678 metadata = HMACKeyMetadata(self)
1679 metadata._properties = api_response["metadata"]
1680 secret = api_response["secret"]
1681 return metadata, secret
1683 def list_hmac_keys(
1684 self,
1685 max_results=None,
1686 service_account_email=None,
1687 show_deleted_keys=None,
1688 project_id=None,
1689 user_project=None,
1690 timeout=_DEFAULT_TIMEOUT,
1691 retry=DEFAULT_RETRY,
1692 ):
1693 """List HMAC keys for a project.
1695 :type max_results: int
1696 :param max_results:
1697 (Optional) Max number of keys to return in a given page.
1699 :type service_account_email: str
1700 :param service_account_email:
1701 (Optional) Limit keys to those created by the given service account.
1703 :type show_deleted_keys: bool
1704 :param show_deleted_keys:
1705 (Optional) Included deleted keys in the list. Default is to
1706 exclude them.
1708 :type project_id: str
1709 :param project_id: (Optional) Explicit project ID for the key.
1710 Defaults to the client's project.
1712 :type user_project: str
1713 :param user_project: (Optional) This parameter is currently ignored.
1715 :type timeout: float or tuple
1716 :param timeout:
1717 (Optional) The amount of time, in seconds, to wait
1718 for the server response. See: :ref:`configuring_timeouts`
1720 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1721 :param retry:
1722 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1724 :rtype:
1725 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1726 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1727 """
1728 with create_trace_span(name="Storage.Client.listHmacKeys"):
1729 if project_id is None:
1730 project_id = self.project
1732 path = f"/projects/{project_id}/hmacKeys"
1733 extra_params = {}
1735 if service_account_email is not None:
1736 extra_params["serviceAccountEmail"] = service_account_email
1738 if show_deleted_keys is not None:
1739 extra_params["showDeletedKeys"] = show_deleted_keys
1741 if user_project is not None:
1742 extra_params["userProject"] = user_project
1744 return self._list_resource(
1745 path,
1746 _item_to_hmac_key_metadata,
1747 max_results=max_results,
1748 extra_params=extra_params,
1749 timeout=timeout,
1750 retry=retry,
1751 )
1753 def get_hmac_key_metadata(
1754 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT
1755 ):
1756 """Return a metadata instance for the given HMAC key.
1758 :type access_id: str
1759 :param access_id: Unique ID of an existing key.
1761 :type project_id: str
1762 :param project_id: (Optional) Project ID of an existing key.
1763 Defaults to client's project.
1765 :type timeout: float or tuple
1766 :param timeout:
1767 (Optional) The amount of time, in seconds, to wait
1768 for the server response. See: :ref:`configuring_timeouts`
1770 :type user_project: str
1771 :param user_project: (Optional) This parameter is currently ignored.
1772 """
1773 with create_trace_span(name="Storage.Client.getHmacKeyMetadata"):
1774 metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
1775 metadata.reload(timeout=timeout) # raises NotFound for missing key
1776 return metadata
1778 def generate_signed_post_policy_v4(
1779 self,
1780 bucket_name,
1781 blob_name,
1782 expiration,
1783 conditions=None,
1784 fields=None,
1785 credentials=None,
1786 virtual_hosted_style=False,
1787 bucket_bound_hostname=None,
1788 scheme="http",
1789 service_account_email=None,
1790 access_token=None,
1791 ):
1792 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request.
1794 .. note::
1796 Assumes ``credentials`` implements the
1797 :class:`google.auth.credentials.Signing` interface. Also assumes
1798 ``credentials`` has a ``service_account_email`` property which
1799 identifies the credentials.
1801 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py).
1803 :type bucket_name: str
1804 :param bucket_name: Bucket name.
1806 :type blob_name: str
1807 :param blob_name: Object name.
1809 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
1810 :param expiration: Policy expiration time. If a ``datetime`` instance is
1811 passed without an explicit ``tzinfo`` set, it will be
1812 assumed to be ``UTC``.
1814 :type conditions: list
1815 :param conditions: (Optional) List of POST policy conditions, which are
1816 used to restrict what is allowed in the request.
1818 :type fields: dict
1819 :param fields: (Optional) Additional elements to include into request.
1821 :type credentials: :class:`google.auth.credentials.Signing`
1822 :param credentials: (Optional) Credentials object with an associated private
1823 key to sign text.
1825 :type virtual_hosted_style: bool
1826 :param virtual_hosted_style:
1827 (Optional) If True, construct the URL relative to the bucket
1828 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
1829 Incompatible with bucket_bound_hostname.
1831 :type bucket_bound_hostname: str
1832 :param bucket_bound_hostname:
1833 (Optional) If passed, construct the URL relative to the bucket-bound hostname.
1834 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
1835 Incompatible with virtual_hosted_style.
1836 See: https://cloud.google.com/storage/docs/request-endpoints#cname
1838 :type scheme: str
1839 :param scheme:
1840 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
1841 this value as a scheme. ``https`` will work only when using a CDN.
1842 Defaults to ``"http"``.
1844 :type service_account_email: str
1845 :param service_account_email: (Optional) E-mail address of the service account.
1847 :type access_token: str
1848 :param access_token: (Optional) Access token for a service account.
1850 :raises: :exc:`ValueError` when mutually exclusive arguments are used.
1852 :rtype: dict
1853 :returns: Signed POST policy.
1854 """
1855 if virtual_hosted_style and bucket_bound_hostname:
1856 raise ValueError(
1857 "Only one of virtual_hosted_style and bucket_bound_hostname "
1858 "can be specified."
1859 )
1861 credentials = self._credentials if credentials is None else credentials
1862 client_email = service_account_email
1863 if not access_token or not service_account_email:
1864 ensure_signed_credentials(credentials)
1865 client_email = credentials.signer_email
1867 # prepare policy conditions and fields
1868 timestamp, datestamp = get_v4_now_dtstamps()
1870 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
1871 email=client_email, datestamp=datestamp
1872 )
1873 required_conditions = [
1874 {"bucket": bucket_name},
1875 {"key": blob_name},
1876 {"x-goog-date": timestamp},
1877 {"x-goog-credential": x_goog_credential},
1878 {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
1879 ]
1881 conditions = conditions or []
1882 policy_fields = {}
1883 for key, value in sorted((fields or {}).items()):
1884 if not key.startswith("x-ignore-"):
1885 policy_fields[key] = value
1886 conditions.append({key: value})
1888 conditions += required_conditions
1890 # calculate policy expiration time
1891 now = _NOW(_UTC).replace(tzinfo=None)
1892 if expiration is None:
1893 expiration = now + datetime.timedelta(hours=1)
1895 policy_expires = now + datetime.timedelta(
1896 seconds=get_expiration_seconds_v4(expiration)
1897 )
1899 # encode policy for signing
1900 policy = json.dumps(
1901 collections.OrderedDict(
1902 sorted(
1903 {
1904 "conditions": conditions,
1905 "expiration": policy_expires.isoformat() + "Z",
1906 }.items()
1907 )
1908 ),
1909 separators=(",", ":"),
1910 )
1911 str_to_sign = base64.b64encode(policy.encode("utf-8"))
1913 # sign the policy and get its cryptographic signature
1914 if access_token and service_account_email:
1915 signature = _sign_message(str_to_sign, access_token, service_account_email)
1916 signature_bytes = base64.b64decode(signature)
1917 else:
1918 signature_bytes = credentials.sign_bytes(str_to_sign)
1920 # get hexadecimal representation of the signature
1921 signature = binascii.hexlify(signature_bytes).decode("utf-8")
1923 policy_fields.update(
1924 {
1925 "key": blob_name,
1926 "x-goog-algorithm": "GOOG4-RSA-SHA256",
1927 "x-goog-credential": x_goog_credential,
1928 "x-goog-date": timestamp,
1929 "x-goog-signature": signature,
1930 "policy": str_to_sign.decode("utf-8"),
1931 }
1932 )
1933 # designate URL
1934 if virtual_hosted_style:
1935 url = _virtual_hosted_style_base_url(
1936 self.api_endpoint, bucket_name, trailing_slash=True
1937 )
1938 elif bucket_bound_hostname:
1939 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/"
1940 else:
1941 url = f"{self.api_endpoint}/{bucket_name}/"
1943 return {"url": url, "fields": policy_fields}
1946def _item_to_bucket(iterator, item):
1947 """Convert a JSON bucket to the native object.
1949 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1950 :param iterator: The iterator that has retrieved the item.
1952 :type item: dict
1953 :param item: An item to be converted to a bucket.
1955 :rtype: :class:`.Bucket`
1956 :returns: The next bucket in the page.
1957 """
1958 name = item.get("name")
1959 bucket = Bucket(iterator.client, name)
1960 bucket._set_properties(item)
1961 return bucket
1964def _item_to_hmac_key_metadata(iterator, item):
1965 """Convert a JSON key metadata resource to the native object.
1967 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1968 :param iterator: The iterator that has retrieved the item.
1970 :type item: dict
1971 :param item: An item to be converted to a key metadata instance.
1973 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`
1974 :returns: The next key metadata instance in the page.
1975 """
1976 metadata = HMACKeyMetadata(iterator.client)
1977 metadata._properties = item
1978 return metadata