Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Client for interacting with the Google Cloud Storage API."""
17import base64
18import binascii
19import collections
20import datetime
21import functools
22import json
23import warnings
24import google.api_core.client_options
26from google.auth.credentials import AnonymousCredentials
28from google.api_core import page_iterator
29from google.cloud._helpers import _LocalStack
30from google.cloud.client import ClientWithProject
31from google.cloud.exceptions import NotFound
33from google.cloud.storage._helpers import _add_generation_match_parameters
34from google.cloud.storage._helpers import _bucket_bound_hostname_url
35from google.cloud.storage._helpers import _get_api_endpoint_override
36from google.cloud.storage._helpers import _get_environ_project
37from google.cloud.storage._helpers import _get_storage_emulator_override
38from google.cloud.storage._helpers import _use_client_cert
39from google.cloud.storage._helpers import _virtual_hosted_style_base_url
40from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
41from google.cloud.storage._helpers import _DEFAULT_SCHEME
42from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
43from google.cloud.storage._helpers import _NOW
44from google.cloud.storage._helpers import _UTC
45from google.cloud.storage._opentelemetry_tracing import create_trace_span
47from google.cloud.storage._http import Connection
48from google.cloud.storage._signing import (
49 get_expiration_seconds_v4,
50 get_v4_now_dtstamps,
51 ensure_signed_credentials,
52 _sign_message,
53)
54from google.cloud.storage.batch import Batch
55from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start
56from google.cloud.storage.blob import Blob
57from google.cloud.storage.hmac_key import HMACKeyMetadata
58from google.cloud.storage.acl import BucketACL
59from google.cloud.storage.acl import DefaultObjectACL
60from google.cloud.storage.constants import _DEFAULT_TIMEOUT
61from google.cloud.storage.retry import DEFAULT_RETRY
64_marker = object()
67class Client(ClientWithProject):
68 """Client to bundle configuration needed for API requests.
70 :type project: str or None
71 :param project: the project which the client acts on behalf of. Will be
72 passed when creating a topic. If not passed,
73 falls back to the default inferred from the environment.
75 :type credentials: :class:`~google.auth.credentials.Credentials`
76 :param credentials: (Optional) The OAuth2 Credentials to use for this
77 client. If not passed (and if no ``_http`` object is
78 passed), falls back to the default inferred from the
79 environment.
81 :type _http: :class:`~requests.Session`
82 :param _http: (Optional) HTTP object to make requests. Can be any object
83 that defines ``request()`` with the same interface as
84 :meth:`requests.Session.request`. If not passed, an
85 ``_http`` object is created that is bound to the
86 ``credentials`` for the current object.
87 This parameter should be considered private, and could
88 change in the future.
90 :type client_info: :class:`~google.api_core.client_info.ClientInfo`
91 :param client_info:
92 The client info used to send a user-agent string along with API
93 requests. If ``None``, then default info will be used. Generally,
94 you only need to set this if you're developing your own library
95 or partner tool.
97 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
98 :param client_options: (Optional) Client options used to set user options on the client.
99 A non-default universe domain or api endpoint should be set through client_options.
101 :type use_auth_w_custom_endpoint: bool
102 :param use_auth_w_custom_endpoint:
103 (Optional) Whether authentication is required under custom endpoints.
104 If false, uses AnonymousCredentials and bypasses authentication.
105 Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
107 :type extra_headers: dict
108 :param extra_headers:
109 (Optional) Custom headers to be sent with the requests attached to the client.
110 For example, you can add custom audit logging headers.
112 :type api_key: string
113 :param api_key:
114 (Optional) An API key. Mutually exclusive with any other credentials.
115 This parameter is an alias for setting `client_options.api_key` and
116 will supercede any api key set in the `client_options` parameter.
117 """
119 SCOPE = (
120 "https://www.googleapis.com/auth/devstorage.full_control",
121 "https://www.googleapis.com/auth/devstorage.read_only",
122 "https://www.googleapis.com/auth/devstorage.read_write",
123 )
124 """The scopes required for authenticating as a Cloud Storage consumer."""
126 def __init__(
127 self,
128 project=_marker,
129 credentials=None,
130 _http=None,
131 client_info=None,
132 client_options=None,
133 use_auth_w_custom_endpoint=True,
134 extra_headers={},
135 *,
136 api_key=None,
137 ):
138 self._base_connection = None
140 if project is None:
141 no_project = True
142 project = "<none>"
143 else:
144 no_project = False
146 if project is _marker:
147 project = None
149 # Save the initial value of constructor arguments before they
150 # are passed along, for use in __reduce__ defined elsewhere.
151 self._initial_client_info = client_info
152 self._initial_client_options = client_options
153 self._extra_headers = extra_headers
155 connection_kw_args = {"client_info": client_info}
157 # api_key should set client_options.api_key. Set it here whether
158 # client_options was specified as a dict, as a ClientOptions object, or
159 # None.
160 if api_key:
161 if client_options and not isinstance(client_options, dict):
162 client_options.api_key = api_key
163 else:
164 if not client_options:
165 client_options = {}
166 client_options["api_key"] = api_key
168 if client_options:
169 if isinstance(client_options, dict):
170 client_options = google.api_core.client_options.from_dict(
171 client_options
172 )
174 if client_options and client_options.universe_domain:
175 self._universe_domain = client_options.universe_domain
176 else:
177 self._universe_domain = None
179 storage_emulator_override = _get_storage_emulator_override()
180 api_endpoint_override = _get_api_endpoint_override()
182 # Determine the api endpoint. The rules are as follows:
184 # 1. If the `api_endpoint` is set in `client_options`, use that as the
185 # endpoint.
186 if client_options and client_options.api_endpoint:
187 api_endpoint = client_options.api_endpoint
189 # 2. Elif the "STORAGE_EMULATOR_HOST" env var is set, then use that as the
190 # endpoint.
191 elif storage_emulator_override:
192 api_endpoint = storage_emulator_override
194 # 3. Elif the "API_ENDPOINT_OVERRIDE" env var is set, then use that as the
195 # endpoint.
196 elif api_endpoint_override:
197 api_endpoint = api_endpoint_override
199 # 4. Elif the `universe_domain` is set in `client_options`,
200 # create the endpoint using that as the default.
201 #
202 # Mutual TLS is not compatible with a non-default universe domain
203 # at this time. If such settings are enabled along with the
204 # "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable, a ValueError will
205 # be raised.
207 elif self._universe_domain:
208 # The final decision of whether to use mTLS takes place in
209 # google-auth-library-python. We peek at the environment variable
210 # here only to issue an exception in case of a conflict.
211 if _use_client_cert():
212 raise ValueError(
213 'The "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable is '
214 'set to "true" and a non-default universe domain is '
215 "configured. mTLS is not supported in any universe other than"
216 "googleapis.com."
217 )
218 api_endpoint = _DEFAULT_SCHEME + _STORAGE_HOST_TEMPLATE.format(
219 universe_domain=self._universe_domain
220 )
222 # 5. Else, use the default, which is to use the default
223 # universe domain of "googleapis.com" and create the endpoint
224 # "storage.googleapis.com" from that.
225 else:
226 api_endpoint = None
228 connection_kw_args["api_endpoint"] = api_endpoint
230 self._is_emulator_set = True if storage_emulator_override else False
232 # If a custom endpoint is set, the client checks for credentials
233 # or finds the default credentials based on the current environment.
234 # Authentication may be bypassed under certain conditions:
235 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
236 # (2) use_auth_w_custom_endpoint is set to False.
237 if connection_kw_args["api_endpoint"] is not None:
238 if self._is_emulator_set or not use_auth_w_custom_endpoint:
239 if credentials is None:
240 credentials = AnonymousCredentials()
241 if project is None:
242 project = _get_environ_project()
243 if project is None:
244 no_project = True
245 project = "<none>"
247 super(Client, self).__init__(
248 project=project,
249 credentials=credentials,
250 client_options=client_options,
251 _http=_http,
252 )
254 # Validate that the universe domain of the credentials matches the
255 # universe domain of the client.
256 if self._credentials.universe_domain != self.universe_domain:
257 raise ValueError(
258 "The configured universe domain ({client_ud}) does not match "
259 "the universe domain found in the credentials ({cred_ud}). If "
260 "you haven't configured the universe domain explicitly, "
261 "`googleapis.com` is the default.".format(
262 client_ud=self.universe_domain,
263 cred_ud=self._credentials.universe_domain,
264 )
265 )
267 if no_project:
268 self.project = None
270 # Pass extra_headers to Connection
271 connection = Connection(self, **connection_kw_args)
272 connection.extra_headers = extra_headers
273 self._connection = connection
274 self._batch_stack = _LocalStack()
276 @classmethod
277 def create_anonymous_client(cls):
278 """Factory: return client with anonymous credentials.
280 .. note::
282 Such a client has only limited access to "public" buckets:
283 listing their contents and downloading their blobs.
285 :rtype: :class:`google.cloud.storage.client.Client`
286 :returns: Instance w/ anonymous credentials and no project.
287 """
288 client = cls(project="<none>", credentials=AnonymousCredentials())
289 client.project = None
290 return client
292 @property
293 def universe_domain(self):
294 return self._universe_domain or _DEFAULT_UNIVERSE_DOMAIN
296 @property
297 def api_endpoint(self):
298 return self._connection.API_BASE_URL
300 def update_user_agent(self, user_agent):
301 """Update the user-agent string for this client.
303 :type user_agent: str
304 :param user_agent: The string to add to the user-agent.
305 """
306 existing_user_agent = self._connection._client_info.user_agent
307 if existing_user_agent is None:
308 self._connection.user_agent = user_agent
309 else:
310 self._connection.user_agent = f"{user_agent} {existing_user_agent}"
312 @property
313 def _connection(self):
314 """Get connection or batch on the client.
316 :rtype: :class:`google.cloud.storage._http.Connection`
317 :returns: The connection set on the client, or the batch
318 if one is set.
319 """
320 if self.current_batch is not None:
321 return self.current_batch
322 else:
323 return self._base_connection
325 @_connection.setter
326 def _connection(self, value):
327 """Set connection on the client.
329 Intended to be used by constructor (since the base class calls)
330 self._connection = connection
331 Will raise if the connection is set more than once.
333 :type value: :class:`google.cloud.storage._http.Connection`
334 :param value: The connection set on the client.
336 :raises: :class:`ValueError` if connection has already been set.
337 """
338 if self._base_connection is not None:
339 raise ValueError("Connection already set on client")
340 self._base_connection = value
342 def _push_batch(self, batch):
343 """Push a batch onto our stack.
345 "Protected", intended for use by batch context mgrs.
347 :type batch: :class:`google.cloud.storage.batch.Batch`
348 :param batch: newly-active batch
349 """
350 self._batch_stack.push(batch)
352 def _pop_batch(self):
353 """Pop a batch from our stack.
355 "Protected", intended for use by batch context mgrs.
357 :raises: IndexError if the stack is empty.
358 :rtype: :class:`google.cloud.storage.batch.Batch`
359 :returns: the top-most batch/transaction, after removing it.
360 """
361 return self._batch_stack.pop()
363 @property
364 def current_batch(self):
365 """Currently-active batch.
367 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if
368 no batch is active).
369 :returns: The batch at the top of the batch stack.
370 """
371 return self._batch_stack.top
373 def get_service_account_email(
374 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
375 ):
376 """Get the email address of the project's GCS service account
378 :type project: str
379 :param project:
380 (Optional) Project ID to use for retreiving GCS service account
381 email address. Defaults to the client's project.
382 :type timeout: float or tuple
383 :param timeout:
384 (Optional) The amount of time, in seconds, to wait
385 for the server response. See: :ref:`configuring_timeouts`
387 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
388 :param retry:
389 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
391 :rtype: str
392 :returns: service account email address
393 """
394 with create_trace_span(name="Storage.Client.getServiceAccountEmail"):
395 if project is None:
396 project = self.project
398 path = f"/projects/{project}/serviceAccount"
399 api_response = self._get_resource(path, timeout=timeout, retry=retry)
400 return api_response["email_address"]
402 def bucket(self, bucket_name, user_project=None, generation=None):
403 """Factory constructor for bucket object.
405 .. note::
406 This will not make an HTTP request; it simply instantiates
407 a bucket object owned by this client.
409 :type bucket_name: str
410 :param bucket_name: The name of the bucket to be instantiated.
412 :type user_project: str
413 :param user_project: (Optional) The project ID to be billed for API
414 requests made via the bucket.
416 :type generation: int
417 :param generation: (Optional) If present, selects a specific revision of
418 this bucket.
420 :rtype: :class:`google.cloud.storage.bucket.Bucket`
421 :returns: The bucket object created.
422 """
423 return Bucket(
424 client=self,
425 name=bucket_name,
426 user_project=user_project,
427 generation=generation,
428 )
430 def batch(self, raise_exception=True):
431 """Factory constructor for batch object.
433 .. note::
434 This will not make an HTTP request; it simply instantiates
435 a batch object owned by this client.
437 :type raise_exception: bool
438 :param raise_exception:
439 (Optional) Defaults to True. If True, instead of adding exceptions
440 to the list of return responses, the final exception will be raised.
441 Note that exceptions are unwrapped after all operations are complete
442 in success or failure, and only the last exception is raised.
444 :rtype: :class:`google.cloud.storage.batch.Batch`
445 :returns: The batch object created.
446 """
447 return Batch(client=self, raise_exception=raise_exception)
449 def _get_resource(
450 self,
451 path,
452 query_params=None,
453 headers=None,
454 timeout=_DEFAULT_TIMEOUT,
455 retry=DEFAULT_RETRY,
456 _target_object=None,
457 ):
458 """Helper for bucket / blob methods making API 'GET' calls.
460 Args:
461 path str:
462 The path of the resource to fetch.
464 query_params Optional[dict]:
465 HTTP query parameters to be passed
467 headers Optional[dict]:
468 HTTP headers to be passed
470 timeout (Optional[Union[float, Tuple[float, float]]]):
471 The amount of time, in seconds, to wait for the server response.
473 Can also be passed as a tuple (connect_timeout, read_timeout).
474 See :meth:`requests.Session.request` documentation for details.
476 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
477 How to retry the RPC. A None value will disable retries.
478 A google.api_core.retry.Retry value will enable retries, and the object will
479 define retriable response codes and errors and configure backoff and timeout options.
481 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
482 activates it only if certain conditions are met. This class exists to provide safe defaults
483 for RPC calls that are not technically safe to retry normally (due to potential data
484 duplication or other side-effects) but become safe to retry if a condition such as
485 if_metageneration_match is set.
487 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
488 information on retry types and how to configure them.
490 _target_object (Union[ \
491 :class:`~google.cloud.storage.bucket.Bucket`, \
492 :class:`~google.cloud.storage.bucket.blob`, \
493 ]):
494 Object to which future data is to be applied -- only relevant
495 in the context of a batch.
497 Returns:
498 dict
499 The JSON resource fetched
501 Raises:
502 google.cloud.exceptions.NotFound
503 If the bucket is not found.
504 """
505 return self._connection.api_request(
506 method="GET",
507 path=path,
508 query_params=query_params,
509 headers=headers,
510 timeout=timeout,
511 retry=retry,
512 _target_object=_target_object,
513 )
515 def _list_resource(
516 self,
517 path,
518 item_to_value,
519 page_token=None,
520 max_results=None,
521 extra_params=None,
522 page_start=page_iterator._do_nothing_page_start,
523 page_size=None,
524 timeout=_DEFAULT_TIMEOUT,
525 retry=DEFAULT_RETRY,
526 ):
527 kwargs = {
528 "method": "GET",
529 "path": path,
530 "timeout": timeout,
531 }
532 with create_trace_span(
533 name="Storage.Client._list_resource_returns_iterator",
534 client=self,
535 api_request=kwargs,
536 retry=retry,
537 ):
538 api_request = functools.partial(
539 self._connection.api_request, timeout=timeout, retry=retry
540 )
541 return page_iterator.HTTPIterator(
542 client=self,
543 api_request=api_request,
544 path=path,
545 item_to_value=item_to_value,
546 page_token=page_token,
547 max_results=max_results,
548 extra_params=extra_params,
549 page_start=page_start,
550 page_size=page_size,
551 )
553 def _patch_resource(
554 self,
555 path,
556 data,
557 query_params=None,
558 headers=None,
559 timeout=_DEFAULT_TIMEOUT,
560 retry=None,
561 _target_object=None,
562 ):
563 """Helper for bucket / blob methods making API 'PATCH' calls.
565 Args:
566 path str:
567 The path of the resource to fetch.
569 data dict:
570 The data to be patched.
572 query_params Optional[dict]:
573 HTTP query parameters to be passed
575 headers Optional[dict]:
576 HTTP headers to be passed
578 timeout (Optional[Union[float, Tuple[float, float]]]):
579 The amount of time, in seconds, to wait for the server response.
581 Can also be passed as a tuple (connect_timeout, read_timeout).
582 See :meth:`requests.Session.request` documentation for details.
584 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
585 How to retry the RPC. A None value will disable retries.
586 A google.api_core.retry.Retry value will enable retries, and the object will
587 define retriable response codes and errors and configure backoff and timeout options.
589 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
590 activates it only if certain conditions are met. This class exists to provide safe defaults
591 for RPC calls that are not technically safe to retry normally (due to potential data
592 duplication or other side-effects) but become safe to retry if a condition such as
593 if_metageneration_match is set.
595 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
596 information on retry types and how to configure them.
598 _target_object (Union[ \
599 :class:`~google.cloud.storage.bucket.Bucket`, \
600 :class:`~google.cloud.storage.bucket.blob`, \
601 ]):
602 Object to which future data is to be applied -- only relevant
603 in the context of a batch.
605 Returns:
606 dict
607 The JSON resource fetched
609 Raises:
610 google.cloud.exceptions.NotFound
611 If the bucket is not found.
612 """
613 return self._connection.api_request(
614 method="PATCH",
615 path=path,
616 data=data,
617 query_params=query_params,
618 headers=headers,
619 timeout=timeout,
620 retry=retry,
621 _target_object=_target_object,
622 )
624 def _put_resource(
625 self,
626 path,
627 data,
628 query_params=None,
629 headers=None,
630 timeout=_DEFAULT_TIMEOUT,
631 retry=None,
632 _target_object=None,
633 ):
634 """Helper for bucket / blob methods making API 'PUT' calls.
636 Args:
637 path str:
638 The path of the resource to fetch.
640 data dict:
641 The data to be patched.
643 query_params Optional[dict]:
644 HTTP query parameters to be passed
646 headers Optional[dict]:
647 HTTP headers to be passed
649 timeout (Optional[Union[float, Tuple[float, float]]]):
650 The amount of time, in seconds, to wait for the server response.
652 Can also be passed as a tuple (connect_timeout, read_timeout).
653 See :meth:`requests.Session.request` documentation for details.
655 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
656 How to retry the RPC. A None value will disable retries.
657 A google.api_core.retry.Retry value will enable retries, and the object will
658 define retriable response codes and errors and configure backoff and timeout options.
660 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
661 activates it only if certain conditions are met. This class exists to provide safe defaults
662 for RPC calls that are not technically safe to retry normally (due to potential data
663 duplication or other side-effects) but become safe to retry if a condition such as
664 if_metageneration_match is set.
666 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
667 information on retry types and how to configure them.
669 _target_object (Union[ \
670 :class:`~google.cloud.storage.bucket.Bucket`, \
671 :class:`~google.cloud.storage.bucket.blob`, \
672 ]):
673 Object to which future data is to be applied -- only relevant
674 in the context of a batch.
676 Returns:
677 dict
678 The JSON resource fetched
680 Raises:
681 google.cloud.exceptions.NotFound
682 If the bucket is not found.
683 """
684 return self._connection.api_request(
685 method="PUT",
686 path=path,
687 data=data,
688 query_params=query_params,
689 headers=headers,
690 timeout=timeout,
691 retry=retry,
692 _target_object=_target_object,
693 )
695 def _post_resource(
696 self,
697 path,
698 data,
699 query_params=None,
700 headers=None,
701 timeout=_DEFAULT_TIMEOUT,
702 retry=None,
703 _target_object=None,
704 ):
705 """Helper for bucket / blob methods making API 'POST' calls.
707 Args:
708 path str:
709 The path of the resource to which to post.
711 data dict:
712 The data to be posted.
714 query_params Optional[dict]:
715 HTTP query parameters to be passed
717 headers Optional[dict]:
718 HTTP headers to be passed
720 timeout (Optional[Union[float, Tuple[float, float]]]):
721 The amount of time, in seconds, to wait for the server response.
723 Can also be passed as a tuple (connect_timeout, read_timeout).
724 See :meth:`requests.Session.request` documentation for details.
726 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
727 How to retry the RPC. A None value will disable retries.
728 A google.api_core.retry.Retry value will enable retries, and the object will
729 define retriable response codes and errors and configure backoff and timeout options.
731 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
732 activates it only if certain conditions are met. This class exists to provide safe defaults
733 for RPC calls that are not technically safe to retry normally (due to potential data
734 duplication or other side-effects) but become safe to retry if a condition such as
735 if_metageneration_match is set.
737 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
738 information on retry types and how to configure them.
740 _target_object (Union[ \
741 :class:`~google.cloud.storage.bucket.Bucket`, \
742 :class:`~google.cloud.storage.bucket.blob`, \
743 ]):
744 Object to which future data is to be applied -- only relevant
745 in the context of a batch.
747 Returns:
748 dict
749 The JSON resource returned from the post.
751 Raises:
752 google.cloud.exceptions.NotFound
753 If the bucket is not found.
754 """
756 return self._connection.api_request(
757 method="POST",
758 path=path,
759 data=data,
760 query_params=query_params,
761 headers=headers,
762 timeout=timeout,
763 retry=retry,
764 _target_object=_target_object,
765 )
767 def _delete_resource(
768 self,
769 path,
770 query_params=None,
771 headers=None,
772 timeout=_DEFAULT_TIMEOUT,
773 retry=DEFAULT_RETRY,
774 _target_object=None,
775 ):
776 """Helper for bucket / blob methods making API 'DELETE' calls.
778 Args:
779 path str:
780 The path of the resource to delete.
782 query_params Optional[dict]:
783 HTTP query parameters to be passed
785 headers Optional[dict]:
786 HTTP headers to be passed
788 timeout (Optional[Union[float, Tuple[float, float]]]):
789 The amount of time, in seconds, to wait for the server response.
791 Can also be passed as a tuple (connect_timeout, read_timeout).
792 See :meth:`requests.Session.request` documentation for details.
794 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
795 How to retry the RPC. A None value will disable retries.
796 A google.api_core.retry.Retry value will enable retries, and the object will
797 define retriable response codes and errors and configure backoff and timeout options.
799 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
800 activates it only if certain conditions are met. This class exists to provide safe defaults
801 for RPC calls that are not technically safe to retry normally (due to potential data
802 duplication or other side-effects) but become safe to retry if a condition such as
803 if_metageneration_match is set.
805 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
806 information on retry types and how to configure them.
808 _target_object (Union[ \
809 :class:`~google.cloud.storage.bucket.Bucket`, \
810 :class:`~google.cloud.storage.bucket.blob`, \
811 ]):
812 Object to which future data is to be applied -- only relevant
813 in the context of a batch.
815 Returns:
816 dict
817 The JSON resource fetched
819 Raises:
820 google.cloud.exceptions.NotFound
821 If the bucket is not found.
822 """
823 return self._connection.api_request(
824 method="DELETE",
825 path=path,
826 query_params=query_params,
827 headers=headers,
828 timeout=timeout,
829 retry=retry,
830 _target_object=_target_object,
831 )
833 def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
834 """Helper to return given bucket or create new by name.
836 Args:
837 bucket_or_name (Union[ \
838 :class:`~google.cloud.storage.bucket.Bucket`, \
839 str, \
840 ]):
841 The bucket resource to pass or name to create.
842 generation (Optional[int]):
843 The bucket generation. If generation is specified,
844 bucket_or_name must be a name (str).
846 Returns:
847 google.cloud.storage.bucket.Bucket
848 The newly created bucket or the given one.
849 """
850 if isinstance(bucket_or_name, Bucket):
851 if generation:
852 raise ValueError(
853 "The generation can only be specified if a "
854 "name is used to specify a bucket, not a Bucket object. "
855 "Create a new Bucket object with the correct generation "
856 "instead."
857 )
858 bucket = bucket_or_name
859 if bucket.client is None:
860 bucket._client = self
861 else:
862 bucket = Bucket(self, name=bucket_or_name, generation=generation)
863 return bucket
865 def get_bucket(
866 self,
867 bucket_or_name,
868 timeout=_DEFAULT_TIMEOUT,
869 if_metageneration_match=None,
870 if_metageneration_not_match=None,
871 retry=DEFAULT_RETRY,
872 *,
873 generation=None,
874 soft_deleted=None,
875 ):
876 """Retrieve a bucket via a GET request.
878 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python).
880 Args:
881 bucket_or_name (Union[ \
882 :class:`~google.cloud.storage.bucket.Bucket`, \
883 str, \
884 ]):
885 The bucket resource to pass or name to create.
887 timeout (Optional[Union[float, Tuple[float, float]]]):
888 The amount of time, in seconds, to wait for the server response.
890 Can also be passed as a tuple (connect_timeout, read_timeout).
891 See :meth:`requests.Session.request` documentation for details.
893 if_metageneration_match (Optional[int]):
894 Make the operation conditional on whether the
895 bucket's current metageneration matches the given value.
897 if_metageneration_not_match (Optional[int]):
898 Make the operation conditional on whether the bucket's
899 current metageneration does not match the given value.
901 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
902 How to retry the RPC. A None value will disable retries.
903 A google.api_core.retry.Retry value will enable retries, and the object will
904 define retriable response codes and errors and configure backoff and timeout options.
906 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
907 activates it only if certain conditions are met. This class exists to provide safe defaults
908 for RPC calls that are not technically safe to retry normally (due to potential data
909 duplication or other side-effects) but become safe to retry if a condition such as
910 if_metageneration_match is set.
912 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
913 information on retry types and how to configure them.
915 generation (Optional[int]):
916 The generation of the bucket. The generation can be used to
917 specify a specific soft-deleted version of the bucket, in
918 conjunction with the ``soft_deleted`` argument below. If
919 ``soft_deleted`` is not True, the generation is unused.
921 soft_deleted (Optional[bool]):
922 If True, looks for a soft-deleted bucket. Will only return
923 the bucket metadata if the bucket exists and is in a
924 soft-deleted state. The bucket ``generation`` is required if
925 ``soft_deleted`` is set to True.
926 See: https://cloud.google.com/storage/docs/soft-delete
928 Returns:
929 google.cloud.storage.bucket.Bucket
930 The bucket matching the name provided.
932 Raises:
933 google.cloud.exceptions.NotFound
934 If the bucket is not found.
935 """
936 with create_trace_span(name="Storage.Client.getBucket"):
937 bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
938 bucket.reload(
939 client=self,
940 timeout=timeout,
941 if_metageneration_match=if_metageneration_match,
942 if_metageneration_not_match=if_metageneration_not_match,
943 retry=retry,
944 soft_deleted=soft_deleted,
945 )
946 return bucket
948 def lookup_bucket(
949 self,
950 bucket_name,
951 timeout=_DEFAULT_TIMEOUT,
952 if_metageneration_match=None,
953 if_metageneration_not_match=None,
954 retry=DEFAULT_RETRY,
955 ):
956 """Get a bucket by name, returning None if not found.
958 You can use this if you would rather check for a None value
959 than catching a NotFound exception.
961 :type bucket_name: str
962 :param bucket_name: The name of the bucket to get.
964 :type timeout: float or tuple
965 :param timeout:
966 (Optional) The amount of time, in seconds, to wait
967 for the server response. See: :ref:`configuring_timeouts`
969 :type if_metageneration_match: long
970 :param if_metageneration_match: (Optional) Make the operation conditional on whether the
971 blob's current metageneration matches the given value.
973 :type if_metageneration_not_match: long
974 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
975 blob's current metageneration does not match the given value.
977 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
978 :param retry:
979 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
981 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType``
982 :returns: The bucket matching the name provided or None if not found.
983 """
984 with create_trace_span(name="Storage.Client.lookupBucket"):
985 try:
986 return self.get_bucket(
987 bucket_name,
988 timeout=timeout,
989 if_metageneration_match=if_metageneration_match,
990 if_metageneration_not_match=if_metageneration_not_match,
991 retry=retry,
992 )
993 except NotFound:
994 return None
996 def create_bucket(
997 self,
998 bucket_or_name,
999 requester_pays=None,
1000 project=None,
1001 user_project=None,
1002 location=None,
1003 data_locations=None,
1004 predefined_acl=None,
1005 predefined_default_object_acl=None,
1006 enable_object_retention=False,
1007 timeout=_DEFAULT_TIMEOUT,
1008 retry=DEFAULT_RETRY,
1009 ):
1010 """Create a new bucket via a POST request.
1012 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python).
1014 Args:
1015 bucket_or_name (Union[ \
1016 :class:`~google.cloud.storage.bucket.Bucket`, \
1017 str, \
1018 ]):
1019 The bucket resource to pass or name to create.
1020 requester_pays (bool):
1021 DEPRECATED. Use Bucket().requester_pays instead.
1022 (Optional) Whether requester pays for API requests for
1023 this bucket and its blobs.
1024 project (str):
1025 (Optional) The project under which the bucket is to be created.
1026 If not passed, uses the project set on the client.
1027 user_project (str):
1028 (Optional) The project ID to be billed for API requests
1029 made via created bucket.
1030 location (str):
1031 (Optional) The location of the bucket. If not passed,
1032 the default location, US, will be used. If specifying a dual-region,
1033 `data_locations` should be set in conjunction. See:
1034 https://cloud.google.com/storage/docs/locations
1035 data_locations (list of str):
1036 (Optional) The list of regional locations of a custom dual-region bucket.
1037 Dual-regions require exactly 2 regional locations. See:
1038 https://cloud.google.com/storage/docs/locations
1039 predefined_acl (str):
1040 (Optional) Name of predefined ACL to apply to bucket. See:
1041 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1042 predefined_default_object_acl (str):
1043 (Optional) Name of predefined ACL to apply to bucket's objects. See:
1044 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1045 enable_object_retention (bool):
1046 (Optional) Whether object retention should be enabled on this bucket. See:
1047 https://cloud.google.com/storage/docs/object-lock
1048 timeout (Optional[Union[float, Tuple[float, float]]]):
1049 The amount of time, in seconds, to wait for the server response.
1051 Can also be passed as a tuple (connect_timeout, read_timeout).
1052 See :meth:`requests.Session.request` documentation for details.
1054 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1055 How to retry the RPC. A None value will disable retries.
1056 A google.api_core.retry.Retry value will enable retries, and the object will
1057 define retriable response codes and errors and configure backoff and timeout options.
1059 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1060 activates it only if certain conditions are met. This class exists to provide safe defaults
1061 for RPC calls that are not technically safe to retry normally (due to potential data
1062 duplication or other side-effects) but become safe to retry if a condition such as
1063 if_metageneration_match is set.
1065 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1066 information on retry types and how to configure them.
1068 Returns:
1069 google.cloud.storage.bucket.Bucket
1070 The newly created bucket.
1072 Raises:
1073 google.cloud.exceptions.Conflict
1074 If the bucket already exists.
1075 """
1076 with create_trace_span(name="Storage.Client.createBucket"):
1077 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1078 query_params = {}
1080 if project is None:
1081 project = self.project
1083 # Use no project if STORAGE_EMULATOR_HOST is set
1084 if self._is_emulator_set:
1085 if project is None:
1086 project = _get_environ_project()
1087 if project is None:
1088 project = "<none>"
1090 # Only include the project parameter if a project is set.
1091 # If a project is not set, falls back to API validation (BadRequest).
1092 if project is not None:
1093 query_params = {"project": project}
1095 if requester_pays is not None:
1096 warnings.warn(
1097 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.",
1098 PendingDeprecationWarning,
1099 stacklevel=1,
1100 )
1101 bucket.requester_pays = requester_pays
1103 if predefined_acl is not None:
1104 predefined_acl = BucketACL.validate_predefined(predefined_acl)
1105 query_params["predefinedAcl"] = predefined_acl
1107 if predefined_default_object_acl is not None:
1108 predefined_default_object_acl = DefaultObjectACL.validate_predefined(
1109 predefined_default_object_acl
1110 )
1111 query_params[
1112 "predefinedDefaultObjectAcl"
1113 ] = predefined_default_object_acl
1115 if user_project is not None:
1116 query_params["userProject"] = user_project
1118 if enable_object_retention:
1119 query_params["enableObjectRetention"] = enable_object_retention
1121 properties = {key: bucket._properties[key] for key in bucket._changes}
1122 properties["name"] = bucket.name
1124 if location is not None:
1125 properties["location"] = location
1127 if data_locations is not None:
1128 properties["customPlacementConfig"] = {"dataLocations": data_locations}
1130 api_response = self._post_resource(
1131 "/b",
1132 properties,
1133 query_params=query_params,
1134 timeout=timeout,
1135 retry=retry,
1136 _target_object=bucket,
1137 )
1139 bucket._set_properties(api_response)
1140 return bucket
1142 def download_blob_to_file(
1143 self,
1144 blob_or_uri,
1145 file_obj,
1146 start=None,
1147 end=None,
1148 raw_download=False,
1149 if_etag_match=None,
1150 if_etag_not_match=None,
1151 if_generation_match=None,
1152 if_generation_not_match=None,
1153 if_metageneration_match=None,
1154 if_metageneration_not_match=None,
1155 timeout=_DEFAULT_TIMEOUT,
1156 checksum="auto",
1157 retry=DEFAULT_RETRY,
1158 single_shot_download=False,
1159 ):
1160 """Download the contents of a blob object or blob URI into a file-like object.
1162 See https://cloud.google.com/storage/docs/downloading-objects
1164 Args:
1165 blob_or_uri (Union[ \
1166 :class:`~google.cloud.storage.blob.Blob`, \
1167 str, \
1168 ]):
1169 The blob resource to pass or URI to download.
1171 file_obj (file):
1172 A file handle to which to write the blob's data.
1174 start (int):
1175 (Optional) The first byte in a range to be downloaded.
1177 end (int):
1178 (Optional) The last byte in a range to be downloaded.
1180 raw_download (bool):
1181 (Optional) If true, download the object without any expansion.
1183 if_etag_match (Union[str, Set[str]]):
1184 (Optional) See :ref:`using-if-etag-match`
1186 if_etag_not_match (Union[str, Set[str]]):
1187 (Optional) See :ref:`using-if-etag-not-match`
1189 if_generation_match (long):
1190 (Optional) See :ref:`using-if-generation-match`
1192 if_generation_not_match (long):
1193 (Optional) See :ref:`using-if-generation-not-match`
1195 if_metageneration_match (long):
1196 (Optional) See :ref:`using-if-metageneration-match`
1198 if_metageneration_not_match (long):
1199 (Optional) See :ref:`using-if-metageneration-not-match`
1201 timeout ([Union[float, Tuple[float, float]]]):
1202 (Optional) The amount of time, in seconds, to wait
1203 for the server response. See: :ref:`configuring_timeouts`
1205 checksum (str):
1206 (Optional) The type of checksum to compute to verify the integrity
1207 of the object. The response headers must contain a checksum of the
1208 requested type. If the headers lack an appropriate checksum (for
1209 instance in the case of transcoded or ranged downloads where the
1210 remote service does not know the correct checksum, including
1211 downloads where chunk_size is set) an INFO-level log will be
1212 emitted. Supported values are "md5", "crc32c", "auto" and None.
1213 The default is "auto", which will try to detect if the C
1214 extension for crc32c is installed and fall back to md5 otherwise.
1216 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy)
1217 (Optional) How to retry the RPC. A None value will disable
1218 retries. A google.api_core.retry.Retry value will enable retries,
1219 and the object will define retriable response codes and errors and
1220 configure backoff and timeout options.
1222 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1223 Retry object and activates it only if certain conditions are met.
1224 This class exists to provide safe defaults for RPC calls that are
1225 not technically safe to retry normally (due to potential data
1226 duplication or other side-effects) but become safe to retry if a
1227 condition such as if_metageneration_match is set.
1229 See the retry.py source code and docstrings in this package
1230 (google.cloud.storage.retry) for information on retry types and how
1231 to configure them.
1233 single_shot_download (bool):
1234 (Optional) If true, download the object in a single request.
1235 """
1236 with create_trace_span(name="Storage.Client.downloadBlobToFile"):
1237 if not isinstance(blob_or_uri, Blob):
1238 blob_or_uri = Blob.from_uri(blob_or_uri)
1240 blob_or_uri._prep_and_do_download(
1241 file_obj,
1242 client=self,
1243 start=start,
1244 end=end,
1245 raw_download=raw_download,
1246 if_etag_match=if_etag_match,
1247 if_etag_not_match=if_etag_not_match,
1248 if_generation_match=if_generation_match,
1249 if_generation_not_match=if_generation_not_match,
1250 if_metageneration_match=if_metageneration_match,
1251 if_metageneration_not_match=if_metageneration_not_match,
1252 timeout=timeout,
1253 checksum=checksum,
1254 retry=retry,
1255 single_shot_download=single_shot_download,
1256 )
1258 def list_blobs(
1259 self,
1260 bucket_or_name,
1261 max_results=None,
1262 page_token=None,
1263 prefix=None,
1264 delimiter=None,
1265 start_offset=None,
1266 end_offset=None,
1267 include_trailing_delimiter=None,
1268 versions=None,
1269 projection="noAcl",
1270 fields=None,
1271 page_size=None,
1272 timeout=_DEFAULT_TIMEOUT,
1273 retry=DEFAULT_RETRY,
1274 match_glob=None,
1275 include_folders_as_prefixes=None,
1276 soft_deleted=None,
1277 ):
1278 """Return an iterator used to find blobs in the bucket.
1280 If :attr:`user_project` is set, bills the API request to that project.
1282 .. note::
1283 List prefixes (directories) in a bucket using a prefix and delimiter.
1284 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python)
1285 listing objects using a prefix filter.
1287 Args:
1288 bucket_or_name (Union[ \
1289 :class:`~google.cloud.storage.bucket.Bucket`, \
1290 str, \
1291 ]):
1292 The bucket resource to pass or name to create.
1294 max_results (int):
1295 (Optional) The maximum number of blobs to return.
1297 page_token (str):
1298 (Optional) If present, return the next batch of blobs, using the
1299 value, which must correspond to the ``nextPageToken`` value
1300 returned in the previous response. Deprecated: use the ``pages``
1301 property of the returned iterator instead of manually passing the
1302 token.
1304 prefix (str):
1305 (Optional) Prefix used to filter blobs.
1307 delimiter (str):
1308 (Optional) Delimiter, used with ``prefix`` to
1309 emulate hierarchy.
1311 start_offset (str):
1312 (Optional) Filter results to objects whose names are
1313 lexicographically equal to or after ``startOffset``. If
1314 ``endOffset`` is also set, the objects listed will have names
1315 between ``startOffset`` (inclusive) and ``endOffset``
1316 (exclusive).
1318 end_offset (str):
1319 (Optional) Filter results to objects whose names are
1320 lexicographically before ``endOffset``. If ``startOffset`` is
1321 also set, the objects listed will have names between
1322 ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
1324 include_trailing_delimiter (boolean):
1325 (Optional) If true, objects that end in exactly one instance of
1326 ``delimiter`` will have their metadata included in ``items`` in
1327 addition to ``prefixes``.
1329 versions (bool):
1330 (Optional) Whether object versions should be returned
1331 as separate blobs.
1333 projection (str):
1334 (Optional) If used, must be 'full' or 'noAcl'.
1335 Defaults to ``'noAcl'``. Specifies the set of
1336 properties to return.
1338 fields (str):
1339 (Optional) Selector specifying which fields to include
1340 in a partial response. Must be a list of fields. For
1341 example to get a partial response with just the next
1342 page token and the name and language of each blob returned:
1343 ``'items(name,contentLanguage),nextPageToken'``.
1344 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
1346 page_size (int):
1347 (Optional) Maximum number of blobs to return in each page.
1348 Defaults to a value set by the API.
1350 timeout (Optional[Union[float, Tuple[float, float]]]):
1351 The amount of time, in seconds, to wait for the server response.
1353 Can also be passed as a tuple (connect_timeout, read_timeout).
1354 See :meth:`requests.Session.request` documentation for details.
1356 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1357 How to retry the RPC. A None value will disable retries.
1358 A google.api_core.retry.Retry value will enable retries, and the object will
1359 define retriable response codes and errors and configure backoff and timeout options.
1361 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1362 activates it only if certain conditions are met. This class exists to provide safe defaults
1363 for RPC calls that are not technically safe to retry normally (due to potential data
1364 duplication or other side-effects) but become safe to retry if a condition such as
1365 if_metageneration_match is set.
1367 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1368 information on retry types and how to configure them.
1370 match_glob (str):
1371 (Optional) A glob pattern used to filter results (for example, foo*bar).
1372 The string value must be UTF-8 encoded. See:
1373 https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
1375 include_folders_as_prefixes (bool):
1376 (Optional) If true, includes Folders and Managed Folders in the set of
1377 ``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /.
1378 See: https://cloud.google.com/storage/docs/managed-folders
1380 soft_deleted (bool):
1381 (Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
1382 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1383 Note ``soft_deleted`` and ``versions`` cannot be set to True simultaneously. See:
1384 https://cloud.google.com/storage/docs/soft-delete
1386 Returns:
1387 Iterator of all :class:`~google.cloud.storage.blob.Blob`
1388 in this bucket matching the arguments. The RPC call
1389 returns a response when the iterator is consumed.
1391 As part of the response, you'll also get back an iterator.prefixes entity that lists object names
1392 up to and including the requested delimiter. Duplicate entries are omitted from this list.
1393 """
1394 with create_trace_span(name="Storage.Client.listBlobs"):
1395 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1397 extra_params = {"projection": projection}
1399 if prefix is not None:
1400 extra_params["prefix"] = prefix
1402 if delimiter is not None:
1403 extra_params["delimiter"] = delimiter
1405 if match_glob is not None:
1406 extra_params["matchGlob"] = match_glob
1408 if start_offset is not None:
1409 extra_params["startOffset"] = start_offset
1411 if end_offset is not None:
1412 extra_params["endOffset"] = end_offset
1414 if include_trailing_delimiter is not None:
1415 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1417 if versions is not None:
1418 extra_params["versions"] = versions
1420 if fields is not None:
1421 extra_params["fields"] = fields
1423 if include_folders_as_prefixes is not None:
1424 extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes
1426 if soft_deleted is not None:
1427 extra_params["softDeleted"] = soft_deleted
1429 if bucket.user_project is not None:
1430 extra_params["userProject"] = bucket.user_project
1432 path = bucket.path + "/o"
1433 iterator = self._list_resource(
1434 path,
1435 _item_to_blob,
1436 page_token=page_token,
1437 max_results=max_results,
1438 extra_params=extra_params,
1439 page_start=_blobs_page_start,
1440 page_size=page_size,
1441 timeout=timeout,
1442 retry=retry,
1443 )
1444 iterator.bucket = bucket
1445 iterator.prefixes = set()
1446 return iterator
1448 def list_buckets(
1449 self,
1450 max_results=None,
1451 page_token=None,
1452 prefix=None,
1453 projection="noAcl",
1454 fields=None,
1455 project=None,
1456 page_size=None,
1457 timeout=_DEFAULT_TIMEOUT,
1458 retry=DEFAULT_RETRY,
1459 *,
1460 soft_deleted=None,
1461 ):
1462 """Get all buckets in the project associated to the client.
1464 This will not populate the list of blobs available in each
1465 bucket.
1467 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python).
1469 :type max_results: int
1470 :param max_results: (Optional) The maximum number of buckets to return.
1472 :type page_token: str
1473 :param page_token:
1474 (Optional) If present, return the next batch of buckets, using the
1475 value, which must correspond to the ``nextPageToken`` value
1476 returned in the previous response. Deprecated: use the ``pages``
1477 property of the returned iterator instead of manually passing the
1478 token.
1480 :type prefix: str
1481 :param prefix: (Optional) Filter results to buckets whose names begin
1482 with this prefix.
1484 :type projection: str
1485 :param projection:
1486 (Optional) Specifies the set of properties to return. If used, must
1487 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1489 :type fields: str
1490 :param fields:
1491 (Optional) Selector specifying which fields to include in a partial
1492 response. Must be a list of fields. For example to get a partial
1493 response with just the next page token and the language of each
1494 bucket returned: 'items/id,nextPageToken'
1496 :type project: str
1497 :param project: (Optional) The project whose buckets are to be listed.
1498 If not passed, uses the project set on the client.
1500 :type page_size: int
1501 :param page_size: (Optional) Maximum number of buckets to return in each page.
1502 Defaults to a value set by the API.
1504 :type timeout: float or tuple
1505 :param timeout:
1506 (Optional) The amount of time, in seconds, to wait
1507 for the server response. See: :ref:`configuring_timeouts`
1509 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1510 :param retry:
1511 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1513 :type soft_deleted: bool
1514 :param soft_deleted:
1515 (Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
1516 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1517 See: https://cloud.google.com/storage/docs/soft-delete
1519 :rtype: :class:`~google.api_core.page_iterator.Iterator`
1520 :raises ValueError: if both ``project`` is ``None`` and the client's
1521 project is also ``None``.
1522 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
1523 belonging to this project.
1524 """
1525 with create_trace_span(name="Storage.Client.listBuckets"):
1526 extra_params = {}
1528 if project is None:
1529 project = self.project
1531 # Use no project if STORAGE_EMULATOR_HOST is set
1532 if self._is_emulator_set:
1533 if project is None:
1534 project = _get_environ_project()
1535 if project is None:
1536 project = "<none>"
1538 # Only include the project parameter if a project is set.
1539 # If a project is not set, falls back to API validation (BadRequest).
1540 if project is not None:
1541 extra_params = {"project": project}
1543 if prefix is not None:
1544 extra_params["prefix"] = prefix
1546 extra_params["projection"] = projection
1548 if fields is not None:
1549 extra_params["fields"] = fields
1551 if soft_deleted is not None:
1552 extra_params["softDeleted"] = soft_deleted
1554 return self._list_resource(
1555 "/b",
1556 _item_to_bucket,
1557 page_token=page_token,
1558 max_results=max_results,
1559 extra_params=extra_params,
1560 page_size=page_size,
1561 timeout=timeout,
1562 retry=retry,
1563 )
1565 def restore_bucket(
1566 self,
1567 bucket_name,
1568 generation,
1569 projection="noAcl",
1570 if_metageneration_match=None,
1571 if_metageneration_not_match=None,
1572 timeout=_DEFAULT_TIMEOUT,
1573 retry=DEFAULT_RETRY,
1574 ):
1575 """Restores a soft-deleted bucket.
1577 :type bucket_name: str
1578 :param bucket_name: The name of the bucket to be restored.
1580 :type generation: int
1581 :param generation: Selects the specific revision of the bucket.
1583 :type projection: str
1584 :param projection:
1585 (Optional) Specifies the set of properties to return. If used, must
1586 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1588 if_metageneration_match (Optional[int]):
1589 Make the operation conditional on whether the
1590 blob's current metageneration matches the given value.
1592 if_metageneration_not_match (Optional[int]):
1593 Make the operation conditional on whether the blob's
1594 current metageneration does not match the given value.
1596 :type timeout: float or tuple
1597 :param timeout:
1598 (Optional) The amount of time, in seconds, to wait
1599 for the server response. See: :ref:`configuring_timeouts`
1601 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1602 :param retry:
1603 (Optional) How to retry the RPC.
1605 Users can configure non-default retry behavior. A ``None`` value will
1606 disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
1608 :rtype: :class:`google.cloud.storage.bucket.Bucket`
1609 :returns: The restored Bucket.
1610 """
1611 query_params = {"generation": generation, "projection": projection}
1613 _add_generation_match_parameters(
1614 query_params,
1615 if_metageneration_match=if_metageneration_match,
1616 if_metageneration_not_match=if_metageneration_not_match,
1617 )
1619 bucket = self.bucket(bucket_name)
1620 api_response = self._post_resource(
1621 f"{bucket.path}/restore",
1622 None,
1623 query_params=query_params,
1624 timeout=timeout,
1625 retry=retry,
1626 )
1627 bucket._set_properties(api_response)
1628 return bucket
1630 def create_hmac_key(
1631 self,
1632 service_account_email,
1633 project_id=None,
1634 user_project=None,
1635 timeout=_DEFAULT_TIMEOUT,
1636 retry=None,
1637 ):
1638 """Create an HMAC key for a service account.
1640 :type service_account_email: str
1641 :param service_account_email: e-mail address of the service account
1643 :type project_id: str
1644 :param project_id: (Optional) Explicit project ID for the key.
1645 Defaults to the client's project.
1647 :type user_project: str
1648 :param user_project: (Optional) This parameter is currently ignored.
1650 :type timeout: float or tuple
1651 :param timeout:
1652 (Optional) The amount of time, in seconds, to wait
1653 for the server response. See: :ref:`configuring_timeouts`
1655 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1656 :param retry: (Optional) How to retry the RPC. A None value will disable retries.
1657 A google.api_core.retry.Retry value will enable retries, and the object will
1658 define retriable response codes and errors and configure backoff and timeout options.
1660 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1661 activates it only if certain conditions are met. This class exists to provide safe defaults
1662 for RPC calls that are not technically safe to retry normally (due to potential data
1663 duplication or other side-effects) but become safe to retry if a condition such as
1664 if_metageneration_match is set.
1666 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1667 information on retry types and how to configure them.
1669 :rtype:
1670 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1671 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1672 """
1673 with create_trace_span(name="Storage.Client.createHmacKey"):
1674 if project_id is None:
1675 project_id = self.project
1677 path = f"/projects/{project_id}/hmacKeys"
1678 qs_params = {"serviceAccountEmail": service_account_email}
1680 if user_project is not None:
1681 qs_params["userProject"] = user_project
1683 api_response = self._post_resource(
1684 path,
1685 None,
1686 query_params=qs_params,
1687 timeout=timeout,
1688 retry=retry,
1689 )
1690 metadata = HMACKeyMetadata(self)
1691 metadata._properties = api_response["metadata"]
1692 secret = api_response["secret"]
1693 return metadata, secret
1695 def list_hmac_keys(
1696 self,
1697 max_results=None,
1698 service_account_email=None,
1699 show_deleted_keys=None,
1700 project_id=None,
1701 user_project=None,
1702 timeout=_DEFAULT_TIMEOUT,
1703 retry=DEFAULT_RETRY,
1704 ):
1705 """List HMAC keys for a project.
1707 :type max_results: int
1708 :param max_results:
1709 (Optional) Max number of keys to return in a given page.
1711 :type service_account_email: str
1712 :param service_account_email:
1713 (Optional) Limit keys to those created by the given service account.
1715 :type show_deleted_keys: bool
1716 :param show_deleted_keys:
1717 (Optional) Included deleted keys in the list. Default is to
1718 exclude them.
1720 :type project_id: str
1721 :param project_id: (Optional) Explicit project ID for the key.
1722 Defaults to the client's project.
1724 :type user_project: str
1725 :param user_project: (Optional) This parameter is currently ignored.
1727 :type timeout: float or tuple
1728 :param timeout:
1729 (Optional) The amount of time, in seconds, to wait
1730 for the server response. See: :ref:`configuring_timeouts`
1732 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1733 :param retry:
1734 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1736 :rtype:
1737 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1738 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1739 """
1740 with create_trace_span(name="Storage.Client.listHmacKeys"):
1741 if project_id is None:
1742 project_id = self.project
1744 path = f"/projects/{project_id}/hmacKeys"
1745 extra_params = {}
1747 if service_account_email is not None:
1748 extra_params["serviceAccountEmail"] = service_account_email
1750 if show_deleted_keys is not None:
1751 extra_params["showDeletedKeys"] = show_deleted_keys
1753 if user_project is not None:
1754 extra_params["userProject"] = user_project
1756 return self._list_resource(
1757 path,
1758 _item_to_hmac_key_metadata,
1759 max_results=max_results,
1760 extra_params=extra_params,
1761 timeout=timeout,
1762 retry=retry,
1763 )
1765 def get_hmac_key_metadata(
1766 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT
1767 ):
1768 """Return a metadata instance for the given HMAC key.
1770 :type access_id: str
1771 :param access_id: Unique ID of an existing key.
1773 :type project_id: str
1774 :param project_id: (Optional) Project ID of an existing key.
1775 Defaults to client's project.
1777 :type timeout: float or tuple
1778 :param timeout:
1779 (Optional) The amount of time, in seconds, to wait
1780 for the server response. See: :ref:`configuring_timeouts`
1782 :type user_project: str
1783 :param user_project: (Optional) This parameter is currently ignored.
1784 """
1785 with create_trace_span(name="Storage.Client.getHmacKeyMetadata"):
1786 metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
1787 metadata.reload(timeout=timeout) # raises NotFound for missing key
1788 return metadata
1790 def generate_signed_post_policy_v4(
1791 self,
1792 bucket_name,
1793 blob_name,
1794 expiration,
1795 conditions=None,
1796 fields=None,
1797 credentials=None,
1798 virtual_hosted_style=False,
1799 bucket_bound_hostname=None,
1800 scheme="http",
1801 service_account_email=None,
1802 access_token=None,
1803 ):
1804 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request.
1806 .. note::
1808 Assumes ``credentials`` implements the
1809 :class:`google.auth.credentials.Signing` interface. Also assumes
1810 ``credentials`` has a ``service_account_email`` property which
1811 identifies the credentials.
1813 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py).
1815 :type bucket_name: str
1816 :param bucket_name: Bucket name.
1818 :type blob_name: str
1819 :param blob_name: Object name.
1821 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
1822 :param expiration: Policy expiration time. If a ``datetime`` instance is
1823 passed without an explicit ``tzinfo`` set, it will be
1824 assumed to be ``UTC``.
1826 :type conditions: list
1827 :param conditions: (Optional) List of POST policy conditions, which are
1828 used to restrict what is allowed in the request.
1830 :type fields: dict
1831 :param fields: (Optional) Additional elements to include into request.
1833 :type credentials: :class:`google.auth.credentials.Signing`
1834 :param credentials: (Optional) Credentials object with an associated private
1835 key to sign text.
1837 :type virtual_hosted_style: bool
1838 :param virtual_hosted_style:
1839 (Optional) If True, construct the URL relative to the bucket
1840 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
1841 Incompatible with bucket_bound_hostname.
1843 :type bucket_bound_hostname: str
1844 :param bucket_bound_hostname:
1845 (Optional) If passed, construct the URL relative to the bucket-bound hostname.
1846 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
1847 Incompatible with virtual_hosted_style.
1848 See: https://cloud.google.com/storage/docs/request-endpoints#cname
1850 :type scheme: str
1851 :param scheme:
1852 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
1853 this value as a scheme. ``https`` will work only when using a CDN.
1854 Defaults to ``"http"``.
1856 :type service_account_email: str
1857 :param service_account_email: (Optional) E-mail address of the service account.
1859 :type access_token: str
1860 :param access_token: (Optional) Access token for a service account.
1862 :raises: :exc:`ValueError` when mutually exclusive arguments are used.
1864 :rtype: dict
1865 :returns: Signed POST policy.
1866 """
1867 if virtual_hosted_style and bucket_bound_hostname:
1868 raise ValueError(
1869 "Only one of virtual_hosted_style and bucket_bound_hostname "
1870 "can be specified."
1871 )
1873 credentials = self._credentials if credentials is None else credentials
1874 client_email = service_account_email
1875 if not access_token or not service_account_email:
1876 ensure_signed_credentials(credentials)
1877 client_email = credentials.signer_email
1879 # prepare policy conditions and fields
1880 timestamp, datestamp = get_v4_now_dtstamps()
1882 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
1883 email=client_email, datestamp=datestamp
1884 )
1885 required_conditions = [
1886 {"bucket": bucket_name},
1887 {"key": blob_name},
1888 {"x-goog-date": timestamp},
1889 {"x-goog-credential": x_goog_credential},
1890 {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
1891 ]
1893 conditions = conditions or []
1894 policy_fields = {}
1895 for key, value in sorted((fields or {}).items()):
1896 if not key.startswith("x-ignore-"):
1897 policy_fields[key] = value
1898 conditions.append({key: value})
1900 conditions += required_conditions
1902 # calculate policy expiration time
1903 now = _NOW(_UTC).replace(tzinfo=None)
1904 if expiration is None:
1905 expiration = now + datetime.timedelta(hours=1)
1907 policy_expires = now + datetime.timedelta(
1908 seconds=get_expiration_seconds_v4(expiration)
1909 )
1911 # encode policy for signing
1912 policy = json.dumps(
1913 collections.OrderedDict(
1914 sorted(
1915 {
1916 "conditions": conditions,
1917 "expiration": policy_expires.isoformat() + "Z",
1918 }.items()
1919 )
1920 ),
1921 separators=(",", ":"),
1922 )
1923 str_to_sign = base64.b64encode(policy.encode("utf-8"))
1925 # sign the policy and get its cryptographic signature
1926 if access_token and service_account_email:
1927 signature = _sign_message(str_to_sign, access_token, service_account_email)
1928 signature_bytes = base64.b64decode(signature)
1929 else:
1930 signature_bytes = credentials.sign_bytes(str_to_sign)
1932 # get hexadecimal representation of the signature
1933 signature = binascii.hexlify(signature_bytes).decode("utf-8")
1935 policy_fields.update(
1936 {
1937 "key": blob_name,
1938 "x-goog-algorithm": "GOOG4-RSA-SHA256",
1939 "x-goog-credential": x_goog_credential,
1940 "x-goog-date": timestamp,
1941 "x-goog-signature": signature,
1942 "policy": str_to_sign.decode("utf-8"),
1943 }
1944 )
1945 # designate URL
1946 if virtual_hosted_style:
1947 url = _virtual_hosted_style_base_url(
1948 self.api_endpoint, bucket_name, trailing_slash=True
1949 )
1950 elif bucket_bound_hostname:
1951 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/"
1952 else:
1953 url = f"{self.api_endpoint}/{bucket_name}/"
1955 return {"url": url, "fields": policy_fields}
1958def _item_to_bucket(iterator, item):
1959 """Convert a JSON bucket to the native object.
1961 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1962 :param iterator: The iterator that has retrieved the item.
1964 :type item: dict
1965 :param item: An item to be converted to a bucket.
1967 :rtype: :class:`.Bucket`
1968 :returns: The next bucket in the page.
1969 """
1970 name = item.get("name")
1971 bucket = Bucket(iterator.client, name)
1972 bucket._set_properties(item)
1973 return bucket
1976def _item_to_hmac_key_metadata(iterator, item):
1977 """Convert a JSON key metadata resource to the native object.
1979 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1980 :param iterator: The iterator that has retrieved the item.
1982 :type item: dict
1983 :param item: An item to be converted to a key metadata instance.
1985 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`
1986 :returns: The next key metadata instance in the page.
1987 """
1988 metadata = HMACKeyMetadata(iterator.client)
1989 metadata._properties = item
1990 return metadata