Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Client for interacting with the Google Cloud Storage API."""
17import base64
18import binascii
19import collections
20import datetime
21import functools
22import json
23import warnings
24import google.api_core.client_options
26from google.auth.credentials import AnonymousCredentials
28from google.api_core import page_iterator
29from google.cloud._helpers import _LocalStack
30from google.cloud.client import ClientWithProject
31from google.cloud.exceptions import NotFound
33from google.cloud.storage._helpers import _add_generation_match_parameters
34from google.cloud.storage._helpers import _bucket_bound_hostname_url
35from google.cloud.storage._helpers import _get_api_endpoint_override
36from google.cloud.storage._helpers import _get_environ_project
37from google.cloud.storage._helpers import _get_storage_emulator_override
38from google.cloud.storage._helpers import _use_client_cert
39from google.cloud.storage._helpers import _virtual_hosted_style_base_url
40from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
41from google.cloud.storage._helpers import _DEFAULT_SCHEME
42from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
43from google.cloud.storage._helpers import _NOW
44from google.cloud.storage._helpers import _UTC
45from google.cloud.storage._opentelemetry_tracing import create_trace_span
47from google.cloud.storage._http import Connection
48from google.cloud.storage._signing import (
49 get_expiration_seconds_v4,
50 get_v4_now_dtstamps,
51 ensure_signed_credentials,
52 _sign_message,
53)
54from google.cloud.storage.batch import Batch
55from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start
56from google.cloud.storage.blob import Blob
57from google.cloud.storage.hmac_key import HMACKeyMetadata
58from google.cloud.storage.acl import BucketACL
59from google.cloud.storage.acl import DefaultObjectACL
60from google.cloud.storage.constants import _DEFAULT_TIMEOUT
61from google.cloud.storage.retry import DEFAULT_RETRY
64_marker = object()
67def _buckets_page_start(iterator, page, response):
68 """Grab unreachable buckets after a :class:`~google.cloud.iterator.Page` started."""
69 unreachable = response.get("unreachable", [])
70 if not isinstance(unreachable, list):
71 raise TypeError(
72 f"expected unreachable to be list, but obtained {type(unreachable)}"
73 )
74 page.unreachable = unreachable
77class Client(ClientWithProject):
78 """Client to bundle configuration needed for API requests.
80 :type project: str or None
81 :param project: the project which the client acts on behalf of. Will be
82 passed when creating a topic. If not passed,
83 falls back to the default inferred from the environment.
85 :type credentials: :class:`~google.auth.credentials.Credentials`
86 :param credentials: (Optional) The OAuth2 Credentials to use for this
87 client. If not passed (and if no ``_http`` object is
88 passed), falls back to the default inferred from the
89 environment.
91 :type _http: :class:`~requests.Session`
92 :param _http: (Optional) HTTP object to make requests. Can be any object
93 that defines ``request()`` with the same interface as
94 :meth:`requests.Session.request`. If not passed, an
95 ``_http`` object is created that is bound to the
96 ``credentials`` for the current object.
97 This parameter should be considered private, and could
98 change in the future.
100 :type client_info: :class:`~google.api_core.client_info.ClientInfo`
101 :param client_info:
102 The client info used to send a user-agent string along with API
103 requests. If ``None``, then default info will be used. Generally,
104 you only need to set this if you're developing your own library
105 or partner tool.
107 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
108 :param client_options: (Optional) Client options used to set user options on the client.
109 A non-default universe domain or api endpoint should be set through client_options.
111 :type use_auth_w_custom_endpoint: bool
112 :param use_auth_w_custom_endpoint:
113 (Optional) Whether authentication is required under custom endpoints.
114 If false, uses AnonymousCredentials and bypasses authentication.
115 Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
117 :type extra_headers: dict
118 :param extra_headers:
119 (Optional) Custom headers to be sent with the requests attached to the client.
120 For example, you can add custom audit logging headers.
122 :type api_key: string
123 :param api_key:
124 (Optional) An API key. Mutually exclusive with any other credentials.
125 This parameter is an alias for setting `client_options.api_key` and
126 will supercede any api key set in the `client_options` parameter.
127 """
129 SCOPE = (
130 "https://www.googleapis.com/auth/devstorage.full_control",
131 "https://www.googleapis.com/auth/devstorage.read_only",
132 "https://www.googleapis.com/auth/devstorage.read_write",
133 )
134 """The scopes required for authenticating as a Cloud Storage consumer."""
136 def __init__(
137 self,
138 project=_marker,
139 credentials=None,
140 _http=None,
141 client_info=None,
142 client_options=None,
143 use_auth_w_custom_endpoint=True,
144 extra_headers={},
145 *,
146 api_key=None,
147 ):
148 self._base_connection = None
150 if project is None:
151 no_project = True
152 project = "<none>"
153 else:
154 no_project = False
156 if project is _marker:
157 project = None
159 # Save the initial value of constructor arguments before they
160 # are passed along, for use in __reduce__ defined elsewhere.
161 self._initial_client_info = client_info
162 self._initial_client_options = client_options
163 self._extra_headers = extra_headers
165 connection_kw_args = {"client_info": client_info}
167 # api_key should set client_options.api_key. Set it here whether
168 # client_options was specified as a dict, as a ClientOptions object, or
169 # None.
170 if api_key:
171 if client_options and not isinstance(client_options, dict):
172 client_options.api_key = api_key
173 else:
174 if not client_options:
175 client_options = {}
176 client_options["api_key"] = api_key
178 if client_options:
179 if isinstance(client_options, dict):
180 client_options = google.api_core.client_options.from_dict(
181 client_options
182 )
184 if client_options and client_options.universe_domain:
185 self._universe_domain = client_options.universe_domain
186 else:
187 self._universe_domain = None
189 storage_emulator_override = _get_storage_emulator_override()
190 api_endpoint_override = _get_api_endpoint_override()
192 # Determine the api endpoint. The rules are as follows:
194 # 1. If the `api_endpoint` is set in `client_options`, use that as the
195 # endpoint.
196 if client_options and client_options.api_endpoint:
197 api_endpoint = client_options.api_endpoint
199 # 2. Elif the "STORAGE_EMULATOR_HOST" env var is set, then use that as the
200 # endpoint.
201 elif storage_emulator_override:
202 api_endpoint = storage_emulator_override
204 # 3. Elif the "API_ENDPOINT_OVERRIDE" env var is set, then use that as the
205 # endpoint.
206 elif api_endpoint_override:
207 api_endpoint = api_endpoint_override
209 # 4. Elif the `universe_domain` is set in `client_options`,
210 # create the endpoint using that as the default.
211 #
212 # Mutual TLS is not compatible with a non-default universe domain
213 # at this time. If such settings are enabled along with the
214 # "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable, a ValueError will
215 # be raised.
217 elif self._universe_domain:
218 # The final decision of whether to use mTLS takes place in
219 # google-auth-library-python. We peek at the environment variable
220 # here only to issue an exception in case of a conflict.
221 if _use_client_cert():
222 raise ValueError(
223 'The "GOOGLE_API_USE_CLIENT_CERTIFICATE" env variable is '
224 'set to "true" and a non-default universe domain is '
225 "configured. mTLS is not supported in any universe other than"
226 "googleapis.com."
227 )
228 api_endpoint = _DEFAULT_SCHEME + _STORAGE_HOST_TEMPLATE.format(
229 universe_domain=self._universe_domain
230 )
232 # 5. Else, use the default, which is to use the default
233 # universe domain of "googleapis.com" and create the endpoint
234 # "storage.googleapis.com" from that.
235 else:
236 api_endpoint = None
238 connection_kw_args["api_endpoint"] = api_endpoint
240 self._is_emulator_set = True if storage_emulator_override else False
242 # If a custom endpoint is set, the client checks for credentials
243 # or finds the default credentials based on the current environment.
244 # Authentication may be bypassed under certain conditions:
245 # (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
246 # (2) use_auth_w_custom_endpoint is set to False.
247 if connection_kw_args["api_endpoint"] is not None:
248 if self._is_emulator_set or not use_auth_w_custom_endpoint:
249 if credentials is None:
250 credentials = AnonymousCredentials()
251 if project is None:
252 project = _get_environ_project()
253 if project is None:
254 no_project = True
255 project = "<none>"
257 super(Client, self).__init__(
258 project=project,
259 credentials=credentials,
260 client_options=client_options,
261 _http=_http,
262 )
264 # Validate that the universe domain of the credentials matches the
265 # universe domain of the client.
266 if self._credentials.universe_domain != self.universe_domain:
267 raise ValueError(
268 "The configured universe domain ({client_ud}) does not match "
269 "the universe domain found in the credentials ({cred_ud}). If "
270 "you haven't configured the universe domain explicitly, "
271 "`googleapis.com` is the default.".format(
272 client_ud=self.universe_domain,
273 cred_ud=self._credentials.universe_domain,
274 )
275 )
277 if no_project:
278 self.project = None
280 # Pass extra_headers to Connection
281 connection = Connection(self, **connection_kw_args)
282 connection.extra_headers = extra_headers
283 self._connection = connection
284 self._batch_stack = _LocalStack()
286 @classmethod
287 def create_anonymous_client(cls):
288 """Factory: return client with anonymous credentials.
290 .. note::
292 Such a client has only limited access to "public" buckets:
293 listing their contents and downloading their blobs.
295 :rtype: :class:`google.cloud.storage.client.Client`
296 :returns: Instance w/ anonymous credentials and no project.
297 """
298 client = cls(project="<none>", credentials=AnonymousCredentials())
299 client.project = None
300 return client
302 @property
303 def universe_domain(self):
304 return self._universe_domain or _DEFAULT_UNIVERSE_DOMAIN
306 @property
307 def api_endpoint(self):
308 return self._connection.API_BASE_URL
310 def update_user_agent(self, user_agent):
311 """Update the user-agent string for this client.
313 :type user_agent: str
314 :param user_agent: The string to add to the user-agent.
315 """
316 existing_user_agent = self._connection._client_info.user_agent
317 if existing_user_agent is None:
318 self._connection.user_agent = user_agent
319 else:
320 self._connection.user_agent = f"{user_agent} {existing_user_agent}"
322 @property
323 def _connection(self):
324 """Get connection or batch on the client.
326 :rtype: :class:`google.cloud.storage._http.Connection`
327 :returns: The connection set on the client, or the batch
328 if one is set.
329 """
330 if self.current_batch is not None:
331 return self.current_batch
332 else:
333 return self._base_connection
335 @_connection.setter
336 def _connection(self, value):
337 """Set connection on the client.
339 Intended to be used by constructor (since the base class calls)
340 self._connection = connection
341 Will raise if the connection is set more than once.
343 :type value: :class:`google.cloud.storage._http.Connection`
344 :param value: The connection set on the client.
346 :raises: :class:`ValueError` if connection has already been set.
347 """
348 if self._base_connection is not None:
349 raise ValueError("Connection already set on client")
350 self._base_connection = value
352 def _push_batch(self, batch):
353 """Push a batch onto our stack.
355 "Protected", intended for use by batch context mgrs.
357 :type batch: :class:`google.cloud.storage.batch.Batch`
358 :param batch: newly-active batch
359 """
360 self._batch_stack.push(batch)
362 def _pop_batch(self):
363 """Pop a batch from our stack.
365 "Protected", intended for use by batch context mgrs.
367 :raises: IndexError if the stack is empty.
368 :rtype: :class:`google.cloud.storage.batch.Batch`
369 :returns: the top-most batch/transaction, after removing it.
370 """
371 return self._batch_stack.pop()
373 @property
374 def current_batch(self):
375 """Currently-active batch.
377 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if
378 no batch is active).
379 :returns: The batch at the top of the batch stack.
380 """
381 return self._batch_stack.top
383 def get_service_account_email(
384 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
385 ):
386 """Get the email address of the project's GCS service account
388 :type project: str
389 :param project:
390 (Optional) Project ID to use for retreiving GCS service account
391 email address. Defaults to the client's project.
392 :type timeout: float or tuple
393 :param timeout:
394 (Optional) The amount of time, in seconds, to wait
395 for the server response. See: :ref:`configuring_timeouts`
397 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
398 :param retry:
399 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
401 :rtype: str
402 :returns: service account email address
403 """
404 with create_trace_span(name="Storage.Client.getServiceAccountEmail"):
405 if project is None:
406 project = self.project
408 path = f"/projects/{project}/serviceAccount"
409 api_response = self._get_resource(path, timeout=timeout, retry=retry)
410 return api_response["email_address"]
412 def bucket(self, bucket_name, user_project=None, generation=None):
413 """Factory constructor for bucket object.
415 .. note::
416 This will not make an HTTP request; it simply instantiates
417 a bucket object owned by this client.
419 :type bucket_name: str
420 :param bucket_name: The name of the bucket to be instantiated.
422 :type user_project: str
423 :param user_project: (Optional) The project ID to be billed for API
424 requests made via the bucket.
426 :type generation: int
427 :param generation: (Optional) If present, selects a specific revision of
428 this bucket.
430 :rtype: :class:`google.cloud.storage.bucket.Bucket`
431 :returns: The bucket object created.
432 """
433 return Bucket(
434 client=self,
435 name=bucket_name,
436 user_project=user_project,
437 generation=generation,
438 )
440 def batch(self, raise_exception=True):
441 """Factory constructor for batch object.
443 .. note::
444 This will not make an HTTP request; it simply instantiates
445 a batch object owned by this client.
447 :type raise_exception: bool
448 :param raise_exception:
449 (Optional) Defaults to True. If True, instead of adding exceptions
450 to the list of return responses, the final exception will be raised.
451 Note that exceptions are unwrapped after all operations are complete
452 in success or failure, and only the last exception is raised.
454 :rtype: :class:`google.cloud.storage.batch.Batch`
455 :returns: The batch object created.
456 """
457 return Batch(client=self, raise_exception=raise_exception)
459 def _get_resource(
460 self,
461 path,
462 query_params=None,
463 headers=None,
464 timeout=_DEFAULT_TIMEOUT,
465 retry=DEFAULT_RETRY,
466 _target_object=None,
467 ):
468 """Helper for bucket / blob methods making API 'GET' calls.
470 Args:
471 path str:
472 The path of the resource to fetch.
474 query_params Optional[dict]:
475 HTTP query parameters to be passed
477 headers Optional[dict]:
478 HTTP headers to be passed
480 timeout (Optional[Union[float, Tuple[float, float]]]):
481 The amount of time, in seconds, to wait for the server response.
483 Can also be passed as a tuple (connect_timeout, read_timeout).
484 See :meth:`requests.Session.request` documentation for details.
486 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
487 How to retry the RPC. A None value will disable retries.
488 A google.api_core.retry.Retry value will enable retries, and the object will
489 define retriable response codes and errors and configure backoff and timeout options.
491 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
492 activates it only if certain conditions are met. This class exists to provide safe defaults
493 for RPC calls that are not technically safe to retry normally (due to potential data
494 duplication or other side-effects) but become safe to retry if a condition such as
495 if_metageneration_match is set.
497 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
498 information on retry types and how to configure them.
500 _target_object (Union[ \
501 :class:`~google.cloud.storage.bucket.Bucket`, \
502 :class:`~google.cloud.storage.bucket.blob`, \
503 ]):
504 Object to which future data is to be applied -- only relevant
505 in the context of a batch.
507 Returns:
508 dict
509 The JSON resource fetched
511 Raises:
512 google.cloud.exceptions.NotFound
513 If the bucket is not found.
514 """
515 return self._connection.api_request(
516 method="GET",
517 path=path,
518 query_params=query_params,
519 headers=headers,
520 timeout=timeout,
521 retry=retry,
522 _target_object=_target_object,
523 )
525 def _list_resource(
526 self,
527 path,
528 item_to_value,
529 page_token=None,
530 max_results=None,
531 extra_params=None,
532 page_start=page_iterator._do_nothing_page_start,
533 page_size=None,
534 timeout=_DEFAULT_TIMEOUT,
535 retry=DEFAULT_RETRY,
536 ):
537 kwargs = {
538 "method": "GET",
539 "path": path,
540 "timeout": timeout,
541 }
542 with create_trace_span(
543 name="Storage.Client._list_resource_returns_iterator",
544 client=self,
545 api_request=kwargs,
546 retry=retry,
547 ):
548 api_request = functools.partial(
549 self._connection.api_request, timeout=timeout, retry=retry
550 )
551 return page_iterator.HTTPIterator(
552 client=self,
553 api_request=api_request,
554 path=path,
555 item_to_value=item_to_value,
556 page_token=page_token,
557 max_results=max_results,
558 extra_params=extra_params,
559 page_start=page_start,
560 page_size=page_size,
561 )
563 def _patch_resource(
564 self,
565 path,
566 data,
567 query_params=None,
568 headers=None,
569 timeout=_DEFAULT_TIMEOUT,
570 retry=None,
571 _target_object=None,
572 ):
573 """Helper for bucket / blob methods making API 'PATCH' calls.
575 Args:
576 path str:
577 The path of the resource to fetch.
579 data dict:
580 The data to be patched.
582 query_params Optional[dict]:
583 HTTP query parameters to be passed
585 headers Optional[dict]:
586 HTTP headers to be passed
588 timeout (Optional[Union[float, Tuple[float, float]]]):
589 The amount of time, in seconds, to wait for the server response.
591 Can also be passed as a tuple (connect_timeout, read_timeout).
592 See :meth:`requests.Session.request` documentation for details.
594 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
595 How to retry the RPC. A None value will disable retries.
596 A google.api_core.retry.Retry value will enable retries, and the object will
597 define retriable response codes and errors and configure backoff and timeout options.
599 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
600 activates it only if certain conditions are met. This class exists to provide safe defaults
601 for RPC calls that are not technically safe to retry normally (due to potential data
602 duplication or other side-effects) but become safe to retry if a condition such as
603 if_metageneration_match is set.
605 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
606 information on retry types and how to configure them.
608 _target_object (Union[ \
609 :class:`~google.cloud.storage.bucket.Bucket`, \
610 :class:`~google.cloud.storage.bucket.blob`, \
611 ]):
612 Object to which future data is to be applied -- only relevant
613 in the context of a batch.
615 Returns:
616 dict
617 The JSON resource fetched
619 Raises:
620 google.cloud.exceptions.NotFound
621 If the bucket is not found.
622 """
623 return self._connection.api_request(
624 method="PATCH",
625 path=path,
626 data=data,
627 query_params=query_params,
628 headers=headers,
629 timeout=timeout,
630 retry=retry,
631 _target_object=_target_object,
632 )
634 def _put_resource(
635 self,
636 path,
637 data,
638 query_params=None,
639 headers=None,
640 timeout=_DEFAULT_TIMEOUT,
641 retry=None,
642 _target_object=None,
643 ):
644 """Helper for bucket / blob methods making API 'PUT' calls.
646 Args:
647 path str:
648 The path of the resource to fetch.
650 data dict:
651 The data to be patched.
653 query_params Optional[dict]:
654 HTTP query parameters to be passed
656 headers Optional[dict]:
657 HTTP headers to be passed
659 timeout (Optional[Union[float, Tuple[float, float]]]):
660 The amount of time, in seconds, to wait for the server response.
662 Can also be passed as a tuple (connect_timeout, read_timeout).
663 See :meth:`requests.Session.request` documentation for details.
665 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
666 How to retry the RPC. A None value will disable retries.
667 A google.api_core.retry.Retry value will enable retries, and the object will
668 define retriable response codes and errors and configure backoff and timeout options.
670 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
671 activates it only if certain conditions are met. This class exists to provide safe defaults
672 for RPC calls that are not technically safe to retry normally (due to potential data
673 duplication or other side-effects) but become safe to retry if a condition such as
674 if_metageneration_match is set.
676 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
677 information on retry types and how to configure them.
679 _target_object (Union[ \
680 :class:`~google.cloud.storage.bucket.Bucket`, \
681 :class:`~google.cloud.storage.bucket.blob`, \
682 ]):
683 Object to which future data is to be applied -- only relevant
684 in the context of a batch.
686 Returns:
687 dict
688 The JSON resource fetched
690 Raises:
691 google.cloud.exceptions.NotFound
692 If the bucket is not found.
693 """
694 return self._connection.api_request(
695 method="PUT",
696 path=path,
697 data=data,
698 query_params=query_params,
699 headers=headers,
700 timeout=timeout,
701 retry=retry,
702 _target_object=_target_object,
703 )
705 def _post_resource(
706 self,
707 path,
708 data,
709 query_params=None,
710 headers=None,
711 timeout=_DEFAULT_TIMEOUT,
712 retry=None,
713 _target_object=None,
714 ):
715 """Helper for bucket / blob methods making API 'POST' calls.
717 Args:
718 path str:
719 The path of the resource to which to post.
721 data dict:
722 The data to be posted.
724 query_params Optional[dict]:
725 HTTP query parameters to be passed
727 headers Optional[dict]:
728 HTTP headers to be passed
730 timeout (Optional[Union[float, Tuple[float, float]]]):
731 The amount of time, in seconds, to wait for the server response.
733 Can also be passed as a tuple (connect_timeout, read_timeout).
734 See :meth:`requests.Session.request` documentation for details.
736 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
737 How to retry the RPC. A None value will disable retries.
738 A google.api_core.retry.Retry value will enable retries, and the object will
739 define retriable response codes and errors and configure backoff and timeout options.
741 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
742 activates it only if certain conditions are met. This class exists to provide safe defaults
743 for RPC calls that are not technically safe to retry normally (due to potential data
744 duplication or other side-effects) but become safe to retry if a condition such as
745 if_metageneration_match is set.
747 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
748 information on retry types and how to configure them.
750 _target_object (Union[ \
751 :class:`~google.cloud.storage.bucket.Bucket`, \
752 :class:`~google.cloud.storage.bucket.blob`, \
753 ]):
754 Object to which future data is to be applied -- only relevant
755 in the context of a batch.
757 Returns:
758 dict
759 The JSON resource returned from the post.
761 Raises:
762 google.cloud.exceptions.NotFound
763 If the bucket is not found.
764 """
766 return self._connection.api_request(
767 method="POST",
768 path=path,
769 data=data,
770 query_params=query_params,
771 headers=headers,
772 timeout=timeout,
773 retry=retry,
774 _target_object=_target_object,
775 )
777 def _delete_resource(
778 self,
779 path,
780 query_params=None,
781 headers=None,
782 timeout=_DEFAULT_TIMEOUT,
783 retry=DEFAULT_RETRY,
784 _target_object=None,
785 ):
786 """Helper for bucket / blob methods making API 'DELETE' calls.
788 Args:
789 path str:
790 The path of the resource to delete.
792 query_params Optional[dict]:
793 HTTP query parameters to be passed
795 headers Optional[dict]:
796 HTTP headers to be passed
798 timeout (Optional[Union[float, Tuple[float, float]]]):
799 The amount of time, in seconds, to wait for the server response.
801 Can also be passed as a tuple (connect_timeout, read_timeout).
802 See :meth:`requests.Session.request` documentation for details.
804 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
805 How to retry the RPC. A None value will disable retries.
806 A google.api_core.retry.Retry value will enable retries, and the object will
807 define retriable response codes and errors and configure backoff and timeout options.
809 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
810 activates it only if certain conditions are met. This class exists to provide safe defaults
811 for RPC calls that are not technically safe to retry normally (due to potential data
812 duplication or other side-effects) but become safe to retry if a condition such as
813 if_metageneration_match is set.
815 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
816 information on retry types and how to configure them.
818 _target_object (Union[ \
819 :class:`~google.cloud.storage.bucket.Bucket`, \
820 :class:`~google.cloud.storage.bucket.blob`, \
821 ]):
822 Object to which future data is to be applied -- only relevant
823 in the context of a batch.
825 Returns:
826 dict
827 The JSON resource fetched
829 Raises:
830 google.cloud.exceptions.NotFound
831 If the bucket is not found.
832 """
833 return self._connection.api_request(
834 method="DELETE",
835 path=path,
836 query_params=query_params,
837 headers=headers,
838 timeout=timeout,
839 retry=retry,
840 _target_object=_target_object,
841 )
843 def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
844 """Helper to return given bucket or create new by name.
846 Args:
847 bucket_or_name (Union[ \
848 :class:`~google.cloud.storage.bucket.Bucket`, \
849 str, \
850 ]):
851 The bucket resource to pass or name to create.
852 generation (Optional[int]):
853 The bucket generation. If generation is specified,
854 bucket_or_name must be a name (str).
856 Returns:
857 google.cloud.storage.bucket.Bucket
858 The newly created bucket or the given one.
859 """
860 if isinstance(bucket_or_name, Bucket):
861 if generation:
862 raise ValueError(
863 "The generation can only be specified if a "
864 "name is used to specify a bucket, not a Bucket object. "
865 "Create a new Bucket object with the correct generation "
866 "instead."
867 )
868 bucket = bucket_or_name
869 if bucket.client is None:
870 bucket._client = self
871 else:
872 bucket = Bucket(self, name=bucket_or_name, generation=generation)
873 return bucket
875 def get_bucket(
876 self,
877 bucket_or_name,
878 timeout=_DEFAULT_TIMEOUT,
879 if_metageneration_match=None,
880 if_metageneration_not_match=None,
881 retry=DEFAULT_RETRY,
882 *,
883 generation=None,
884 soft_deleted=None,
885 ):
886 """Retrieve a bucket via a GET request.
888 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/get) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-get-bucket-metadata#storage_get_bucket_metadata-python).
890 Args:
891 bucket_or_name (Union[ \
892 :class:`~google.cloud.storage.bucket.Bucket`, \
893 str, \
894 ]):
895 The bucket resource to pass or name to create.
897 timeout (Optional[Union[float, Tuple[float, float]]]):
898 The amount of time, in seconds, to wait for the server response.
900 Can also be passed as a tuple (connect_timeout, read_timeout).
901 See :meth:`requests.Session.request` documentation for details.
903 if_metageneration_match (Optional[int]):
904 Make the operation conditional on whether the
905 bucket's current metageneration matches the given value.
907 if_metageneration_not_match (Optional[int]):
908 Make the operation conditional on whether the bucket's
909 current metageneration does not match the given value.
911 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
912 How to retry the RPC. A None value will disable retries.
913 A google.api_core.retry.Retry value will enable retries, and the object will
914 define retriable response codes and errors and configure backoff and timeout options.
916 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
917 activates it only if certain conditions are met. This class exists to provide safe defaults
918 for RPC calls that are not technically safe to retry normally (due to potential data
919 duplication or other side-effects) but become safe to retry if a condition such as
920 if_metageneration_match is set.
922 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
923 information on retry types and how to configure them.
925 generation (Optional[int]):
926 The generation of the bucket. The generation can be used to
927 specify a specific soft-deleted version of the bucket, in
928 conjunction with the ``soft_deleted`` argument below. If
929 ``soft_deleted`` is not True, the generation is unused.
931 soft_deleted (Optional[bool]):
932 If True, looks for a soft-deleted bucket. Will only return
933 the bucket metadata if the bucket exists and is in a
934 soft-deleted state. The bucket ``generation`` is required if
935 ``soft_deleted`` is set to True.
936 See: https://cloud.google.com/storage/docs/soft-delete
938 Returns:
939 google.cloud.storage.bucket.Bucket
940 The bucket matching the name provided.
942 Raises:
943 google.cloud.exceptions.NotFound
944 If the bucket is not found.
945 """
946 with create_trace_span(name="Storage.Client.getBucket"):
947 bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
948 bucket.reload(
949 client=self,
950 timeout=timeout,
951 if_metageneration_match=if_metageneration_match,
952 if_metageneration_not_match=if_metageneration_not_match,
953 retry=retry,
954 soft_deleted=soft_deleted,
955 )
956 return bucket
958 def lookup_bucket(
959 self,
960 bucket_name,
961 timeout=_DEFAULT_TIMEOUT,
962 if_metageneration_match=None,
963 if_metageneration_not_match=None,
964 retry=DEFAULT_RETRY,
965 ):
966 """Get a bucket by name, returning None if not found.
968 You can use this if you would rather check for a None value
969 than catching a NotFound exception.
971 :type bucket_name: str
972 :param bucket_name: The name of the bucket to get.
974 :type timeout: float or tuple
975 :param timeout:
976 (Optional) The amount of time, in seconds, to wait
977 for the server response. See: :ref:`configuring_timeouts`
979 :type if_metageneration_match: long
980 :param if_metageneration_match: (Optional) Make the operation conditional on whether the
981 blob's current metageneration matches the given value.
983 :type if_metageneration_not_match: long
984 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
985 blob's current metageneration does not match the given value.
987 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
988 :param retry:
989 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
991 :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType``
992 :returns: The bucket matching the name provided or None if not found.
993 """
994 with create_trace_span(name="Storage.Client.lookupBucket"):
995 try:
996 return self.get_bucket(
997 bucket_name,
998 timeout=timeout,
999 if_metageneration_match=if_metageneration_match,
1000 if_metageneration_not_match=if_metageneration_not_match,
1001 retry=retry,
1002 )
1003 except NotFound:
1004 return None
1006 def create_bucket(
1007 self,
1008 bucket_or_name,
1009 requester_pays=None,
1010 project=None,
1011 user_project=None,
1012 location=None,
1013 data_locations=None,
1014 predefined_acl=None,
1015 predefined_default_object_acl=None,
1016 enable_object_retention=False,
1017 timeout=_DEFAULT_TIMEOUT,
1018 retry=DEFAULT_RETRY,
1019 ):
1020 """Create a new bucket via a POST request.
1022 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-create-bucket#storage_create_bucket-python).
1024 Args:
1025 bucket_or_name (Union[ \
1026 :class:`~google.cloud.storage.bucket.Bucket`, \
1027 str, \
1028 ]):
1029 The bucket resource to pass or name to create.
1030 requester_pays (bool):
1031 DEPRECATED. Use Bucket().requester_pays instead.
1032 (Optional) Whether requester pays for API requests for
1033 this bucket and its blobs.
1034 project (str):
1035 (Optional) The project under which the bucket is to be created.
1036 If not passed, uses the project set on the client.
1037 user_project (str):
1038 (Optional) The project ID to be billed for API requests
1039 made via created bucket.
1040 location (str):
1041 (Optional) The location of the bucket. If not passed,
1042 the default location, US, will be used. If specifying a dual-region,
1043 `data_locations` should be set in conjunction. See:
1044 https://cloud.google.com/storage/docs/locations
1045 data_locations (list of str):
1046 (Optional) The list of regional locations of a custom dual-region bucket.
1047 Dual-regions require exactly 2 regional locations. See:
1048 https://cloud.google.com/storage/docs/locations
1049 predefined_acl (str):
1050 (Optional) Name of predefined ACL to apply to bucket. See:
1051 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1052 predefined_default_object_acl (str):
1053 (Optional) Name of predefined ACL to apply to bucket's objects. See:
1054 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
1055 enable_object_retention (bool):
1056 (Optional) Whether object retention should be enabled on this bucket. See:
1057 https://cloud.google.com/storage/docs/object-lock
1058 timeout (Optional[Union[float, Tuple[float, float]]]):
1059 The amount of time, in seconds, to wait for the server response.
1061 Can also be passed as a tuple (connect_timeout, read_timeout).
1062 See :meth:`requests.Session.request` documentation for details.
1064 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1065 How to retry the RPC. A None value will disable retries.
1066 A google.api_core.retry.Retry value will enable retries, and the object will
1067 define retriable response codes and errors and configure backoff and timeout options.
1069 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1070 activates it only if certain conditions are met. This class exists to provide safe defaults
1071 for RPC calls that are not technically safe to retry normally (due to potential data
1072 duplication or other side-effects) but become safe to retry if a condition such as
1073 if_metageneration_match is set.
1075 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1076 information on retry types and how to configure them.
1078 Returns:
1079 google.cloud.storage.bucket.Bucket
1080 The newly created bucket.
1082 Raises:
1083 google.cloud.exceptions.Conflict
1084 If the bucket already exists.
1085 """
1086 with create_trace_span(name="Storage.Client.createBucket"):
1087 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1088 query_params = {}
1090 if project is None:
1091 project = self.project
1093 # Use no project if STORAGE_EMULATOR_HOST is set
1094 if self._is_emulator_set:
1095 if project is None:
1096 project = _get_environ_project()
1097 if project is None:
1098 project = "<none>"
1100 # Only include the project parameter if a project is set.
1101 # If a project is not set, falls back to API validation (BadRequest).
1102 if project is not None:
1103 query_params = {"project": project}
1105 if requester_pays is not None:
1106 warnings.warn(
1107 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.",
1108 PendingDeprecationWarning,
1109 stacklevel=1,
1110 )
1111 bucket.requester_pays = requester_pays
1113 if predefined_acl is not None:
1114 predefined_acl = BucketACL.validate_predefined(predefined_acl)
1115 query_params["predefinedAcl"] = predefined_acl
1117 if predefined_default_object_acl is not None:
1118 predefined_default_object_acl = DefaultObjectACL.validate_predefined(
1119 predefined_default_object_acl
1120 )
1121 query_params[
1122 "predefinedDefaultObjectAcl"
1123 ] = predefined_default_object_acl
1125 if user_project is not None:
1126 query_params["userProject"] = user_project
1128 if enable_object_retention:
1129 query_params["enableObjectRetention"] = enable_object_retention
1131 properties = {key: bucket._properties[key] for key in bucket._changes}
1132 properties["name"] = bucket.name
1134 if location is not None:
1135 properties["location"] = location
1137 if data_locations is not None:
1138 properties["customPlacementConfig"] = {"dataLocations": data_locations}
1140 api_response = self._post_resource(
1141 "/b",
1142 properties,
1143 query_params=query_params,
1144 timeout=timeout,
1145 retry=retry,
1146 _target_object=bucket,
1147 )
1149 bucket._set_properties(api_response)
1150 return bucket
1152 def download_blob_to_file(
1153 self,
1154 blob_or_uri,
1155 file_obj,
1156 start=None,
1157 end=None,
1158 raw_download=False,
1159 if_etag_match=None,
1160 if_etag_not_match=None,
1161 if_generation_match=None,
1162 if_generation_not_match=None,
1163 if_metageneration_match=None,
1164 if_metageneration_not_match=None,
1165 timeout=_DEFAULT_TIMEOUT,
1166 checksum="auto",
1167 retry=DEFAULT_RETRY,
1168 single_shot_download=False,
1169 ):
1170 """Download the contents of a blob object or blob URI into a file-like object.
1172 See https://cloud.google.com/storage/docs/downloading-objects
1174 Args:
1175 blob_or_uri (Union[ \
1176 :class:`~google.cloud.storage.blob.Blob`, \
1177 str, \
1178 ]):
1179 The blob resource to pass or URI to download.
1181 file_obj (file):
1182 A file handle to which to write the blob's data.
1184 start (int):
1185 (Optional) The first byte in a range to be downloaded.
1187 end (int):
1188 (Optional) The last byte in a range to be downloaded.
1190 raw_download (bool):
1191 (Optional) If true, download the object without any expansion.
1193 if_etag_match (Union[str, Set[str]]):
1194 (Optional) See :ref:`using-if-etag-match`
1196 if_etag_not_match (Union[str, Set[str]]):
1197 (Optional) See :ref:`using-if-etag-not-match`
1199 if_generation_match (long):
1200 (Optional) See :ref:`using-if-generation-match`
1202 if_generation_not_match (long):
1203 (Optional) See :ref:`using-if-generation-not-match`
1205 if_metageneration_match (long):
1206 (Optional) See :ref:`using-if-metageneration-match`
1208 if_metageneration_not_match (long):
1209 (Optional) See :ref:`using-if-metageneration-not-match`
1211 timeout ([Union[float, Tuple[float, float]]]):
1212 (Optional) The amount of time, in seconds, to wait
1213 for the server response. See: :ref:`configuring_timeouts`
1215 checksum (str):
1216 (Optional) The type of checksum to compute to verify the integrity
1217 of the object. The response headers must contain a checksum of the
1218 requested type. If the headers lack an appropriate checksum (for
1219 instance in the case of transcoded or ranged downloads where the
1220 remote service does not know the correct checksum, including
1221 downloads where chunk_size is set) an INFO-level log will be
1222 emitted. Supported values are "md5", "crc32c", "auto" and None.
1223 The default is "auto", which will try to detect if the C
1224 extension for crc32c is installed and fall back to md5 otherwise.
1226 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy)
1227 (Optional) How to retry the RPC. A None value will disable
1228 retries. A google.api_core.retry.Retry value will enable retries,
1229 and the object will define retriable response codes and errors and
1230 configure backoff and timeout options.
1232 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
1233 Retry object and activates it only if certain conditions are met.
1234 This class exists to provide safe defaults for RPC calls that are
1235 not technically safe to retry normally (due to potential data
1236 duplication or other side-effects) but become safe to retry if a
1237 condition such as if_metageneration_match is set.
1239 See the retry.py source code and docstrings in this package
1240 (google.cloud.storage.retry) for information on retry types and how
1241 to configure them.
1243 single_shot_download (bool):
1244 (Optional) If true, download the object in a single request.
1245 """
1246 with create_trace_span(name="Storage.Client.downloadBlobToFile"):
1247 if not isinstance(blob_or_uri, Blob):
1248 blob_or_uri = Blob.from_uri(blob_or_uri)
1250 blob_or_uri._prep_and_do_download(
1251 file_obj,
1252 client=self,
1253 start=start,
1254 end=end,
1255 raw_download=raw_download,
1256 if_etag_match=if_etag_match,
1257 if_etag_not_match=if_etag_not_match,
1258 if_generation_match=if_generation_match,
1259 if_generation_not_match=if_generation_not_match,
1260 if_metageneration_match=if_metageneration_match,
1261 if_metageneration_not_match=if_metageneration_not_match,
1262 timeout=timeout,
1263 checksum=checksum,
1264 retry=retry,
1265 single_shot_download=single_shot_download,
1266 )
1268 def list_blobs(
1269 self,
1270 bucket_or_name,
1271 max_results=None,
1272 page_token=None,
1273 prefix=None,
1274 delimiter=None,
1275 start_offset=None,
1276 end_offset=None,
1277 include_trailing_delimiter=None,
1278 versions=None,
1279 projection="noAcl",
1280 fields=None,
1281 page_size=None,
1282 timeout=_DEFAULT_TIMEOUT,
1283 retry=DEFAULT_RETRY,
1284 match_glob=None,
1285 include_folders_as_prefixes=None,
1286 soft_deleted=None,
1287 ):
1288 """Return an iterator used to find blobs in the bucket.
1290 If :attr:`user_project` is set, bills the API request to that project.
1292 .. note::
1293 List prefixes (directories) in a bucket using a prefix and delimiter.
1294 See a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-files-with-prefix#storage_list_files_with_prefix-python)
1295 listing objects using a prefix filter.
1297 Args:
1298 bucket_or_name (Union[ \
1299 :class:`~google.cloud.storage.bucket.Bucket`, \
1300 str, \
1301 ]):
1302 The bucket resource to pass or name to create.
1304 max_results (int):
1305 (Optional) The maximum number of blobs to return.
1307 page_token (str):
1308 (Optional) If present, return the next batch of blobs, using the
1309 value, which must correspond to the ``nextPageToken`` value
1310 returned in the previous response. Deprecated: use the ``pages``
1311 property of the returned iterator instead of manually passing the
1312 token.
1314 prefix (str):
1315 (Optional) Prefix used to filter blobs.
1317 delimiter (str):
1318 (Optional) Delimiter, used with ``prefix`` to
1319 emulate hierarchy.
1321 start_offset (str):
1322 (Optional) Filter results to objects whose names are
1323 lexicographically equal to or after ``startOffset``. If
1324 ``endOffset`` is also set, the objects listed will have names
1325 between ``startOffset`` (inclusive) and ``endOffset``
1326 (exclusive).
1328 end_offset (str):
1329 (Optional) Filter results to objects whose names are
1330 lexicographically before ``endOffset``. If ``startOffset`` is
1331 also set, the objects listed will have names between
1332 ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
1334 include_trailing_delimiter (boolean):
1335 (Optional) If true, objects that end in exactly one instance of
1336 ``delimiter`` will have their metadata included in ``items`` in
1337 addition to ``prefixes``.
1339 versions (bool):
1340 (Optional) Whether object versions should be returned
1341 as separate blobs.
1343 projection (str):
1344 (Optional) If used, must be 'full' or 'noAcl'.
1345 Defaults to ``'noAcl'``. Specifies the set of
1346 properties to return.
1348 fields (str):
1349 (Optional) Selector specifying which fields to include
1350 in a partial response. Must be a list of fields. For
1351 example to get a partial response with just the next
1352 page token and the name and language of each blob returned:
1353 ``'items(name,contentLanguage),nextPageToken'``.
1354 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
1356 page_size (int):
1357 (Optional) Maximum number of blobs to return in each page.
1358 Defaults to a value set by the API.
1360 timeout (Optional[Union[float, Tuple[float, float]]]):
1361 The amount of time, in seconds, to wait for the server response.
1363 Can also be passed as a tuple (connect_timeout, read_timeout).
1364 See :meth:`requests.Session.request` documentation for details.
1366 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
1367 How to retry the RPC. A None value will disable retries.
1368 A google.api_core.retry.Retry value will enable retries, and the object will
1369 define retriable response codes and errors and configure backoff and timeout options.
1371 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1372 activates it only if certain conditions are met. This class exists to provide safe defaults
1373 for RPC calls that are not technically safe to retry normally (due to potential data
1374 duplication or other side-effects) but become safe to retry if a condition such as
1375 if_metageneration_match is set.
1377 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1378 information on retry types and how to configure them.
1380 match_glob (str):
1381 (Optional) A glob pattern used to filter results (for example, foo*bar).
1382 The string value must be UTF-8 encoded. See:
1383 https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
1385 include_folders_as_prefixes (bool):
1386 (Optional) If true, includes Folders and Managed Folders in the set of
1387 ``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /.
1388 See: https://cloud.google.com/storage/docs/managed-folders
1390 soft_deleted (bool):
1391 (Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
1392 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1393 Note ``soft_deleted`` and ``versions`` cannot be set to True simultaneously. See:
1394 https://cloud.google.com/storage/docs/soft-delete
1396 Returns:
1397 Iterator of all :class:`~google.cloud.storage.blob.Blob`
1398 in this bucket matching the arguments. The RPC call
1399 returns a response when the iterator is consumed.
1401 As part of the response, you'll also get back an iterator.prefixes entity that lists object names
1402 up to and including the requested delimiter. Duplicate entries are omitted from this list.
1403 """
1404 with create_trace_span(name="Storage.Client.listBlobs"):
1405 bucket = self._bucket_arg_to_bucket(bucket_or_name)
1407 extra_params = {"projection": projection}
1409 if prefix is not None:
1410 extra_params["prefix"] = prefix
1412 if delimiter is not None:
1413 extra_params["delimiter"] = delimiter
1415 if match_glob is not None:
1416 extra_params["matchGlob"] = match_glob
1418 if start_offset is not None:
1419 extra_params["startOffset"] = start_offset
1421 if end_offset is not None:
1422 extra_params["endOffset"] = end_offset
1424 if include_trailing_delimiter is not None:
1425 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1427 if versions is not None:
1428 extra_params["versions"] = versions
1430 if fields is not None:
1431 extra_params["fields"] = fields
1433 if include_folders_as_prefixes is not None:
1434 extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes
1436 if soft_deleted is not None:
1437 extra_params["softDeleted"] = soft_deleted
1439 if bucket.user_project is not None:
1440 extra_params["userProject"] = bucket.user_project
1442 path = bucket.path + "/o"
1443 iterator = self._list_resource(
1444 path,
1445 _item_to_blob,
1446 page_token=page_token,
1447 max_results=max_results,
1448 extra_params=extra_params,
1449 page_start=_blobs_page_start,
1450 page_size=page_size,
1451 timeout=timeout,
1452 retry=retry,
1453 )
1454 iterator.bucket = bucket
1455 iterator.prefixes = set()
1456 return iterator
1458 def list_buckets(
1459 self,
1460 max_results=None,
1461 page_token=None,
1462 prefix=None,
1463 projection="noAcl",
1464 fields=None,
1465 project=None,
1466 page_size=None,
1467 timeout=_DEFAULT_TIMEOUT,
1468 retry=DEFAULT_RETRY,
1469 *,
1470 soft_deleted=None,
1471 return_partial_success=None,
1472 ):
1473 """Get all buckets in the project associated to the client.
1475 This will not populate the list of blobs available in each
1476 bucket.
1478 See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/buckets/list) and a [code sample](https://cloud.google.com/storage/docs/samples/storage-list-buckets#storage_list_buckets-python).
1480 :type max_results: int
1481 :param max_results: (Optional) The maximum number of buckets to return.
1483 :type page_token: str
1484 :param page_token:
1485 (Optional) If present, return the next batch of buckets, using the
1486 value, which must correspond to the ``nextPageToken`` value
1487 returned in the previous response. Deprecated: use the ``pages``
1488 property of the returned iterator instead of manually passing the
1489 token.
1491 :type prefix: str
1492 :param prefix: (Optional) Filter results to buckets whose names begin
1493 with this prefix.
1495 :type projection: str
1496 :param projection:
1497 (Optional) Specifies the set of properties to return. If used, must
1498 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1500 :type fields: str
1501 :param fields:
1502 (Optional) Selector specifying which fields to include in a partial
1503 response. Must be a list of fields. For example to get a partial
1504 response with just the next page token and the language of each
1505 bucket returned: 'items/id,nextPageToken'
1507 :type project: str
1508 :param project: (Optional) The project whose buckets are to be listed.
1509 If not passed, uses the project set on the client.
1511 :type page_size: int
1512 :param page_size: (Optional) Maximum number of buckets to return in each page.
1513 Defaults to a value set by the API.
1515 :type timeout: float or tuple
1516 :param timeout:
1517 (Optional) The amount of time, in seconds, to wait
1518 for the server response. See: :ref:`configuring_timeouts`
1520 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1521 :param retry:
1522 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1524 :type soft_deleted: bool
1525 :param soft_deleted:
1526 (Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
1527 generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
1528 See: https://cloud.google.com/storage/docs/soft-delete
1530 :type return_partial_success: bool
1531 :param return_partial_success:
1532 (Optional) If True, the response will also contain a list of
1533 unreachable buckets if the buckets are unavailable. The
1534 unreachable buckets will be available on the ``unreachable``
1535 attribute of the returned iterator.
1537 :rtype: :class:`~google.api_core.page_iterator.Iterator`
1538 :raises ValueError: if both ``project`` is ``None`` and the client's
1539 project is also ``None``.
1540 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
1541 belonging to this project.
1542 """
1543 with create_trace_span(name="Storage.Client.listBuckets"):
1544 extra_params = {}
1546 if project is None:
1547 project = self.project
1549 # Use no project if STORAGE_EMULATOR_HOST is set
1550 if self._is_emulator_set:
1551 if project is None:
1552 project = _get_environ_project()
1553 if project is None:
1554 project = "<none>"
1556 # Only include the project parameter if a project is set.
1557 # If a project is not set, falls back to API validation (BadRequest).
1558 if project is not None:
1559 extra_params = {"project": project}
1561 if prefix is not None:
1562 extra_params["prefix"] = prefix
1564 extra_params["projection"] = projection
1566 if fields is not None:
1567 extra_params["fields"] = fields
1569 if soft_deleted is not None:
1570 extra_params["softDeleted"] = soft_deleted
1572 if return_partial_success is not None:
1573 extra_params["returnPartialSuccess"] = return_partial_success
1575 iterator = self._list_resource(
1576 "/b",
1577 _item_to_bucket,
1578 page_token=page_token,
1579 max_results=max_results,
1580 extra_params=extra_params,
1581 page_size=page_size,
1582 timeout=timeout,
1583 retry=retry,
1584 page_start=_buckets_page_start,
1585 )
1586 return iterator
1588 def restore_bucket(
1589 self,
1590 bucket_name,
1591 generation,
1592 projection="noAcl",
1593 if_metageneration_match=None,
1594 if_metageneration_not_match=None,
1595 timeout=_DEFAULT_TIMEOUT,
1596 retry=DEFAULT_RETRY,
1597 ):
1598 """Restores a soft-deleted bucket.
1600 :type bucket_name: str
1601 :param bucket_name: The name of the bucket to be restored.
1603 :type generation: int
1604 :param generation: Selects the specific revision of the bucket.
1606 :type projection: str
1607 :param projection:
1608 (Optional) Specifies the set of properties to return. If used, must
1609 be 'full' or 'noAcl'. Defaults to 'noAcl'.
1611 if_metageneration_match (Optional[int]):
1612 Make the operation conditional on whether the
1613 blob's current metageneration matches the given value.
1615 if_metageneration_not_match (Optional[int]):
1616 Make the operation conditional on whether the blob's
1617 current metageneration does not match the given value.
1619 :type timeout: float or tuple
1620 :param timeout:
1621 (Optional) The amount of time, in seconds, to wait
1622 for the server response. See: :ref:`configuring_timeouts`
1624 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1625 :param retry:
1626 (Optional) How to retry the RPC.
1628 Users can configure non-default retry behavior. A ``None`` value will
1629 disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
1631 :rtype: :class:`google.cloud.storage.bucket.Bucket`
1632 :returns: The restored Bucket.
1633 """
1634 query_params = {"generation": generation, "projection": projection}
1636 _add_generation_match_parameters(
1637 query_params,
1638 if_metageneration_match=if_metageneration_match,
1639 if_metageneration_not_match=if_metageneration_not_match,
1640 )
1642 bucket = self.bucket(bucket_name)
1643 api_response = self._post_resource(
1644 f"{bucket.path}/restore",
1645 None,
1646 query_params=query_params,
1647 timeout=timeout,
1648 retry=retry,
1649 )
1650 bucket._set_properties(api_response)
1651 return bucket
1653 def create_hmac_key(
1654 self,
1655 service_account_email,
1656 project_id=None,
1657 user_project=None,
1658 timeout=_DEFAULT_TIMEOUT,
1659 retry=None,
1660 ):
1661 """Create an HMAC key for a service account.
1663 :type service_account_email: str
1664 :param service_account_email: e-mail address of the service account
1666 :type project_id: str
1667 :param project_id: (Optional) Explicit project ID for the key.
1668 Defaults to the client's project.
1670 :type user_project: str
1671 :param user_project: (Optional) This parameter is currently ignored.
1673 :type timeout: float or tuple
1674 :param timeout:
1675 (Optional) The amount of time, in seconds, to wait
1676 for the server response. See: :ref:`configuring_timeouts`
1678 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1679 :param retry: (Optional) How to retry the RPC. A None value will disable retries.
1680 A google.api_core.retry.Retry value will enable retries, and the object will
1681 define retriable response codes and errors and configure backoff and timeout options.
1683 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and
1684 activates it only if certain conditions are met. This class exists to provide safe defaults
1685 for RPC calls that are not technically safe to retry normally (due to potential data
1686 duplication or other side-effects) but become safe to retry if a condition such as
1687 if_metageneration_match is set.
1689 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
1690 information on retry types and how to configure them.
1692 :rtype:
1693 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1694 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1695 """
1696 with create_trace_span(name="Storage.Client.createHmacKey"):
1697 if project_id is None:
1698 project_id = self.project
1700 path = f"/projects/{project_id}/hmacKeys"
1701 qs_params = {"serviceAccountEmail": service_account_email}
1703 if user_project is not None:
1704 qs_params["userProject"] = user_project
1706 api_response = self._post_resource(
1707 path,
1708 None,
1709 query_params=qs_params,
1710 timeout=timeout,
1711 retry=retry,
1712 )
1713 metadata = HMACKeyMetadata(self)
1714 metadata._properties = api_response["metadata"]
1715 secret = api_response["secret"]
1716 return metadata, secret
1718 def list_hmac_keys(
1719 self,
1720 max_results=None,
1721 service_account_email=None,
1722 show_deleted_keys=None,
1723 project_id=None,
1724 user_project=None,
1725 timeout=_DEFAULT_TIMEOUT,
1726 retry=DEFAULT_RETRY,
1727 ):
1728 """List HMAC keys for a project.
1730 :type max_results: int
1731 :param max_results:
1732 (Optional) Max number of keys to return in a given page.
1734 :type service_account_email: str
1735 :param service_account_email:
1736 (Optional) Limit keys to those created by the given service account.
1738 :type show_deleted_keys: bool
1739 :param show_deleted_keys:
1740 (Optional) Included deleted keys in the list. Default is to
1741 exclude them.
1743 :type project_id: str
1744 :param project_id: (Optional) Explicit project ID for the key.
1745 Defaults to the client's project.
1747 :type user_project: str
1748 :param user_project: (Optional) This parameter is currently ignored.
1750 :type timeout: float or tuple
1751 :param timeout:
1752 (Optional) The amount of time, in seconds, to wait
1753 for the server response. See: :ref:`configuring_timeouts`
1755 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
1756 :param retry:
1757 (Optional) How to retry the RPC. See: :ref:`configuring_retries`
1759 :rtype:
1760 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
1761 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
1762 """
1763 with create_trace_span(name="Storage.Client.listHmacKeys"):
1764 if project_id is None:
1765 project_id = self.project
1767 path = f"/projects/{project_id}/hmacKeys"
1768 extra_params = {}
1770 if service_account_email is not None:
1771 extra_params["serviceAccountEmail"] = service_account_email
1773 if show_deleted_keys is not None:
1774 extra_params["showDeletedKeys"] = show_deleted_keys
1776 if user_project is not None:
1777 extra_params["userProject"] = user_project
1779 return self._list_resource(
1780 path,
1781 _item_to_hmac_key_metadata,
1782 max_results=max_results,
1783 extra_params=extra_params,
1784 timeout=timeout,
1785 retry=retry,
1786 )
1788 def get_hmac_key_metadata(
1789 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT
1790 ):
1791 """Return a metadata instance for the given HMAC key.
1793 :type access_id: str
1794 :param access_id: Unique ID of an existing key.
1796 :type project_id: str
1797 :param project_id: (Optional) Project ID of an existing key.
1798 Defaults to client's project.
1800 :type timeout: float or tuple
1801 :param timeout:
1802 (Optional) The amount of time, in seconds, to wait
1803 for the server response. See: :ref:`configuring_timeouts`
1805 :type user_project: str
1806 :param user_project: (Optional) This parameter is currently ignored.
1807 """
1808 with create_trace_span(name="Storage.Client.getHmacKeyMetadata"):
1809 metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
1810 metadata.reload(timeout=timeout) # raises NotFound for missing key
1811 return metadata
1813 def generate_signed_post_policy_v4(
1814 self,
1815 bucket_name,
1816 blob_name,
1817 expiration,
1818 conditions=None,
1819 fields=None,
1820 credentials=None,
1821 virtual_hosted_style=False,
1822 bucket_bound_hostname=None,
1823 scheme="http",
1824 service_account_email=None,
1825 access_token=None,
1826 ):
1827 """Generate a V4 signed policy object. Generated policy object allows user to upload objects with a POST request.
1829 .. note::
1831 Assumes ``credentials`` implements the
1832 :class:`google.auth.credentials.Signing` interface. Also assumes
1833 ``credentials`` has a ``service_account_email`` property which
1834 identifies the credentials.
1836 See a [code sample](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_generate_signed_post_policy_v4.py).
1838 :type bucket_name: str
1839 :param bucket_name: Bucket name.
1841 :type blob_name: str
1842 :param blob_name: Object name.
1844 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
1845 :param expiration: Policy expiration time. If a ``datetime`` instance is
1846 passed without an explicit ``tzinfo`` set, it will be
1847 assumed to be ``UTC``.
1849 :type conditions: list
1850 :param conditions: (Optional) List of POST policy conditions, which are
1851 used to restrict what is allowed in the request.
1853 :type fields: dict
1854 :param fields: (Optional) Additional elements to include into request.
1856 :type credentials: :class:`google.auth.credentials.Signing`
1857 :param credentials: (Optional) Credentials object with an associated private
1858 key to sign text.
1860 :type virtual_hosted_style: bool
1861 :param virtual_hosted_style:
1862 (Optional) If True, construct the URL relative to the bucket
1863 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
1864 Incompatible with bucket_bound_hostname.
1866 :type bucket_bound_hostname: str
1867 :param bucket_bound_hostname:
1868 (Optional) If passed, construct the URL relative to the bucket-bound hostname.
1869 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
1870 Incompatible with virtual_hosted_style.
1871 See: https://cloud.google.com/storage/docs/request-endpoints#cname
1873 :type scheme: str
1874 :param scheme:
1875 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
1876 this value as a scheme. ``https`` will work only when using a CDN.
1877 Defaults to ``"http"``.
1879 :type service_account_email: str
1880 :param service_account_email: (Optional) E-mail address of the service account.
1882 :type access_token: str
1883 :param access_token: (Optional) Access token for a service account.
1885 :raises: :exc:`ValueError` when mutually exclusive arguments are used.
1887 :rtype: dict
1888 :returns: Signed POST policy.
1889 """
1890 if virtual_hosted_style and bucket_bound_hostname:
1891 raise ValueError(
1892 "Only one of virtual_hosted_style and bucket_bound_hostname "
1893 "can be specified."
1894 )
1896 credentials = self._credentials if credentials is None else credentials
1897 client_email = service_account_email
1898 if not access_token or not service_account_email:
1899 ensure_signed_credentials(credentials)
1900 client_email = credentials.signer_email
1902 # prepare policy conditions and fields
1903 timestamp, datestamp = get_v4_now_dtstamps()
1905 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
1906 email=client_email, datestamp=datestamp
1907 )
1908 required_conditions = [
1909 {"bucket": bucket_name},
1910 {"key": blob_name},
1911 {"x-goog-date": timestamp},
1912 {"x-goog-credential": x_goog_credential},
1913 {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
1914 ]
1916 conditions = conditions or []
1917 policy_fields = {}
1918 for key, value in sorted((fields or {}).items()):
1919 if not key.startswith("x-ignore-"):
1920 policy_fields[key] = value
1921 conditions.append({key: value})
1923 conditions += required_conditions
1925 # calculate policy expiration time
1926 now = _NOW(_UTC).replace(tzinfo=None)
1927 if expiration is None:
1928 expiration = now + datetime.timedelta(hours=1)
1930 policy_expires = now + datetime.timedelta(
1931 seconds=get_expiration_seconds_v4(expiration)
1932 )
1934 # encode policy for signing
1935 policy = json.dumps(
1936 collections.OrderedDict(
1937 sorted(
1938 {
1939 "conditions": conditions,
1940 "expiration": policy_expires.isoformat() + "Z",
1941 }.items()
1942 )
1943 ),
1944 separators=(",", ":"),
1945 )
1946 str_to_sign = base64.b64encode(policy.encode("utf-8"))
1948 # sign the policy and get its cryptographic signature
1949 if access_token and service_account_email:
1950 signature = _sign_message(str_to_sign, access_token, service_account_email)
1951 signature_bytes = base64.b64decode(signature)
1952 else:
1953 signature_bytes = credentials.sign_bytes(str_to_sign)
1955 # get hexadecimal representation of the signature
1956 signature = binascii.hexlify(signature_bytes).decode("utf-8")
1958 policy_fields.update(
1959 {
1960 "key": blob_name,
1961 "x-goog-algorithm": "GOOG4-RSA-SHA256",
1962 "x-goog-credential": x_goog_credential,
1963 "x-goog-date": timestamp,
1964 "x-goog-signature": signature,
1965 "policy": str_to_sign.decode("utf-8"),
1966 }
1967 )
1968 # designate URL
1969 if virtual_hosted_style:
1970 url = _virtual_hosted_style_base_url(
1971 self.api_endpoint, bucket_name, trailing_slash=True
1972 )
1973 elif bucket_bound_hostname:
1974 url = f"{_bucket_bound_hostname_url(bucket_bound_hostname, scheme)}/"
1975 else:
1976 url = f"{self.api_endpoint}/{bucket_name}/"
1978 return {"url": url, "fields": policy_fields}
1981def _item_to_bucket(iterator, item):
1982 """Convert a JSON bucket to the native object.
1984 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
1985 :param iterator: The iterator that has retrieved the item.
1987 :type item: dict
1988 :param item: An item to be converted to a bucket.
1990 :rtype: :class:`.Bucket`
1991 :returns: The next bucket in the page.
1992 """
1993 name = item.get("name")
1994 bucket = Bucket(iterator.client, name)
1995 bucket._set_properties(item)
1996 return bucket
1999def _item_to_hmac_key_metadata(iterator, item):
2000 """Convert a JSON key metadata resource to the native object.
2002 :type iterator: :class:`~google.api_core.page_iterator.Iterator`
2003 :param iterator: The iterator that has retrieved the item.
2005 :type item: dict
2006 :param item: An item to be converted to a key metadata instance.
2008 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`
2009 :returns: The next key metadata instance in the page.
2010 """
2011 metadata = HMACKeyMetadata(iterator.client)
2012 metadata._properties = item
2013 return metadata