Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2014 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared helpers for Google Cloud packages.
17This module is not part of the public API surface.
18"""
20from __future__ import absolute_import
22import calendar
23import datetime
24import http.client
25import os
26import re
27from threading import local as Local
28from typing import Set, Union
30# PEP 0810: Explicit Lazy Imports
31# Python 3.15+ natively intercepts and defers these imports.
32# Developers can disable this behavior and force eager imports.
33# For more information, see:
34# https://docs.python.org/3.15/library/sys.html#sys.set_lazy_imports_filter
35# Older Python versions safely ignore this variable.
36# NOTE: We statically define all modules here to ensure static analysis tools
37# (mypy, pyright, Ruff) can easily parse them. If support is not present, the
38# imports are ignored, making their presence safe.
39__lazy_modules__: Set[str] = {
40 "google.auth",
41 "google.auth.transport.grpc",
42 "google.auth.transport.requests",
43 "google.protobuf.duration_pb2",
44 "google.protobuf.timestamp_pb2",
45 "grpc",
46}
48import google.auth
49import google.auth.transport.requests
50from google.protobuf import duration_pb2
51from google.protobuf import timestamp_pb2
53try:
54 import grpc
55 import google.auth.transport.grpc
56except ImportError: # pragma: NO COVER
57 grpc = None
59# `google.cloud._helpers._NOW` is deprecated
60_NOW = datetime.datetime.utcnow
61UTC = datetime.timezone.utc # Singleton instance to be used throughout.
62_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
64_RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ"
65_RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S"
66_TIMEONLY_W_MICROS = "%H:%M:%S.%f"
67_TIMEONLY_NO_FRACTION = "%H:%M:%S"
68# datetime.strptime cannot handle nanosecond precision: parse w/ regex
69_RFC3339_NANOS = re.compile(
70 r"""
71 (?P<no_fraction>
72 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS
73 )
74 ( # Optional decimal part
75 \. # decimal point
76 (?P<nanos>\d{1,9}) # nanoseconds, maybe truncated
77 )?
78 Z # Zulu
79""",
80 re.VERBOSE,
81)
82# NOTE: Catching this ImportError is a workaround for GAE not supporting the
83# "pwd" module which is imported lazily when "expanduser" is called.
84_USER_ROOT: Union[str, None]
85try:
86 _USER_ROOT = os.path.expanduser("~")
87except ImportError: # pragma: NO COVER
88 _USER_ROOT = None
89_GCLOUD_CONFIG_FILE = os.path.join("gcloud", "configurations", "config_default")
90_GCLOUD_CONFIG_SECTION = "core"
91_GCLOUD_CONFIG_KEY = "project"
94class _LocalStack(Local):
95 """Manage a thread-local LIFO stack of resources.
97 Intended for use in :class:`google.cloud.datastore.batch.Batch.__enter__`,
98 :class:`google.cloud.storage.batch.Batch.__enter__`, etc.
99 """
101 def __init__(self):
102 super(_LocalStack, self).__init__()
103 self._stack = []
105 def __iter__(self):
106 """Iterate the stack in LIFO order."""
107 return iter(reversed(self._stack))
109 def push(self, resource):
110 """Push a resource onto our stack."""
111 self._stack.append(resource)
113 def pop(self):
114 """Pop a resource from our stack.
116 :rtype: object
117 :returns: the top-most resource, after removing it.
118 :raises IndexError: if the stack is empty.
119 """
120 return self._stack.pop()
122 @property
123 def top(self):
124 """Get the top-most resource
126 :rtype: object
127 :returns: the top-most item, or None if the stack is empty.
128 """
129 if self._stack:
130 return self._stack[-1]
133def _ensure_tuple_or_list(arg_name, tuple_or_list):
134 """Ensures an input is a tuple or list.
136 This effectively reduces the iterable types allowed to a very short
137 allowlist: list and tuple.
139 :type arg_name: str
140 :param arg_name: Name of argument to use in error message.
142 :type tuple_or_list: sequence of str
143 :param tuple_or_list: Sequence to be verified.
145 :rtype: list of str
146 :returns: The ``tuple_or_list`` passed in cast to a ``list``.
147 :raises TypeError: if the ``tuple_or_list`` is not a tuple or list.
148 """
149 if not isinstance(tuple_or_list, (tuple, list)):
150 raise TypeError(
151 "Expected %s to be a tuple or list. "
152 "Received %r" % (arg_name, tuple_or_list)
153 )
154 return list(tuple_or_list)
157def _determine_default_project(project=None):
158 """Determine default project ID explicitly or implicitly as fall-back.
160 See :func:`google.auth.default` for details on how the default project
161 is determined.
163 :type project: str
164 :param project: Optional. The project name to use as default.
166 :rtype: str or ``NoneType``
167 :returns: Default project if it can be determined.
168 """
169 if project is None:
170 _, project = google.auth.default()
171 return project
174def _millis(when):
175 """Convert a zone-aware datetime to integer milliseconds.
177 :type when: :class:`datetime.datetime`
178 :param when: the datetime to convert
180 :rtype: int
181 :returns: milliseconds since epoch for ``when``
182 """
183 micros = _microseconds_from_datetime(when)
184 return micros // 1000
187def _datetime_from_microseconds(value):
188 """Convert timestamp to datetime, assuming UTC.
190 :type value: float
191 :param value: The timestamp to convert
193 :rtype: :class:`datetime.datetime`
194 :returns: The datetime object created from the value.
195 """
196 return _EPOCH + datetime.timedelta(microseconds=value)
199def _microseconds_from_datetime(value):
200 """Convert non-none datetime to microseconds.
202 :type value: :class:`datetime.datetime`
203 :param value: The timestamp to convert.
205 :rtype: int
206 :returns: The timestamp, in microseconds.
207 """
208 if not value.tzinfo:
209 value = value.replace(tzinfo=UTC)
210 # Regardless of what timezone is on the value, convert it to UTC.
211 value = value.astimezone(UTC)
212 # Convert the datetime to a microsecond timestamp.
213 return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond
216def _millis_from_datetime(value):
217 """Convert non-none datetime to timestamp, assuming UTC.
219 :type value: :class:`datetime.datetime`
220 :param value: (Optional) the timestamp
222 :rtype: int, or ``NoneType``
223 :returns: the timestamp, in milliseconds, or None
224 """
225 if value is not None:
226 return _millis(value)
229def _date_from_iso8601_date(value):
230 """Convert a ISO8601 date string to native datetime date
232 :type value: str
233 :param value: The date string to convert
235 :rtype: :class:`datetime.date`
236 :returns: A datetime date object created from the string
238 """
239 return datetime.datetime.strptime(value, "%Y-%m-%d").date()
242def _time_from_iso8601_time_naive(value):
243 """Convert a zoneless ISO8601 time string to naive datetime time
245 :type value: str
246 :param value: The time string to convert
248 :rtype: :class:`datetime.time`
249 :returns: A datetime time object created from the string
250 :raises ValueError: if the value does not match a known format.
251 """
252 if len(value) == 8: # HH:MM:SS
253 fmt = _TIMEONLY_NO_FRACTION
254 elif len(value) == 15: # HH:MM:SS.micros
255 fmt = _TIMEONLY_W_MICROS
256 else:
257 raise ValueError("Unknown time format: {}".format(value))
258 return datetime.datetime.strptime(value, fmt).time()
261def _rfc3339_to_datetime(dt_str):
262 """Convert a microsecond-precision timestamp to a native datetime.
264 :type dt_str: str
265 :param dt_str: The string to convert.
267 :rtype: :class:`datetime.datetime`
268 :returns: The datetime object created from the string.
269 """
270 return datetime.datetime.strptime(dt_str, _RFC3339_MICROS).replace(tzinfo=UTC)
273def _rfc3339_nanos_to_datetime(dt_str):
274 """Convert a nanosecond-precision timestamp to a native datetime.
276 .. note::
278 Python datetimes do not support nanosecond precision; this function
279 therefore truncates such values to microseconds.
281 :type dt_str: str
282 :param dt_str: The string to convert.
284 :rtype: :class:`datetime.datetime`
285 :returns: The datetime object created from the string.
286 :raises ValueError: If the timestamp does not match the RFC 3339
287 regular expression.
288 """
289 with_nanos = _RFC3339_NANOS.match(dt_str)
290 if with_nanos is None:
291 raise ValueError(
292 "Timestamp: %r, does not match pattern: %r"
293 % (dt_str, _RFC3339_NANOS.pattern)
294 )
295 bare_seconds = datetime.datetime.strptime(
296 with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION
297 )
298 fraction = with_nanos.group("nanos")
299 if fraction is None:
300 micros = 0
301 else:
302 scale = 9 - len(fraction)
303 nanos = int(fraction) * (10**scale)
304 micros = nanos // 1000
305 return bare_seconds.replace(microsecond=micros, tzinfo=UTC)
308def _datetime_to_rfc3339(value, ignore_zone=True):
309 """Convert a timestamp to a string.
311 :type value: :class:`datetime.datetime`
312 :param value: The datetime object to be converted to a string.
314 :type ignore_zone: bool
315 :param ignore_zone: If True, then the timezone (if any) of the datetime
316 object is ignored.
318 :rtype: str
319 :returns: The string representing the datetime stamp.
320 """
321 if not ignore_zone and value.tzinfo is not None:
322 # Convert to UTC and remove the time zone info.
323 value = value.replace(tzinfo=None) - value.utcoffset()
325 return value.strftime(_RFC3339_MICROS)
328def _to_bytes(value, encoding="ascii"):
329 """Converts a string value to bytes, if necessary.
331 :type value: str / bytes or unicode
332 :param value: The string/bytes value to be converted.
334 :type encoding: str
335 :param encoding: The encoding to use to convert unicode to bytes. Defaults
336 to "ascii", which will not allow any characters from
337 ordinals larger than 127. Other useful values are
338 "latin-1", which which will only allows byte ordinals
339 (up to 255) and "utf-8", which will encode any unicode
340 that needs to be.
342 :rtype: str / bytes
343 :returns: The original value converted to bytes (if unicode) or as passed
344 in if it started out as bytes.
345 :raises TypeError: if the value could not be converted to bytes.
346 """
347 result = value.encode(encoding) if isinstance(value, str) else value
348 if isinstance(result, bytes):
349 return result
350 else:
351 raise TypeError("%r could not be converted to bytes" % (value,))
354def _bytes_to_unicode(value):
355 """Converts bytes to a unicode value, if necessary.
357 :type value: bytes
358 :param value: bytes value to attempt string conversion on.
360 :rtype: str
361 :returns: The original value converted to unicode (if bytes) or as passed
362 in if it started out as unicode.
364 :raises ValueError: if the value could not be converted to unicode.
365 """
366 result = value.decode("utf-8") if isinstance(value, bytes) else value
367 if isinstance(result, str):
368 return result
369 else:
370 raise ValueError("%r could not be converted to unicode" % (value,))
373def _from_any_pb(pb_type, any_pb):
374 """Converts an Any protobuf to the specified message type
376 Args:
377 pb_type (type): the type of the message that any_pb stores an instance
378 of.
379 any_pb (google.protobuf.any_pb2.Any): the object to be converted.
381 Returns:
382 pb_type: An instance of the pb_type message.
384 Raises:
385 TypeError: if the message could not be converted.
386 """
387 msg = pb_type()
388 if not any_pb.Unpack(msg):
389 raise TypeError(
390 "Could not convert {} to {}".format(
391 any_pb.__class__.__name__, pb_type.__name__
392 )
393 )
395 return msg
398def _pb_timestamp_to_datetime(timestamp_pb):
399 """Convert a Timestamp protobuf to a datetime object.
401 :type timestamp_pb: :class:`google.protobuf.timestamp_pb2.Timestamp`
402 :param timestamp_pb: A Google returned timestamp protobuf.
404 :rtype: :class:`datetime.datetime`
405 :returns: A UTC datetime object converted from a protobuf timestamp.
406 """
407 return _EPOCH + datetime.timedelta(
408 seconds=timestamp_pb.seconds, microseconds=(timestamp_pb.nanos / 1000.0)
409 )
412def _pb_timestamp_to_rfc3339(timestamp_pb):
413 """Convert a Timestamp protobuf to an RFC 3339 string.
415 :type timestamp_pb: :class:`google.protobuf.timestamp_pb2.Timestamp`
416 :param timestamp_pb: A Google returned timestamp protobuf.
418 :rtype: str
419 :returns: An RFC 3339 formatted timestamp string.
420 """
421 timestamp = _pb_timestamp_to_datetime(timestamp_pb)
422 return _datetime_to_rfc3339(timestamp)
425def _datetime_to_pb_timestamp(when):
426 """Convert a datetime object to a Timestamp protobuf.
428 :type when: :class:`datetime.datetime`
429 :param when: the datetime to convert
431 :rtype: :class:`google.protobuf.timestamp_pb2.Timestamp`
432 :returns: A timestamp protobuf corresponding to the object.
433 """
434 ms_value = _microseconds_from_datetime(when)
435 seconds, micros = divmod(ms_value, 10**6)
436 nanos = micros * 10**3
437 return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
440def _timedelta_to_duration_pb(timedelta_val):
441 """Convert a Python timedelta object to a duration protobuf.
443 .. note::
445 The Python timedelta has a granularity of microseconds while
446 the protobuf duration type has a duration of nanoseconds.
448 :type timedelta_val: :class:`datetime.timedelta`
449 :param timedelta_val: A timedelta object.
451 :rtype: :class:`google.protobuf.duration_pb2.Duration`
452 :returns: A duration object equivalent to the time delta.
453 """
454 duration_pb = duration_pb2.Duration()
455 duration_pb.FromTimedelta(timedelta_val)
456 return duration_pb
459def _duration_pb_to_timedelta(duration_pb):
460 """Convert a duration protobuf to a Python timedelta object.
462 .. note::
464 The Python timedelta has a granularity of microseconds while
465 the protobuf duration type has a duration of nanoseconds.
467 :type duration_pb: :class:`google.protobuf.duration_pb2.Duration`
468 :param duration_pb: A protobuf duration object.
470 :rtype: :class:`datetime.timedelta`
471 :returns: The converted timedelta object.
472 """
473 return datetime.timedelta(
474 seconds=duration_pb.seconds, microseconds=(duration_pb.nanos / 1000.0)
475 )
478def _name_from_project_path(path, project, template):
479 """Validate a URI path and get the leaf object's name.
481 :type path: str
482 :param path: URI path containing the name.
484 :type project: str
485 :param project: (Optional) The project associated with the request. It is
486 included for validation purposes. If passed as None,
487 disables validation.
489 :type template: str
490 :param template: Template regex describing the expected form of the path.
491 The regex must have two named groups, 'project' and
492 'name'.
494 :rtype: str
495 :returns: Name parsed from ``path``.
496 :raises ValueError: if the ``path`` is ill-formed or if the project from
497 the ``path`` does not agree with the ``project``
498 passed in.
499 """
500 if isinstance(template, str):
501 template = re.compile(template)
503 match = template.match(path)
505 if not match:
506 raise ValueError(
507 'path "%s" did not match expected pattern "%s"' % (path, template.pattern)
508 )
510 if project is not None:
511 found_project = match.group("project")
512 if found_project != project:
513 raise ValueError(
514 "Project from client (%s) should agree with "
515 "project from resource(%s)." % (project, found_project)
516 )
518 return match.group("name")
521def make_secure_channel(credentials, user_agent, host, extra_options=()):
522 """Makes a secure channel for an RPC service.
524 Uses / depends on gRPC.
526 :type credentials: :class:`google.auth.credentials.Credentials`
527 :param credentials: The OAuth2 Credentials to use for creating
528 access tokens.
530 :type user_agent: str
531 :param user_agent: The user agent to be used with API requests.
533 :type host: str
534 :param host: The host for the service.
536 :type extra_options: tuple
537 :param extra_options: (Optional) Extra gRPC options used when creating the
538 channel.
540 :rtype: :class:`grpc._channel.Channel`
541 :returns: gRPC secure channel with credentials attached.
542 """
543 target = "%s:%d" % (host, http.client.HTTPS_PORT)
544 http_request = google.auth.transport.requests.Request()
546 user_agent_option = ("grpc.primary_user_agent", user_agent)
547 options = (user_agent_option,) + extra_options
548 return google.auth.transport.grpc.secure_authorized_channel(
549 credentials, http_request, target, options=options
550 )
553def make_secure_stub(credentials, user_agent, stub_class, host, extra_options=()):
554 """Makes a secure stub for an RPC service.
556 Uses / depends on gRPC.
558 :type credentials: :class:`google.auth.credentials.Credentials`
559 :param credentials: The OAuth2 Credentials to use for creating
560 access tokens.
562 :type user_agent: str
563 :param user_agent: The user agent to be used with API requests.
565 :type stub_class: type
566 :param stub_class: A gRPC stub type for a given service.
568 :type host: str
569 :param host: The host for the service.
571 :type extra_options: tuple
572 :param extra_options: (Optional) Extra gRPC options passed when creating
573 the channel.
575 :rtype: object, instance of ``stub_class``
576 :returns: The stub object used to make gRPC requests to a given API.
577 """
578 channel = make_secure_channel(
579 credentials, user_agent, host, extra_options=extra_options
580 )
581 return stub_class(channel)
584def make_insecure_stub(stub_class, host, port=None):
585 """Makes an insecure stub for an RPC service.
587 Uses / depends on gRPC.
589 :type stub_class: type
590 :param stub_class: A gRPC stub type for a given service.
592 :type host: str
593 :param host: The host for the service. May also include the port
594 if ``port`` is unspecified.
596 :type port: int
597 :param port: (Optional) The port for the service.
599 :rtype: object, instance of ``stub_class``
600 :returns: The stub object used to make gRPC requests to a given API.
601 """
602 if port is None:
603 target = host
604 else:
605 # NOTE: This assumes port != http.client.HTTPS_PORT:
606 target = "%s:%d" % (host, port)
607 channel = grpc.insecure_channel(target)
608 return stub_class(channel)