Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2014 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Shared helpers for Google Cloud packages.
17This module is not part of the public API surface.
18"""
20from __future__ import absolute_import
22import calendar
23import datetime
24import http.client
25import os
26import re
27from threading import local as Local
28from typing import Union
30import google.auth
31import google.auth.transport.requests
32from google.protobuf import duration_pb2
33from google.protobuf import timestamp_pb2
35try:
36 import grpc
37 import google.auth.transport.grpc
38except ImportError: # pragma: NO COVER
39 grpc = None
41# `google.cloud._helpers._NOW` is deprecated
42_NOW = datetime.datetime.utcnow
43UTC = datetime.timezone.utc # Singleton instance to be used throughout.
44_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
46_RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ"
47_RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S"
48_TIMEONLY_W_MICROS = "%H:%M:%S.%f"
49_TIMEONLY_NO_FRACTION = "%H:%M:%S"
50# datetime.strptime cannot handle nanosecond precision: parse w/ regex
51_RFC3339_NANOS = re.compile(
52 r"""
53 (?P<no_fraction>
54 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS
55 )
56 ( # Optional decimal part
57 \. # decimal point
58 (?P<nanos>\d{1,9}) # nanoseconds, maybe truncated
59 )?
60 Z # Zulu
61""",
62 re.VERBOSE,
63)
64# NOTE: Catching this ImportError is a workaround for GAE not supporting the
65# "pwd" module which is imported lazily when "expanduser" is called.
66_USER_ROOT: Union[str, None]
67try:
68 _USER_ROOT = os.path.expanduser("~")
69except ImportError: # pragma: NO COVER
70 _USER_ROOT = None
71_GCLOUD_CONFIG_FILE = os.path.join("gcloud", "configurations", "config_default")
72_GCLOUD_CONFIG_SECTION = "core"
73_GCLOUD_CONFIG_KEY = "project"
76class _LocalStack(Local):
77 """Manage a thread-local LIFO stack of resources.
79 Intended for use in :class:`google.cloud.datastore.batch.Batch.__enter__`,
80 :class:`google.cloud.storage.batch.Batch.__enter__`, etc.
81 """
83 def __init__(self):
84 super(_LocalStack, self).__init__()
85 self._stack = []
87 def __iter__(self):
88 """Iterate the stack in LIFO order."""
89 return iter(reversed(self._stack))
91 def push(self, resource):
92 """Push a resource onto our stack."""
93 self._stack.append(resource)
95 def pop(self):
96 """Pop a resource from our stack.
98 :rtype: object
99 :returns: the top-most resource, after removing it.
100 :raises IndexError: if the stack is empty.
101 """
102 return self._stack.pop()
104 @property
105 def top(self):
106 """Get the top-most resource
108 :rtype: object
109 :returns: the top-most item, or None if the stack is empty.
110 """
111 if self._stack:
112 return self._stack[-1]
115def _ensure_tuple_or_list(arg_name, tuple_or_list):
116 """Ensures an input is a tuple or list.
118 This effectively reduces the iterable types allowed to a very short
119 allowlist: list and tuple.
121 :type arg_name: str
122 :param arg_name: Name of argument to use in error message.
124 :type tuple_or_list: sequence of str
125 :param tuple_or_list: Sequence to be verified.
127 :rtype: list of str
128 :returns: The ``tuple_or_list`` passed in cast to a ``list``.
129 :raises TypeError: if the ``tuple_or_list`` is not a tuple or list.
130 """
131 if not isinstance(tuple_or_list, (tuple, list)):
132 raise TypeError(
133 "Expected %s to be a tuple or list. "
134 "Received %r" % (arg_name, tuple_or_list)
135 )
136 return list(tuple_or_list)
139def _determine_default_project(project=None):
140 """Determine default project ID explicitly or implicitly as fall-back.
142 See :func:`google.auth.default` for details on how the default project
143 is determined.
145 :type project: str
146 :param project: Optional. The project name to use as default.
148 :rtype: str or ``NoneType``
149 :returns: Default project if it can be determined.
150 """
151 if project is None:
152 _, project = google.auth.default()
153 return project
156def _millis(when):
157 """Convert a zone-aware datetime to integer milliseconds.
159 :type when: :class:`datetime.datetime`
160 :param when: the datetime to convert
162 :rtype: int
163 :returns: milliseconds since epoch for ``when``
164 """
165 micros = _microseconds_from_datetime(when)
166 return micros // 1000
169def _datetime_from_microseconds(value):
170 """Convert timestamp to datetime, assuming UTC.
172 :type value: float
173 :param value: The timestamp to convert
175 :rtype: :class:`datetime.datetime`
176 :returns: The datetime object created from the value.
177 """
178 return _EPOCH + datetime.timedelta(microseconds=value)
181def _microseconds_from_datetime(value):
182 """Convert non-none datetime to microseconds.
184 :type value: :class:`datetime.datetime`
185 :param value: The timestamp to convert.
187 :rtype: int
188 :returns: The timestamp, in microseconds.
189 """
190 if not value.tzinfo:
191 value = value.replace(tzinfo=UTC)
192 # Regardless of what timezone is on the value, convert it to UTC.
193 value = value.astimezone(UTC)
194 # Convert the datetime to a microsecond timestamp.
195 return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond
198def _millis_from_datetime(value):
199 """Convert non-none datetime to timestamp, assuming UTC.
201 :type value: :class:`datetime.datetime`
202 :param value: (Optional) the timestamp
204 :rtype: int, or ``NoneType``
205 :returns: the timestamp, in milliseconds, or None
206 """
207 if value is not None:
208 return _millis(value)
211def _date_from_iso8601_date(value):
212 """Convert a ISO8601 date string to native datetime date
214 :type value: str
215 :param value: The date string to convert
217 :rtype: :class:`datetime.date`
218 :returns: A datetime date object created from the string
220 """
221 return datetime.datetime.strptime(value, "%Y-%m-%d").date()
224def _time_from_iso8601_time_naive(value):
225 """Convert a zoneless ISO8601 time string to naive datetime time
227 :type value: str
228 :param value: The time string to convert
230 :rtype: :class:`datetime.time`
231 :returns: A datetime time object created from the string
232 :raises ValueError: if the value does not match a known format.
233 """
234 if len(value) == 8: # HH:MM:SS
235 fmt = _TIMEONLY_NO_FRACTION
236 elif len(value) == 15: # HH:MM:SS.micros
237 fmt = _TIMEONLY_W_MICROS
238 else:
239 raise ValueError("Unknown time format: {}".format(value))
240 return datetime.datetime.strptime(value, fmt).time()
243def _rfc3339_to_datetime(dt_str):
244 """Convert a microsecond-precision timestamp to a native datetime.
246 :type dt_str: str
247 :param dt_str: The string to convert.
249 :rtype: :class:`datetime.datetime`
250 :returns: The datetime object created from the string.
251 """
252 return datetime.datetime.strptime(dt_str, _RFC3339_MICROS).replace(tzinfo=UTC)
255def _rfc3339_nanos_to_datetime(dt_str):
256 """Convert a nanosecond-precision timestamp to a native datetime.
258 .. note::
260 Python datetimes do not support nanosecond precision; this function
261 therefore truncates such values to microseconds.
263 :type dt_str: str
264 :param dt_str: The string to convert.
266 :rtype: :class:`datetime.datetime`
267 :returns: The datetime object created from the string.
268 :raises ValueError: If the timestamp does not match the RFC 3339
269 regular expression.
270 """
271 with_nanos = _RFC3339_NANOS.match(dt_str)
272 if with_nanos is None:
273 raise ValueError(
274 "Timestamp: %r, does not match pattern: %r"
275 % (dt_str, _RFC3339_NANOS.pattern)
276 )
277 bare_seconds = datetime.datetime.strptime(
278 with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION
279 )
280 fraction = with_nanos.group("nanos")
281 if fraction is None:
282 micros = 0
283 else:
284 scale = 9 - len(fraction)
285 nanos = int(fraction) * (10**scale)
286 micros = nanos // 1000
287 return bare_seconds.replace(microsecond=micros, tzinfo=UTC)
290def _datetime_to_rfc3339(value, ignore_zone=True):
291 """Convert a timestamp to a string.
293 :type value: :class:`datetime.datetime`
294 :param value: The datetime object to be converted to a string.
296 :type ignore_zone: bool
297 :param ignore_zone: If True, then the timezone (if any) of the datetime
298 object is ignored.
300 :rtype: str
301 :returns: The string representing the datetime stamp.
302 """
303 if not ignore_zone and value.tzinfo is not None:
304 # Convert to UTC and remove the time zone info.
305 value = value.replace(tzinfo=None) - value.utcoffset()
307 return value.strftime(_RFC3339_MICROS)
310def _to_bytes(value, encoding="ascii"):
311 """Converts a string value to bytes, if necessary.
313 :type value: str / bytes or unicode
314 :param value: The string/bytes value to be converted.
316 :type encoding: str
317 :param encoding: The encoding to use to convert unicode to bytes. Defaults
318 to "ascii", which will not allow any characters from
319 ordinals larger than 127. Other useful values are
320 "latin-1", which which will only allows byte ordinals
321 (up to 255) and "utf-8", which will encode any unicode
322 that needs to be.
324 :rtype: str / bytes
325 :returns: The original value converted to bytes (if unicode) or as passed
326 in if it started out as bytes.
327 :raises TypeError: if the value could not be converted to bytes.
328 """
329 result = value.encode(encoding) if isinstance(value, str) else value
330 if isinstance(result, bytes):
331 return result
332 else:
333 raise TypeError("%r could not be converted to bytes" % (value,))
336def _bytes_to_unicode(value):
337 """Converts bytes to a unicode value, if necessary.
339 :type value: bytes
340 :param value: bytes value to attempt string conversion on.
342 :rtype: str
343 :returns: The original value converted to unicode (if bytes) or as passed
344 in if it started out as unicode.
346 :raises ValueError: if the value could not be converted to unicode.
347 """
348 result = value.decode("utf-8") if isinstance(value, bytes) else value
349 if isinstance(result, str):
350 return result
351 else:
352 raise ValueError("%r could not be converted to unicode" % (value,))
355def _from_any_pb(pb_type, any_pb):
356 """Converts an Any protobuf to the specified message type
358 Args:
359 pb_type (type): the type of the message that any_pb stores an instance
360 of.
361 any_pb (google.protobuf.any_pb2.Any): the object to be converted.
363 Returns:
364 pb_type: An instance of the pb_type message.
366 Raises:
367 TypeError: if the message could not be converted.
368 """
369 msg = pb_type()
370 if not any_pb.Unpack(msg):
371 raise TypeError(
372 "Could not convert {} to {}".format(
373 any_pb.__class__.__name__, pb_type.__name__
374 )
375 )
377 return msg
380def _pb_timestamp_to_datetime(timestamp_pb):
381 """Convert a Timestamp protobuf to a datetime object.
383 :type timestamp_pb: :class:`google.protobuf.timestamp_pb2.Timestamp`
384 :param timestamp_pb: A Google returned timestamp protobuf.
386 :rtype: :class:`datetime.datetime`
387 :returns: A UTC datetime object converted from a protobuf timestamp.
388 """
389 return _EPOCH + datetime.timedelta(
390 seconds=timestamp_pb.seconds, microseconds=(timestamp_pb.nanos / 1000.0)
391 )
394def _pb_timestamp_to_rfc3339(timestamp_pb):
395 """Convert a Timestamp protobuf to an RFC 3339 string.
397 :type timestamp_pb: :class:`google.protobuf.timestamp_pb2.Timestamp`
398 :param timestamp_pb: A Google returned timestamp protobuf.
400 :rtype: str
401 :returns: An RFC 3339 formatted timestamp string.
402 """
403 timestamp = _pb_timestamp_to_datetime(timestamp_pb)
404 return _datetime_to_rfc3339(timestamp)
407def _datetime_to_pb_timestamp(when):
408 """Convert a datetime object to a Timestamp protobuf.
410 :type when: :class:`datetime.datetime`
411 :param when: the datetime to convert
413 :rtype: :class:`google.protobuf.timestamp_pb2.Timestamp`
414 :returns: A timestamp protobuf corresponding to the object.
415 """
416 ms_value = _microseconds_from_datetime(when)
417 seconds, micros = divmod(ms_value, 10**6)
418 nanos = micros * 10**3
419 return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
422def _timedelta_to_duration_pb(timedelta_val):
423 """Convert a Python timedelta object to a duration protobuf.
425 .. note::
427 The Python timedelta has a granularity of microseconds while
428 the protobuf duration type has a duration of nanoseconds.
430 :type timedelta_val: :class:`datetime.timedelta`
431 :param timedelta_val: A timedelta object.
433 :rtype: :class:`google.protobuf.duration_pb2.Duration`
434 :returns: A duration object equivalent to the time delta.
435 """
436 duration_pb = duration_pb2.Duration()
437 duration_pb.FromTimedelta(timedelta_val)
438 return duration_pb
441def _duration_pb_to_timedelta(duration_pb):
442 """Convert a duration protobuf to a Python timedelta object.
444 .. note::
446 The Python timedelta has a granularity of microseconds while
447 the protobuf duration type has a duration of nanoseconds.
449 :type duration_pb: :class:`google.protobuf.duration_pb2.Duration`
450 :param duration_pb: A protobuf duration object.
452 :rtype: :class:`datetime.timedelta`
453 :returns: The converted timedelta object.
454 """
455 return datetime.timedelta(
456 seconds=duration_pb.seconds, microseconds=(duration_pb.nanos / 1000.0)
457 )
460def _name_from_project_path(path, project, template):
461 """Validate a URI path and get the leaf object's name.
463 :type path: str
464 :param path: URI path containing the name.
466 :type project: str
467 :param project: (Optional) The project associated with the request. It is
468 included for validation purposes. If passed as None,
469 disables validation.
471 :type template: str
472 :param template: Template regex describing the expected form of the path.
473 The regex must have two named groups, 'project' and
474 'name'.
476 :rtype: str
477 :returns: Name parsed from ``path``.
478 :raises ValueError: if the ``path`` is ill-formed or if the project from
479 the ``path`` does not agree with the ``project``
480 passed in.
481 """
482 if isinstance(template, str):
483 template = re.compile(template)
485 match = template.match(path)
487 if not match:
488 raise ValueError(
489 'path "%s" did not match expected pattern "%s"' % (path, template.pattern)
490 )
492 if project is not None:
493 found_project = match.group("project")
494 if found_project != project:
495 raise ValueError(
496 "Project from client (%s) should agree with "
497 "project from resource(%s)." % (project, found_project)
498 )
500 return match.group("name")
503def make_secure_channel(credentials, user_agent, host, extra_options=()):
504 """Makes a secure channel for an RPC service.
506 Uses / depends on gRPC.
508 :type credentials: :class:`google.auth.credentials.Credentials`
509 :param credentials: The OAuth2 Credentials to use for creating
510 access tokens.
512 :type user_agent: str
513 :param user_agent: The user agent to be used with API requests.
515 :type host: str
516 :param host: The host for the service.
518 :type extra_options: tuple
519 :param extra_options: (Optional) Extra gRPC options used when creating the
520 channel.
522 :rtype: :class:`grpc._channel.Channel`
523 :returns: gRPC secure channel with credentials attached.
524 """
525 target = "%s:%d" % (host, http.client.HTTPS_PORT)
526 http_request = google.auth.transport.requests.Request()
528 user_agent_option = ("grpc.primary_user_agent", user_agent)
529 options = (user_agent_option,) + extra_options
530 return google.auth.transport.grpc.secure_authorized_channel(
531 credentials, http_request, target, options=options
532 )
535def make_secure_stub(credentials, user_agent, stub_class, host, extra_options=()):
536 """Makes a secure stub for an RPC service.
538 Uses / depends on gRPC.
540 :type credentials: :class:`google.auth.credentials.Credentials`
541 :param credentials: The OAuth2 Credentials to use for creating
542 access tokens.
544 :type user_agent: str
545 :param user_agent: The user agent to be used with API requests.
547 :type stub_class: type
548 :param stub_class: A gRPC stub type for a given service.
550 :type host: str
551 :param host: The host for the service.
553 :type extra_options: tuple
554 :param extra_options: (Optional) Extra gRPC options passed when creating
555 the channel.
557 :rtype: object, instance of ``stub_class``
558 :returns: The stub object used to make gRPC requests to a given API.
559 """
560 channel = make_secure_channel(
561 credentials, user_agent, host, extra_options=extra_options
562 )
563 return stub_class(channel)
566def make_insecure_stub(stub_class, host, port=None):
567 """Makes an insecure stub for an RPC service.
569 Uses / depends on gRPC.
571 :type stub_class: type
572 :param stub_class: A gRPC stub type for a given service.
574 :type host: str
575 :param host: The host for the service. May also include the port
576 if ``port`` is unspecified.
578 :type port: int
579 :param port: (Optional) The port for the service.
581 :rtype: object, instance of ``stub_class``
582 :returns: The stub object used to make gRPC requests to a given API.
583 """
584 if port is None:
585 target = host
586 else:
587 # NOTE: This assumes port != http.client.HTTPS_PORT:
588 target = "%s:%d" % (host, port)
589 channel = grpc.insecure_channel(target)
590 return stub_class(channel)