Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#
3# Copyright 2020 Google LLC
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# https://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
17"""Parent client for calling the Cloud BigQuery Storage API.
19This is the base from which all interactions with the API occur.
20"""
22from __future__ import absolute_import
24import google.api_core.gapic_v1.method
26from google.cloud.bigquery_storage_v1 import reader
27from google.cloud.bigquery_storage_v1.services import big_query_read, big_query_write
29_SCOPES = (
30 "https://www.googleapis.com/auth/bigquery",
31 "https://www.googleapis.com/auth/cloud-platform",
32)
35class BigQueryReadClient(big_query_read.BigQueryReadClient):
36 """Client for interacting with BigQuery Storage API.
38 The BigQuery storage API can be used to read data stored in BigQuery.
39 """
41 def read_rows(
42 self,
43 name,
44 offset=0,
45 retry=google.api_core.gapic_v1.method.DEFAULT,
46 timeout=google.api_core.gapic_v1.method.DEFAULT,
47 metadata=(),
48 retry_delay_callback=None,
49 ):
50 """
51 Reads rows from the table in the format prescribed by the read
52 session. Each response contains one or more table rows, up to a
53 maximum of 10 MiB per response; read requests which attempt to read
54 individual rows larger than this will fail.
56 Each request also returns a set of stream statistics reflecting the
57 estimated total number of rows in the read stream. This number is
58 computed based on the total table size and the number of active
59 streams in the read session, and may change as other streams continue
60 to read data.
62 Example:
63 >>> from google.cloud import bigquery_storage
64 >>>
65 >>> client = bigquery_storage.BigQueryReadClient()
66 >>>
67 >>> # TODO: Initialize ``table``:
68 >>> table = "projects/{}/datasets/{}/tables/{}".format(
69 ... 'project_id': 'your-data-project-id',
70 ... 'dataset_id': 'your_dataset_id',
71 ... 'table_id': 'your_table_id',
72 ... )
73 >>>
74 >>> # TODO: Initialize `parent`:
75 >>> parent = 'projects/your-billing-project-id'
76 >>>
77 >>> requested_session = bigquery_storage.types.ReadSession(
78 ... table=table,
79 ... data_format=bigquery_storage.types.DataFormat.AVRO,
80 ... )
81 >>> session = client.create_read_session(
82 ... parent=parent, read_session=requested_session
83 ... )
84 >>>
85 >>> stream = session.streams[0], # TODO: Also read any other streams.
86 >>> read_rows_stream = client.read_rows(stream.name)
87 >>>
88 >>> for element in read_rows_stream.rows(session):
89 ... # process element
90 ... pass
92 Args:
93 name (str):
94 Required. Name of the stream to start
95 reading from, of the form
96 `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`
97 offset (Optional[int]):
98 The starting offset from which to begin reading rows from
99 in the stream. The offset requested must be less than the last
100 row read from ReadRows. Requesting a larger offset is
101 undefined.
102 retry (Optional[google.api_core.retry.Retry]): A retry object used
103 to retry requests. If ``None`` is specified, requests will not
104 be retried.
105 timeout (Optional[float]): The amount of time, in seconds, to wait
106 for the request to complete. Note that if ``retry`` is
107 specified, the timeout applies to each individual attempt.
108 metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata
109 that is provided to the method.
110 retry_delay_callback (Optional[Callable[[float], None]]):
111 If the client receives a retryable error that asks the client to
112 delay its next attempt and retry_delay_callback is not None,
113 BigQueryReadClient will call retry_delay_callback with the delay
114 duration (in seconds) before it starts sleeping until the next
115 attempt.
117 Returns:
118 ~google.cloud.bigquery_storage_v1.reader.ReadRowsStream:
119 An iterable of
120 :class:`~google.cloud.bigquery_storage_v1.types.ReadRowsResponse`.
122 Raises:
123 google.api_core.exceptions.GoogleAPICallError: If the request
124 failed for any reason.
125 google.api_core.exceptions.RetryError: If the request failed due
126 to a retryable error and retry attempts failed.
127 ValueError: If the parameters are invalid.
128 """
129 gapic_client = super(BigQueryReadClient, self)
130 stream = reader.ReadRowsStream(
131 gapic_client,
132 name,
133 offset,
134 {"retry": retry, "timeout": timeout, "metadata": metadata},
135 retry_delay_callback=retry_delay_callback,
136 )
137 stream._reconnect()
138 return stream
141class BigQueryWriteClient(big_query_write.BigQueryWriteClient):
142 __doc__ = big_query_write.BigQueryWriteClient.__doc__