Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/bigquery_storage_v1/types/stream.py: 100%

55 statements  

« prev     ^ index     » next       coverage.py v7.2.2, created at 2023-03-26 06:10 +0000

1# -*- coding: utf-8 -*- 

2# Copyright 2022 Google LLC 

3# 

4# Licensed under the Apache License, Version 2.0 (the "License"); 

5# you may not use this file except in compliance with the License. 

6# You may obtain a copy of the License at 

7# 

8# http://www.apache.org/licenses/LICENSE-2.0 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15# 

16from __future__ import annotations 

17 

18from typing import MutableMapping, MutableSequence 

19 

20import proto # type: ignore 

21 

22from google.cloud.bigquery_storage_v1.types import arrow 

23from google.cloud.bigquery_storage_v1.types import avro 

24from google.cloud.bigquery_storage_v1.types import table as gcbs_table 

25from google.protobuf import timestamp_pb2 # type: ignore 

26 

27 

28__protobuf__ = proto.module( 

29 package="google.cloud.bigquery.storage.v1", 

30 manifest={ 

31 "DataFormat", 

32 "WriteStreamView", 

33 "ReadSession", 

34 "ReadStream", 

35 "WriteStream", 

36 }, 

37) 

38 

39 

40class DataFormat(proto.Enum): 

41 r"""Data format for input or output data. 

42 

43 Values: 

44 DATA_FORMAT_UNSPECIFIED (0): 

45 Data format is unspecified. 

46 AVRO (1): 

47 Avro is a standard open source row based file 

48 format. See https://avro.apache.org/ for more 

49 details. 

50 ARROW (2): 

51 Arrow is a standard open source column-based 

52 message format. See https://arrow.apache.org/ 

53 for more details. 

54 """ 

55 DATA_FORMAT_UNSPECIFIED = 0 

56 AVRO = 1 

57 ARROW = 2 

58 

59 

60class WriteStreamView(proto.Enum): 

61 r"""WriteStreamView is a view enum that controls what details 

62 about a write stream should be returned. 

63 

64 Values: 

65 WRITE_STREAM_VIEW_UNSPECIFIED (0): 

66 The default / unset value. 

67 BASIC (1): 

68 The BASIC projection returns basic metadata 

69 about a write stream. The basic view does not 

70 include schema information. This is the default 

71 view returned by GetWriteStream. 

72 FULL (2): 

73 The FULL projection returns all available 

74 write stream metadata, including the schema. 

75 CreateWriteStream returns the full projection of 

76 write stream metadata. 

77 """ 

78 WRITE_STREAM_VIEW_UNSPECIFIED = 0 

79 BASIC = 1 

80 FULL = 2 

81 

82 

83class ReadSession(proto.Message): 

84 r"""Information about the ReadSession. 

85 

86 This message has `oneof`_ fields (mutually exclusive fields). 

87 For each oneof, at most one member field can be set at the same time. 

88 Setting any member of the oneof automatically clears all other 

89 members. 

90 

91 .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields 

92 

93 Attributes: 

94 name (str): 

95 Output only. Unique identifier for the session, in the form 

96 ``projects/{project_id}/locations/{location}/sessions/{session_id}``. 

97 expire_time (google.protobuf.timestamp_pb2.Timestamp): 

98 Output only. Time at which the session becomes invalid. 

99 After this time, subsequent requests to read this Session 

100 will return errors. The expire_time is automatically 

101 assigned and currently cannot be specified or updated. 

102 data_format (google.cloud.bigquery_storage_v1.types.DataFormat): 

103 Immutable. Data format of the output data. 

104 DATA_FORMAT_UNSPECIFIED not supported. 

105 avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): 

106 Output only. Avro schema. 

107 

108 This field is a member of `oneof`_ ``schema``. 

109 arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): 

110 Output only. Arrow schema. 

111 

112 This field is a member of `oneof`_ ``schema``. 

113 table (str): 

114 Immutable. Table that this ReadSession is reading from, in 

115 the form 

116 ``projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`` 

117 table_modifiers (google.cloud.bigquery_storage_v1.types.ReadSession.TableModifiers): 

118 Optional. Any modifiers which are applied 

119 when reading from the specified table. 

120 read_options (google.cloud.bigquery_storage_v1.types.ReadSession.TableReadOptions): 

121 Optional. Read options for this session (e.g. 

122 column selection, filters). 

123 streams (MutableSequence[google.cloud.bigquery_storage_v1.types.ReadStream]): 

124 Output only. A list of streams created with the session. 

125 

126 At least one stream is created with the session. In the 

127 future, larger request_stream_count values *may* result in 

128 this list being unpopulated, in that case, the user will 

129 need to use a List method to get the streams instead, which 

130 is not yet available. 

131 estimated_total_bytes_scanned (int): 

132 Output only. An estimate on the number of 

133 bytes this session will scan when all streams 

134 are completely consumed. This estimate is based 

135 on metadata from the table which might be 

136 incomplete or stale. 

137 estimated_row_count (int): 

138 Output only. An estimate on the number of 

139 rows present in this session's streams. This 

140 estimate is based on metadata from the table 

141 which might be incomplete or stale. 

142 trace_id (str): 

143 Optional. ID set by client to annotate a 

144 session identity. This does not need to be 

145 strictly unique, but instead the same ID should 

146 be used to group logically connected sessions 

147 (e.g. All using the same ID for all sessions 

148 needed to complete a Spark SQL query is 

149 reasonable). 

150 Maximum length is 256 bytes. 

151 """ 

152 

153 class TableModifiers(proto.Message): 

154 r"""Additional attributes when reading a table. 

155 

156 Attributes: 

157 snapshot_time (google.protobuf.timestamp_pb2.Timestamp): 

158 The snapshot time of the table. If not set, 

159 interpreted as now. 

160 """ 

161 

162 snapshot_time: timestamp_pb2.Timestamp = proto.Field( 

163 proto.MESSAGE, 

164 number=1, 

165 message=timestamp_pb2.Timestamp, 

166 ) 

167 

168 class TableReadOptions(proto.Message): 

169 r"""Options dictating how we read a table. 

170 

171 This message has `oneof`_ fields (mutually exclusive fields). 

172 For each oneof, at most one member field can be set at the same time. 

173 Setting any member of the oneof automatically clears all other 

174 members. 

175 

176 .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields 

177 

178 Attributes: 

179 selected_fields (MutableSequence[str]): 

180 Optional. The names of the fields in the table to be 

181 returned. If no field names are specified, then all fields 

182 in the table are returned. 

183 

184 Nested fields -- the child elements of a STRUCT field -- can 

185 be selected individually using their fully-qualified names, 

186 and will be returned as record fields containing only the 

187 selected nested fields. If a STRUCT field is specified in 

188 the selected fields list, all of the child elements will be 

189 returned. 

190 

191 As an example, consider a table with the following schema: 

192 

193 { "name": "struct_field", "type": "RECORD", "mode": 

194 "NULLABLE", "fields": [ { "name": "string_field1", "type": 

195 "STRING", . "mode": "NULLABLE" }, { "name": "string_field2", 

196 "type": "STRING", "mode": "NULLABLE" } ] } 

197 

198 Specifying "struct_field" in the selected fields list will 

199 result in a read session schema with the following logical 

200 structure: 

201 

202 struct_field { string_field1 string_field2 } 

203 

204 Specifying "struct_field.string_field1" in the selected 

205 fields list will result in a read session schema with the 

206 following logical structure: 

207 

208 struct_field { string_field1 } 

209 

210 The order of the fields in the read session schema is 

211 derived from the table schema and does not correspond to the 

212 order in which the fields are specified in this list. 

213 row_restriction (str): 

214 SQL text filtering statement, similar to a WHERE clause in a 

215 query. Aggregates are not supported. 

216 

217 Examples: "int_field > 5" "date_field = CAST('2014-9-27' as 

218 DATE)" "nullable_field is not NULL" "st_equals(geo_field, 

219 st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 

220 AND 5.0" 

221 

222 Restricted to a maximum length for 1 MB. 

223 arrow_serialization_options (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions): 

224 Optional. Options specific to the Apache 

225 Arrow output format. 

226 

227 This field is a member of `oneof`_ ``output_format_serialization_options``. 

228 avro_serialization_options (google.cloud.bigquery_storage_v1.types.AvroSerializationOptions): 

229 Optional. Options specific to the Apache Avro 

230 output format 

231 

232 This field is a member of `oneof`_ ``output_format_serialization_options``. 

233 """ 

234 

235 selected_fields: MutableSequence[str] = proto.RepeatedField( 

236 proto.STRING, 

237 number=1, 

238 ) 

239 row_restriction: str = proto.Field( 

240 proto.STRING, 

241 number=2, 

242 ) 

243 arrow_serialization_options: arrow.ArrowSerializationOptions = proto.Field( 

244 proto.MESSAGE, 

245 number=3, 

246 oneof="output_format_serialization_options", 

247 message=arrow.ArrowSerializationOptions, 

248 ) 

249 avro_serialization_options: avro.AvroSerializationOptions = proto.Field( 

250 proto.MESSAGE, 

251 number=4, 

252 oneof="output_format_serialization_options", 

253 message=avro.AvroSerializationOptions, 

254 ) 

255 

256 name: str = proto.Field( 

257 proto.STRING, 

258 number=1, 

259 ) 

260 expire_time: timestamp_pb2.Timestamp = proto.Field( 

261 proto.MESSAGE, 

262 number=2, 

263 message=timestamp_pb2.Timestamp, 

264 ) 

265 data_format: "DataFormat" = proto.Field( 

266 proto.ENUM, 

267 number=3, 

268 enum="DataFormat", 

269 ) 

270 avro_schema: avro.AvroSchema = proto.Field( 

271 proto.MESSAGE, 

272 number=4, 

273 oneof="schema", 

274 message=avro.AvroSchema, 

275 ) 

276 arrow_schema: arrow.ArrowSchema = proto.Field( 

277 proto.MESSAGE, 

278 number=5, 

279 oneof="schema", 

280 message=arrow.ArrowSchema, 

281 ) 

282 table: str = proto.Field( 

283 proto.STRING, 

284 number=6, 

285 ) 

286 table_modifiers: TableModifiers = proto.Field( 

287 proto.MESSAGE, 

288 number=7, 

289 message=TableModifiers, 

290 ) 

291 read_options: TableReadOptions = proto.Field( 

292 proto.MESSAGE, 

293 number=8, 

294 message=TableReadOptions, 

295 ) 

296 streams: MutableSequence["ReadStream"] = proto.RepeatedField( 

297 proto.MESSAGE, 

298 number=10, 

299 message="ReadStream", 

300 ) 

301 estimated_total_bytes_scanned: int = proto.Field( 

302 proto.INT64, 

303 number=12, 

304 ) 

305 estimated_row_count: int = proto.Field( 

306 proto.INT64, 

307 number=14, 

308 ) 

309 trace_id: str = proto.Field( 

310 proto.STRING, 

311 number=13, 

312 ) 

313 

314 

315class ReadStream(proto.Message): 

316 r"""Information about a single stream that gets data out of the storage 

317 system. Most of the information about ``ReadStream`` instances is 

318 aggregated, making ``ReadStream`` lightweight. 

319 

320 Attributes: 

321 name (str): 

322 Output only. Name of the stream, in the form 

323 ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. 

324 """ 

325 

326 name: str = proto.Field( 

327 proto.STRING, 

328 number=1, 

329 ) 

330 

331 

332class WriteStream(proto.Message): 

333 r"""Information about a single stream that gets data inside the 

334 storage system. 

335 

336 Attributes: 

337 name (str): 

338 Output only. Name of the stream, in the form 

339 ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. 

340 type_ (google.cloud.bigquery_storage_v1.types.WriteStream.Type): 

341 Immutable. Type of the stream. 

342 create_time (google.protobuf.timestamp_pb2.Timestamp): 

343 Output only. Create time of the stream. For the \_default 

344 stream, this is the creation_time of the table. 

345 commit_time (google.protobuf.timestamp_pb2.Timestamp): 

346 Output only. Commit time of the stream. If a stream is of 

347 ``COMMITTED`` type, then it will have a commit_time same as 

348 ``create_time``. If the stream is of ``PENDING`` type, empty 

349 commit_time means it is not committed. 

350 table_schema (google.cloud.bigquery_storage_v1.types.TableSchema): 

351 Output only. The schema of the destination table. It is only 

352 returned in ``CreateWriteStream`` response. Caller should 

353 generate data that's compatible with this schema to send in 

354 initial ``AppendRowsRequest``. The table schema could go out 

355 of date during the life time of the stream. 

356 write_mode (google.cloud.bigquery_storage_v1.types.WriteStream.WriteMode): 

357 Immutable. Mode of the stream. 

358 location (str): 

359 Immutable. The geographic location where the 

360 stream's dataset resides. See 

361 https://cloud.google.com/bigquery/docs/locations 

362 for supported locations. 

363 """ 

364 

365 class Type(proto.Enum): 

366 r"""Type enum of the stream. 

367 

368 Values: 

369 TYPE_UNSPECIFIED (0): 

370 Unknown type. 

371 COMMITTED (1): 

372 Data will commit automatically and appear as 

373 soon as the write is acknowledged. 

374 PENDING (2): 

375 Data is invisible until the stream is 

376 committed. 

377 BUFFERED (3): 

378 Data is only visible up to the offset to 

379 which it was flushed. 

380 """ 

381 TYPE_UNSPECIFIED = 0 

382 COMMITTED = 1 

383 PENDING = 2 

384 BUFFERED = 3 

385 

386 class WriteMode(proto.Enum): 

387 r"""Mode enum of the stream. 

388 

389 Values: 

390 WRITE_MODE_UNSPECIFIED (0): 

391 Unknown type. 

392 INSERT (1): 

393 Insert new records into the table. 

394 It is the default value if customers do not 

395 specify it. 

396 """ 

397 WRITE_MODE_UNSPECIFIED = 0 

398 INSERT = 1 

399 

400 name: str = proto.Field( 

401 proto.STRING, 

402 number=1, 

403 ) 

404 type_: Type = proto.Field( 

405 proto.ENUM, 

406 number=2, 

407 enum=Type, 

408 ) 

409 create_time: timestamp_pb2.Timestamp = proto.Field( 

410 proto.MESSAGE, 

411 number=3, 

412 message=timestamp_pb2.Timestamp, 

413 ) 

414 commit_time: timestamp_pb2.Timestamp = proto.Field( 

415 proto.MESSAGE, 

416 number=4, 

417 message=timestamp_pb2.Timestamp, 

418 ) 

419 table_schema: gcbs_table.TableSchema = proto.Field( 

420 proto.MESSAGE, 

421 number=5, 

422 message=gcbs_table.TableSchema, 

423 ) 

424 write_mode: WriteMode = proto.Field( 

425 proto.ENUM, 

426 number=7, 

427 enum=WriteMode, 

428 ) 

429 location: str = proto.Field( 

430 proto.STRING, 

431 number=8, 

432 ) 

433 

434 

435__all__ = tuple(sorted(__protobuf__.manifest))