Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/external_config.py: 54%

1# Copyright 2017 Google LLC 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15"""Define classes that describe external data sources. 

16 

17 These are used for both Table.externalDataConfiguration and 

18 Job.configuration.query.tableDefinitions. 

19""" 

20 

21from __future__ import absolute_import 

22 

23import base64 

24import copy 

25from typing import Any, Dict, FrozenSet, Iterable, Optional, Union 

26 

27from google.cloud.bigquery._helpers import _to_bytes 

28from google.cloud.bigquery._helpers import _bytes_to_json 

29from google.cloud.bigquery._helpers import _int_or_none 

30from google.cloud.bigquery._helpers import _str_or_none 

31from google.cloud.bigquery.format_options import AvroOptions, ParquetOptions 

32from google.cloud.bigquery.schema import SchemaField 

33 

34 

35class ExternalSourceFormat(object): 

36 """The format for external data files. 

37 

38 Note that the set of allowed values for external data sources is different 

39 than the set used for loading data (see 

40 :class:`~google.cloud.bigquery.job.SourceFormat`). 

41 """ 

42 

43 CSV = "CSV" 

44 """Specifies CSV format.""" 

45 

46 GOOGLE_SHEETS = "GOOGLE_SHEETS" 

47 """Specifies Google Sheets format.""" 

48 

49 NEWLINE_DELIMITED_JSON = "NEWLINE_DELIMITED_JSON" 

50 """Specifies newline delimited JSON format.""" 

51 

52 AVRO = "AVRO" 

53 """Specifies Avro format.""" 

54 

55 DATASTORE_BACKUP = "DATASTORE_BACKUP" 

56 """Specifies datastore backup format""" 

57 

58 ORC = "ORC" 

59 """Specifies ORC format.""" 

60 

61 PARQUET = "PARQUET" 

62 """Specifies Parquet format.""" 

63 

64 BIGTABLE = "BIGTABLE" 

65 """Specifies Bigtable format.""" 

66 

67 

68class BigtableColumn(object): 

69 """Options for a Bigtable column.""" 

70 

71 def __init__(self): 

72 self._properties = {} 

73 

74 @property 

75 def encoding(self): 

76 """str: The encoding of the values when the type is not `STRING` 

77 

78 See 

79 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.encoding 

80 """ 

81 return self._properties.get("encoding") 

82 

83 @encoding.setter 

84 def encoding(self, value): 

85 self._properties["encoding"] = value 

86 

87 @property 

88 def field_name(self): 

89 """str: An identifier to use if the qualifier is not a valid BigQuery 

90 field identifier 

91 

92 See 

93 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.field_name 

94 """ 

95 return self._properties.get("fieldName") 

96 

97 @field_name.setter 

98 def field_name(self, value): 

99 self._properties["fieldName"] = value 

100 

101 @property 

102 def only_read_latest(self): 

103 """bool: If this is set, only the latest version of value in this 

104 column are exposed. 

105 

106 See 

107 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.only_read_latest 

108 """ 

109 return self._properties.get("onlyReadLatest") 

110 

111 @only_read_latest.setter 

112 def only_read_latest(self, value): 

113 self._properties["onlyReadLatest"] = value 

114 

115 @property 

116 def qualifier_encoded(self): 

117 """Union[str, bytes]: The qualifier encoded in binary. 

118 

119 The type is ``str`` (Python 2.x) or ``bytes`` (Python 3.x). The module 

120 will handle base64 encoding for you. 

121 

122 See 

123 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.qualifier_encoded 

124 """ 

125 prop = self._properties.get("qualifierEncoded") 

126 if prop is None: 

127 return None 

128 return base64.standard_b64decode(_to_bytes(prop)) 

129 

130 @qualifier_encoded.setter 

131 def qualifier_encoded(self, value): 

132 self._properties["qualifierEncoded"] = _bytes_to_json(value) 

133 

134 @property 

135 def qualifier_string(self): 

136 """str: A valid UTF-8 string qualifier 

137 

138 See 

139 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.qualifier_string 

140 """ 

141 return self._properties.get("qualifierString") 

142 

143 @qualifier_string.setter 

144 def qualifier_string(self, value): 

145 self._properties["qualifierString"] = value 

146 

147 @property 

148 def type_(self): 

149 """str: The type to convert the value in cells of this column. 

150 

151 See 

152 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.type 

153 """ 

154 return self._properties.get("type") 

155 

156 @type_.setter 

157 def type_(self, value): 

158 self._properties["type"] = value 

159 

160 def to_api_repr(self) -> dict: 

161 """Build an API representation of this object. 

162 

163 Returns: 

164 Dict[str, Any]: 

165 A dictionary in the format used by the BigQuery API. 

166 """ 

167 return copy.deepcopy(self._properties) 

168 

169 @classmethod 

170 def from_api_repr(cls, resource: dict) -> "BigtableColumn": 

171 """Factory: construct a :class:`~.external_config.BigtableColumn` 

172 instance given its API representation. 

173 

174 Args: 

175 resource (Dict[str, Any]): 

176 Definition of a :class:`~.external_config.BigtableColumn` 

177 instance in the same representation as is returned from the 

178 API. 

179 

180 Returns: 

181 external_config.BigtableColumn: Configuration parsed from ``resource``. 

182 """ 

183 config = cls() 

184 config._properties = copy.deepcopy(resource) 

185 return config 

186 

187 

188class BigtableColumnFamily(object): 

189 """Options for a Bigtable column family.""" 

190 

191 def __init__(self): 

192 self._properties = {} 

193 

194 @property 

195 def encoding(self): 

196 """str: The encoding of the values when the type is not `STRING` 

197 

198 See 

199 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.encoding 

200 """ 

201 return self._properties.get("encoding") 

202 

203 @encoding.setter 

204 def encoding(self, value): 

205 self._properties["encoding"] = value 

206 

207 @property 

208 def family_id(self): 

209 """str: Identifier of the column family. 

210 

211 See 

212 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.family_id 

213 """ 

214 return self._properties.get("familyId") 

215 

216 @family_id.setter 

217 def family_id(self, value): 

218 self._properties["familyId"] = value 

219 

220 @property 

221 def only_read_latest(self): 

222 """bool: If this is set only the latest version of value are exposed 

223 for all columns in this column family. 

224 

225 See 

226 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.only_read_latest 

227 """ 

228 return self._properties.get("onlyReadLatest") 

229 

230 @only_read_latest.setter 

231 def only_read_latest(self, value): 

232 self._properties["onlyReadLatest"] = value 

233 

234 @property 

235 def type_(self): 

236 """str: The type to convert the value in cells of this column family. 

237 

238 See 

239 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.type 

240 """ 

241 return self._properties.get("type") 

242 

243 @type_.setter 

244 def type_(self, value): 

245 self._properties["type"] = value 

246 

247 @property 

248 def columns(self): 

249 """List[BigtableColumn]: Lists of columns 

250 that should be exposed as individual fields. 

251 

252 See 

253 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.columns 

254 """ 

255 prop = self._properties.get("columns", []) 

256 return [BigtableColumn.from_api_repr(col) for col in prop] 

257 

258 @columns.setter 

259 def columns(self, value): 

260 self._properties["columns"] = [col.to_api_repr() for col in value] 

261 

262 def to_api_repr(self) -> dict: 

263 """Build an API representation of this object. 

264 

265 Returns: 

266 Dict[str, Any]: 

267 A dictionary in the format used by the BigQuery API. 

268 """ 

269 return copy.deepcopy(self._properties) 

270 

271 @classmethod 

272 def from_api_repr(cls, resource: dict) -> "BigtableColumnFamily": 

273 """Factory: construct a :class:`~.external_config.BigtableColumnFamily` 

274 instance given its API representation. 

275 

276 Args: 

277 resource (Dict[str, Any]): 

278 Definition of a :class:`~.external_config.BigtableColumnFamily` 

279 instance in the same representation as is returned from the 

280 API. 

281 

282 Returns: 

283 :class:`~.external_config.BigtableColumnFamily`: 

284 Configuration parsed from ``resource``. 

285 """ 

286 config = cls() 

287 config._properties = copy.deepcopy(resource) 

288 return config 

289 

290 

291class BigtableOptions(object): 

292 """Options that describe how to treat Bigtable tables as BigQuery tables.""" 

293 

294 _SOURCE_FORMAT = "BIGTABLE" 

295 _RESOURCE_NAME = "bigtableOptions" 

296 

297 def __init__(self): 

298 self._properties = {} 

299 

300 @property 

301 def ignore_unspecified_column_families(self): 

302 """bool: If :data:`True`, ignore columns not specified in 

303 :attr:`column_families` list. Defaults to :data:`False`. 

304 

305 See 

306 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.ignore_unspecified_column_families 

307 """ 

308 return self._properties.get("ignoreUnspecifiedColumnFamilies") 

309 

310 @ignore_unspecified_column_families.setter 

311 def ignore_unspecified_column_families(self, value): 

312 self._properties["ignoreUnspecifiedColumnFamilies"] = value 

313 

314 @property 

315 def read_rowkey_as_string(self): 

316 """bool: If :data:`True`, rowkey column families will be read and 

317 converted to string. Defaults to :data:`False`. 

318 

319 See 

320 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.read_rowkey_as_string 

321 """ 

322 return self._properties.get("readRowkeyAsString") 

323 

324 @read_rowkey_as_string.setter 

325 def read_rowkey_as_string(self, value): 

326 self._properties["readRowkeyAsString"] = value 

327 

328 @property 

329 def column_families(self): 

330 """List[:class:`~.external_config.BigtableColumnFamily`]: List of 

331 column families to expose in the table schema along with their types. 

332 

333 See 

334 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.column_families 

335 """ 

336 prop = self._properties.get("columnFamilies", []) 

337 return [BigtableColumnFamily.from_api_repr(cf) for cf in prop] 

338 

339 @column_families.setter 

340 def column_families(self, value): 

341 self._properties["columnFamilies"] = [cf.to_api_repr() for cf in value] 

342 

343 def to_api_repr(self) -> dict: 

344 """Build an API representation of this object. 

345 

346 Returns: 

347 Dict[str, Any]: 

348 A dictionary in the format used by the BigQuery API. 

349 """ 

350 return copy.deepcopy(self._properties) 

351 

352 @classmethod 

353 def from_api_repr(cls, resource: dict) -> "BigtableOptions": 

354 """Factory: construct a :class:`~.external_config.BigtableOptions` 

355 instance given its API representation. 

356 

357 Args: 

358 resource (Dict[str, Any]): 

359 Definition of a :class:`~.external_config.BigtableOptions` 

360 instance in the same representation as is returned from the 

361 API. 

362 

363 Returns: 

364 BigtableOptions: Configuration parsed from ``resource``. 

365 """ 

366 config = cls() 

367 config._properties = copy.deepcopy(resource) 

368 return config 

369 

370 

371class CSVOptions(object): 

372 """Options that describe how to treat CSV files as BigQuery tables.""" 

373 

374 _SOURCE_FORMAT = "CSV" 

375 _RESOURCE_NAME = "csvOptions" 

376 

377 def __init__(self): 

378 self._properties = {} 

379 

380 @property 

381 def allow_jagged_rows(self): 

382 """bool: If :data:`True`, BigQuery treats missing trailing columns as 

383 null values. Defaults to :data:`False`. 

384 

385 See 

386 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.allow_jagged_rows 

387 """ 

388 return self._properties.get("allowJaggedRows") 

389 

390 @allow_jagged_rows.setter 

391 def allow_jagged_rows(self, value): 

392 self._properties["allowJaggedRows"] = value 

393 

394 @property 

395 def allow_quoted_newlines(self): 

396 """bool: If :data:`True`, quoted data sections that contain newline 

397 characters in a CSV file are allowed. Defaults to :data:`False`. 

398 

399 See 

400 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.allow_quoted_newlines 

401 """ 

402 return self._properties.get("allowQuotedNewlines") 

403 

404 @allow_quoted_newlines.setter 

405 def allow_quoted_newlines(self, value): 

406 self._properties["allowQuotedNewlines"] = value 

407 

408 @property 

409 def encoding(self): 

410 """str: The character encoding of the data. 

411 

412 See 

413 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.encoding 

414 """ 

415 return self._properties.get("encoding") 

416 

417 @encoding.setter 

418 def encoding(self, value): 

419 self._properties["encoding"] = value 

420 

421 @property 

422 def preserve_ascii_control_characters(self): 

423 """bool: Indicates if the embedded ASCII control characters 

424 (the first 32 characters in the ASCII-table, from '\x00' to '\x1F') are preserved. 

425 

426 See 

427 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.preserve_ascii_control_characters 

428 """ 

429 return self._properties.get("preserveAsciiControlCharacters") 

430 

431 @preserve_ascii_control_characters.setter 

432 def preserve_ascii_control_characters(self, value): 

433 self._properties["preserveAsciiControlCharacters"] = value 

434 

435 @property 

436 def field_delimiter(self): 

437 """str: The separator for fields in a CSV file. Defaults to comma (','). 

438 

439 See 

440 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.field_delimiter 

441 """ 

442 return self._properties.get("fieldDelimiter") 

443 

444 @field_delimiter.setter 

445 def field_delimiter(self, value): 

446 self._properties["fieldDelimiter"] = value 

447 

448 @property 

449 def quote_character(self): 

450 """str: The value that is used to quote data sections in a CSV file. 

451 

452 See 

453 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.quote 

454 """ 

455 return self._properties.get("quote") 

456 

457 @quote_character.setter 

458 def quote_character(self, value): 

459 self._properties["quote"] = value 

460 

461 @property 

462 def skip_leading_rows(self): 

463 """int: The number of rows at the top of a CSV file. 

464 

465 See 

466 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.skip_leading_rows 

467 """ 

468 return _int_or_none(self._properties.get("skipLeadingRows")) 

469 

470 @skip_leading_rows.setter 

471 def skip_leading_rows(self, value): 

472 self._properties["skipLeadingRows"] = str(value) 

473 

474 def to_api_repr(self) -> dict: 

475 """Build an API representation of this object. 

476 

477 Returns: 

478 Dict[str, Any]: A dictionary in the format used by the BigQuery API. 

479 """ 

480 return copy.deepcopy(self._properties) 

481 

482 @classmethod 

483 def from_api_repr(cls, resource: dict) -> "CSVOptions": 

484 """Factory: construct a :class:`~.external_config.CSVOptions` instance 

485 given its API representation. 

486 

487 Args: 

488 resource (Dict[str, Any]): 

489 Definition of a :class:`~.external_config.CSVOptions` 

490 instance in the same representation as is returned from the 

491 API. 

492 

493 Returns: 

494 CSVOptions: Configuration parsed from ``resource``. 

495 """ 

496 config = cls() 

497 config._properties = copy.deepcopy(resource) 

498 return config 

499 

500 

501class GoogleSheetsOptions(object): 

502 """Options that describe how to treat Google Sheets as BigQuery tables.""" 

503 

504 _SOURCE_FORMAT = "GOOGLE_SHEETS" 

505 _RESOURCE_NAME = "googleSheetsOptions" 

506 

507 def __init__(self): 

508 self._properties = {} 

509 

510 @property 

511 def skip_leading_rows(self): 

512 """int: The number of rows at the top of a sheet that BigQuery will 

513 skip when reading the data. 

514 

515 See 

516 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#GoogleSheetsOptions.FIELDS.skip_leading_rows 

517 """ 

518 return _int_or_none(self._properties.get("skipLeadingRows")) 

519 

520 @skip_leading_rows.setter 

521 def skip_leading_rows(self, value): 

522 self._properties["skipLeadingRows"] = str(value) 

523 

524 @property 

525 def range(self): 

526 """str: The range of a sheet that BigQuery will query from. 

527 

528 See 

529 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#GoogleSheetsOptions.FIELDS.range 

530 """ 

531 return _str_or_none(self._properties.get("range")) 

532 

533 @range.setter 

534 def range(self, value): 

535 self._properties["range"] = value 

536 

537 def to_api_repr(self) -> dict: 

538 """Build an API representation of this object. 

539 

540 Returns: 

541 Dict[str, Any]: A dictionary in the format used by the BigQuery API. 

542 """ 

543 return copy.deepcopy(self._properties) 

544 

545 @classmethod 

546 def from_api_repr(cls, resource: dict) -> "GoogleSheetsOptions": 

547 """Factory: construct a :class:`~.external_config.GoogleSheetsOptions` 

548 instance given its API representation. 

549 

550 Args: 

551 resource (Dict[str, Any]): 

552 Definition of a :class:`~.external_config.GoogleSheetsOptions` 

553 instance in the same representation as is returned from the 

554 API. 

555 

556 Returns: 

557 GoogleSheetsOptions: Configuration parsed from ``resource``. 

558 """ 

559 config = cls() 

560 config._properties = copy.deepcopy(resource) 

561 return config 

562 

563 

564_OPTION_CLASSES = ( 

565 AvroOptions, 

566 BigtableOptions, 

567 CSVOptions, 

568 GoogleSheetsOptions, 

569 ParquetOptions, 

570) 

571 

572OptionsType = Union[ 

573 AvroOptions, 

574 BigtableOptions, 

575 CSVOptions, 

576 GoogleSheetsOptions, 

577 ParquetOptions, 

578] 

579 

580 

581class HivePartitioningOptions(object): 

582 """[Beta] Options that configure hive partitioning. 

583 

584 .. note:: 

585 **Experimental**. This feature is experimental and might change or 

586 have limited support. 

587 

588 See 

589 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions 

590 """ 

591 

592 def __init__(self) -> None: 

593 self._properties: Dict[str, Any] = {} 

594 

595 @property 

596 def mode(self): 

597 """Optional[str]: When set, what mode of hive partitioning to use when reading data. 

598 

599 Two modes are supported: "AUTO" and "STRINGS". 

600 

601 See 

602 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.mode 

603 """ 

604 return self._properties.get("mode") 

605 

606 @mode.setter 

607 def mode(self, value): 

608 self._properties["mode"] = value 

609 

610 @property 

611 def source_uri_prefix(self): 

612 """Optional[str]: When hive partition detection is requested, a common prefix for 

613 all source URIs is required. 

614 

615 See 

616 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.source_uri_prefix 

617 """ 

618 return self._properties.get("sourceUriPrefix") 

619 

620 @source_uri_prefix.setter 

621 def source_uri_prefix(self, value): 

622 self._properties["sourceUriPrefix"] = value 

623 

624 @property 

625 def require_partition_filter(self): 

626 """Optional[bool]: If set to true, queries over the partitioned table require a 

627 partition filter that can be used for partition elimination to be 

628 specified. 

629 

630 See 

631 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.mode 

632 """ 

633 return self._properties.get("requirePartitionFilter") 

634 

635 @require_partition_filter.setter 

636 def require_partition_filter(self, value): 

637 self._properties["requirePartitionFilter"] = value 

638 

639 def to_api_repr(self) -> dict: 

640 """Build an API representation of this object. 

641 

642 Returns: 

643 Dict[str, Any]: A dictionary in the format used by the BigQuery API. 

644 """ 

645 return copy.deepcopy(self._properties) 

646 

647 @classmethod 

648 def from_api_repr(cls, resource: dict) -> "HivePartitioningOptions": 

649 """Factory: construct a :class:`~.external_config.HivePartitioningOptions` 

650 instance given its API representation. 

651 

652 Args: 

653 resource (Dict[str, Any]): 

654 Definition of a :class:`~.external_config.HivePartitioningOptions` 

655 instance in the same representation as is returned from the 

656 API. 

657 

658 Returns: 

659 HivePartitioningOptions: Configuration parsed from ``resource``. 

660 """ 

661 config = cls() 

662 config._properties = copy.deepcopy(resource) 

663 return config 

664 

665 

666class ExternalConfig(object): 

667 """Description of an external data source. 

668 

669 Args: 

670 source_format (ExternalSourceFormat): 

671 See :attr:`source_format`. 

672 """ 

673 

674 def __init__(self, source_format) -> None: 

675 self._properties = {"sourceFormat": source_format} 

676 

677 @property 

678 def source_format(self): 

679 """:class:`~.external_config.ExternalSourceFormat`: 

680 Format of external source. 

681 

682 See 

683 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.source_format 

684 """ 

685 return self._properties["sourceFormat"] 

686 

687 @property 

688 def options(self) -> Optional[OptionsType]: 

689 """Source-specific options.""" 

690 for optcls in _OPTION_CLASSES: 

691 # The code below is too much magic for mypy to handle. 

692 if self.source_format == optcls._SOURCE_FORMAT: # type: ignore 

693 options: OptionsType = optcls() # type: ignore 

694 options._properties = self._properties.setdefault( 

695 optcls._RESOURCE_NAME, {} # type: ignore 

696 ) 

697 return options 

698 

699 # No matching source format found. 

700 return None 

701 

702 @property 

703 def autodetect(self): 

704 """bool: If :data:`True`, try to detect schema and format options 

705 automatically. 

706 

707 See 

708 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.autodetect 

709 """ 

710 return self._properties.get("autodetect") 

711 

712 @autodetect.setter 

713 def autodetect(self, value): 

714 self._properties["autodetect"] = value 

715 

716 @property 

717 def compression(self): 

718 """str: The compression type of the data source. 

719 

720 See 

721 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.compression 

722 """ 

723 return self._properties.get("compression") 

724 

725 @compression.setter 

726 def compression(self, value): 

727 self._properties["compression"] = value 

728 

729 @property 

730 def decimal_target_types(self) -> Optional[FrozenSet[str]]: 

731 """Possible SQL data types to which the source decimal values are converted. 

732 

733 See: 

734 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.decimal_target_types 

735 

736 .. versionadded:: 2.21.0 

737 """ 

738 prop = self._properties.get("decimalTargetTypes") 

739 if prop is not None: 

740 prop = frozenset(prop) 

741 return prop 

742 

743 @decimal_target_types.setter 

744 def decimal_target_types(self, value: Optional[Iterable[str]]): 

745 if value is not None: 

746 self._properties["decimalTargetTypes"] = list(value) 

747 else: 

748 if "decimalTargetTypes" in self._properties: 

749 del self._properties["decimalTargetTypes"] 

750 

751 @property 

752 def hive_partitioning(self): 

753 """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \ 

754 it configures hive partitioning support. 

755 

756 .. note:: 

757 **Experimental**. This feature is experimental and might change or 

758 have limited support. 

759 

760 See 

761 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.hive_partitioning_options 

762 """ 

763 prop = self._properties.get("hivePartitioningOptions") 

764 if prop is None: 

765 return None 

766 return HivePartitioningOptions.from_api_repr(prop) 

767 

768 @hive_partitioning.setter 

769 def hive_partitioning(self, value): 

770 prop = value.to_api_repr() if value is not None else None 

771 self._properties["hivePartitioningOptions"] = prop 

772 

773 @property 

774 def reference_file_schema_uri(self): 

775 """Optional[str]: 

776 When creating an external table, the user can provide a reference file with the 

777 table schema. This is enabled for the following formats: 

778 

779 AVRO, PARQUET, ORC 

780 """ 

781 return self._properties.get("referenceFileSchemaUri") 

782 

783 @reference_file_schema_uri.setter 

784 def reference_file_schema_uri(self, value): 

785 self._properties["referenceFileSchemaUri"] = value 

786 

787 @property 

788 def ignore_unknown_values(self): 

789 """bool: If :data:`True`, extra values that are not represented in the 

790 table schema are ignored. Defaults to :data:`False`. 

791 

792 See 

793 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.ignore_unknown_values 

794 """ 

795 return self._properties.get("ignoreUnknownValues") 

796 

797 @ignore_unknown_values.setter 

798 def ignore_unknown_values(self, value): 

799 self._properties["ignoreUnknownValues"] = value 

800 

801 @property 

802 def max_bad_records(self): 

803 """int: The maximum number of bad records that BigQuery can ignore when 

804 reading data. 

805 

806 See 

807 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.max_bad_records 

808 """ 

809 return self._properties.get("maxBadRecords") 

810 

811 @max_bad_records.setter 

812 def max_bad_records(self, value): 

813 self._properties["maxBadRecords"] = value 

814 

815 @property 

816 def source_uris(self): 

817 """List[str]: URIs that point to your data in Google Cloud. 

818 

819 See 

820 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.source_uris 

821 """ 

822 return self._properties.get("sourceUris", []) 

823 

824 @source_uris.setter 

825 def source_uris(self, value): 

826 self._properties["sourceUris"] = value 

827 

828 @property 

829 def schema(self): 

830 """List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema 

831 for the data. 

832 

833 See 

834 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.schema 

835 """ 

836 prop = self._properties.get("schema", {}) 

837 return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])] 

838 

839 @schema.setter 

840 def schema(self, value): 

841 prop = value 

842 if value is not None: 

843 prop = {"fields": [field.to_api_repr() for field in value]} 

844 self._properties["schema"] = prop 

845 

846 @property 

847 def connection_id(self): 

848 """Optional[str]: [Experimental] ID of a BigQuery Connection API 

849 resource. 

850 

851 .. WARNING:: 

852 

853 This feature is experimental. Pre-GA features may have limited 

854 support, and changes to pre-GA features may not be compatible with 

855 other pre-GA versions. 

856 """ 

857 return self._properties.get("connectionId") 

858 

859 @connection_id.setter 

860 def connection_id(self, value): 

861 self._properties["connectionId"] = value 

862 

863 @property 

864 def avro_options(self) -> Optional[AvroOptions]: 

865 """Additional properties to set if ``sourceFormat`` is set to AVRO. 

866 

867 See: 

868 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.avro_options 

869 """ 

870 if self.source_format == ExternalSourceFormat.AVRO: 

871 self._properties.setdefault(AvroOptions._RESOURCE_NAME, {}) 

872 resource = self._properties.get(AvroOptions._RESOURCE_NAME) 

873 if resource is None: 

874 return None 

875 options = AvroOptions() 

876 options._properties = resource 

877 return options 

878 

879 @avro_options.setter 

880 def avro_options(self, value): 

881 if self.source_format != ExternalSourceFormat.AVRO: 

882 msg = f"Cannot set Avro options, source format is {self.source_format}" 

883 raise TypeError(msg) 

884 self._properties[AvroOptions._RESOURCE_NAME] = value._properties 

885 

886 @property 

887 def bigtable_options(self) -> Optional[BigtableOptions]: 

888 """Additional properties to set if ``sourceFormat`` is set to BIGTABLE. 

889 

890 See: 

891 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.bigtable_options 

892 """ 

893 if self.source_format == ExternalSourceFormat.BIGTABLE: 

894 self._properties.setdefault(BigtableOptions._RESOURCE_NAME, {}) 

895 resource = self._properties.get(BigtableOptions._RESOURCE_NAME) 

896 if resource is None: 

897 return None 

898 options = BigtableOptions() 

899 options._properties = resource 

900 return options 

901 

902 @bigtable_options.setter 

903 def bigtable_options(self, value): 

904 if self.source_format != ExternalSourceFormat.BIGTABLE: 

905 msg = f"Cannot set Bigtable options, source format is {self.source_format}" 

906 raise TypeError(msg) 

907 self._properties[BigtableOptions._RESOURCE_NAME] = value._properties 

908 

909 @property 

910 def csv_options(self) -> Optional[CSVOptions]: 

911 """Additional properties to set if ``sourceFormat`` is set to CSV. 

912 

913 See: 

914 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.csv_options 

915 """ 

916 if self.source_format == ExternalSourceFormat.CSV: 

917 self._properties.setdefault(CSVOptions._RESOURCE_NAME, {}) 

918 resource = self._properties.get(CSVOptions._RESOURCE_NAME) 

919 if resource is None: 

920 return None 

921 options = CSVOptions() 

922 options._properties = resource 

923 return options 

924 

925 @csv_options.setter 

926 def csv_options(self, value): 

927 if self.source_format != ExternalSourceFormat.CSV: 

928 msg = f"Cannot set CSV options, source format is {self.source_format}" 

929 raise TypeError(msg) 

930 self._properties[CSVOptions._RESOURCE_NAME] = value._properties 

931 

932 @property 

933 def google_sheets_options(self) -> Optional[GoogleSheetsOptions]: 

934 """Additional properties to set if ``sourceFormat`` is set to 

935 GOOGLE_SHEETS. 

936 

937 See: 

938 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.google_sheets_options 

939 """ 

940 if self.source_format == ExternalSourceFormat.GOOGLE_SHEETS: 

941 self._properties.setdefault(GoogleSheetsOptions._RESOURCE_NAME, {}) 

942 resource = self._properties.get(GoogleSheetsOptions._RESOURCE_NAME) 

943 if resource is None: 

944 return None 

945 options = GoogleSheetsOptions() 

946 options._properties = resource 

947 return options 

948 

949 @google_sheets_options.setter 

950 def google_sheets_options(self, value): 

951 if self.source_format != ExternalSourceFormat.GOOGLE_SHEETS: 

952 msg = f"Cannot set Google Sheets options, source format is {self.source_format}" 

953 raise TypeError(msg) 

954 self._properties[GoogleSheetsOptions._RESOURCE_NAME] = value._properties 

955 

956 @property 

957 def parquet_options(self) -> Optional[ParquetOptions]: 

958 """Additional properties to set if ``sourceFormat`` is set to PARQUET. 

959 

960 See: 

961 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.parquet_options 

962 """ 

963 if self.source_format == ExternalSourceFormat.PARQUET: 

964 self._properties.setdefault(ParquetOptions._RESOURCE_NAME, {}) 

965 resource = self._properties.get(ParquetOptions._RESOURCE_NAME) 

966 if resource is None: 

967 return None 

968 options = ParquetOptions() 

969 options._properties = resource 

970 return options 

971 

972 @parquet_options.setter 

973 def parquet_options(self, value): 

974 if self.source_format != ExternalSourceFormat.PARQUET: 

975 msg = f"Cannot set Parquet options, source format is {self.source_format}" 

976 raise TypeError(msg) 

977 self._properties[ParquetOptions._RESOURCE_NAME] = value._properties 

978 

979 def to_api_repr(self) -> dict: