Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2017 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Define classes that describe external data sources.
17 These are used for both Table.externalDataConfiguration and
18 Job.configuration.query.tableDefinitions.
19"""
21from __future__ import absolute_import, annotations
23import base64
24import copy
25import typing
26from typing import Any, Dict, FrozenSet, Iterable, Optional, Union
28from google.cloud.bigquery._helpers import _to_bytes
29from google.cloud.bigquery._helpers import _bytes_to_json
30from google.cloud.bigquery._helpers import _int_or_none
31from google.cloud.bigquery._helpers import _str_or_none
32from google.cloud.bigquery import _helpers
33from google.cloud.bigquery.format_options import AvroOptions, ParquetOptions
34from google.cloud.bigquery import schema
35from google.cloud.bigquery.schema import SchemaField
38class ExternalSourceFormat(object):
39 """The format for external data files.
41 Note that the set of allowed values for external data sources is different
42 than the set used for loading data (see
43 :class:`~google.cloud.bigquery.job.SourceFormat`).
44 """
46 CSV = "CSV"
47 """Specifies CSV format."""
49 GOOGLE_SHEETS = "GOOGLE_SHEETS"
50 """Specifies Google Sheets format."""
52 NEWLINE_DELIMITED_JSON = "NEWLINE_DELIMITED_JSON"
53 """Specifies newline delimited JSON format."""
55 AVRO = "AVRO"
56 """Specifies Avro format."""
58 DATASTORE_BACKUP = "DATASTORE_BACKUP"
59 """Specifies datastore backup format"""
61 ORC = "ORC"
62 """Specifies ORC format."""
64 PARQUET = "PARQUET"
65 """Specifies Parquet format."""
67 BIGTABLE = "BIGTABLE"
68 """Specifies Bigtable format."""
71class BigtableColumn(object):
72 """Options for a Bigtable column."""
74 def __init__(self):
75 self._properties = {}
77 @property
78 def encoding(self):
79 """str: The encoding of the values when the type is not `STRING`
81 See
82 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.encoding
83 """
84 return self._properties.get("encoding")
86 @encoding.setter
87 def encoding(self, value):
88 self._properties["encoding"] = value
90 @property
91 def field_name(self):
92 """str: An identifier to use if the qualifier is not a valid BigQuery
93 field identifier
95 See
96 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.field_name
97 """
98 return self._properties.get("fieldName")
100 @field_name.setter
101 def field_name(self, value):
102 self._properties["fieldName"] = value
104 @property
105 def only_read_latest(self):
106 """bool: If this is set, only the latest version of value in this
107 column are exposed.
109 See
110 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.only_read_latest
111 """
112 return self._properties.get("onlyReadLatest")
114 @only_read_latest.setter
115 def only_read_latest(self, value):
116 self._properties["onlyReadLatest"] = value
118 @property
119 def qualifier_encoded(self):
120 """Union[str, bytes]: The qualifier encoded in binary.
122 The type is ``str`` (Python 2.x) or ``bytes`` (Python 3.x). The module
123 will handle base64 encoding for you.
125 See
126 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.qualifier_encoded
127 """
128 prop = self._properties.get("qualifierEncoded")
129 if prop is None:
130 return None
131 return base64.standard_b64decode(_to_bytes(prop))
133 @qualifier_encoded.setter
134 def qualifier_encoded(self, value):
135 self._properties["qualifierEncoded"] = _bytes_to_json(value)
137 @property
138 def qualifier_string(self):
139 """str: A valid UTF-8 string qualifier
141 See
142 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.qualifier_string
143 """
144 return self._properties.get("qualifierString")
146 @qualifier_string.setter
147 def qualifier_string(self, value):
148 self._properties["qualifierString"] = value
150 @property
151 def type_(self):
152 """str: The type to convert the value in cells of this column.
154 See
155 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumn.FIELDS.type
156 """
157 return self._properties.get("type")
159 @type_.setter
160 def type_(self, value):
161 self._properties["type"] = value
163 def to_api_repr(self) -> dict:
164 """Build an API representation of this object.
166 Returns:
167 Dict[str, Any]:
168 A dictionary in the format used by the BigQuery API.
169 """
170 return copy.deepcopy(self._properties)
172 @classmethod
173 def from_api_repr(cls, resource: dict) -> "BigtableColumn":
174 """Factory: construct a :class:`~.external_config.BigtableColumn`
175 instance given its API representation.
177 Args:
178 resource (Dict[str, Any]):
179 Definition of a :class:`~.external_config.BigtableColumn`
180 instance in the same representation as is returned from the
181 API.
183 Returns:
184 external_config.BigtableColumn: Configuration parsed from ``resource``.
185 """
186 config = cls()
187 config._properties = copy.deepcopy(resource)
188 return config
191class BigtableColumnFamily(object):
192 """Options for a Bigtable column family."""
194 def __init__(self):
195 self._properties = {}
197 @property
198 def encoding(self):
199 """str: The encoding of the values when the type is not `STRING`
201 See
202 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.encoding
203 """
204 return self._properties.get("encoding")
206 @encoding.setter
207 def encoding(self, value):
208 self._properties["encoding"] = value
210 @property
211 def family_id(self):
212 """str: Identifier of the column family.
214 See
215 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.family_id
216 """
217 return self._properties.get("familyId")
219 @family_id.setter
220 def family_id(self, value):
221 self._properties["familyId"] = value
223 @property
224 def only_read_latest(self):
225 """bool: If this is set only the latest version of value are exposed
226 for all columns in this column family.
228 See
229 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.only_read_latest
230 """
231 return self._properties.get("onlyReadLatest")
233 @only_read_latest.setter
234 def only_read_latest(self, value):
235 self._properties["onlyReadLatest"] = value
237 @property
238 def type_(self):
239 """str: The type to convert the value in cells of this column family.
241 See
242 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.type
243 """
244 return self._properties.get("type")
246 @type_.setter
247 def type_(self, value):
248 self._properties["type"] = value
250 @property
251 def columns(self):
252 """List[BigtableColumn]: Lists of columns
253 that should be exposed as individual fields.
255 See
256 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableColumnFamily.FIELDS.columns
257 """
258 prop = self._properties.get("columns", [])
259 return [BigtableColumn.from_api_repr(col) for col in prop]
261 @columns.setter
262 def columns(self, value):
263 self._properties["columns"] = [col.to_api_repr() for col in value]
265 def to_api_repr(self) -> dict:
266 """Build an API representation of this object.
268 Returns:
269 Dict[str, Any]:
270 A dictionary in the format used by the BigQuery API.
271 """
272 return copy.deepcopy(self._properties)
274 @classmethod
275 def from_api_repr(cls, resource: dict) -> "BigtableColumnFamily":
276 """Factory: construct a :class:`~.external_config.BigtableColumnFamily`
277 instance given its API representation.
279 Args:
280 resource (Dict[str, Any]):
281 Definition of a :class:`~.external_config.BigtableColumnFamily`
282 instance in the same representation as is returned from the
283 API.
285 Returns:
286 :class:`~.external_config.BigtableColumnFamily`:
287 Configuration parsed from ``resource``.
288 """
289 config = cls()
290 config._properties = copy.deepcopy(resource)
291 return config
294class BigtableOptions(object):
295 """Options that describe how to treat Bigtable tables as BigQuery tables."""
297 _SOURCE_FORMAT = "BIGTABLE"
298 _RESOURCE_NAME = "bigtableOptions"
300 def __init__(self):
301 self._properties = {}
303 @property
304 def ignore_unspecified_column_families(self):
305 """bool: If :data:`True`, ignore columns not specified in
306 :attr:`column_families` list. Defaults to :data:`False`.
308 See
309 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.ignore_unspecified_column_families
310 """
311 return self._properties.get("ignoreUnspecifiedColumnFamilies")
313 @ignore_unspecified_column_families.setter
314 def ignore_unspecified_column_families(self, value):
315 self._properties["ignoreUnspecifiedColumnFamilies"] = value
317 @property
318 def read_rowkey_as_string(self):
319 """bool: If :data:`True`, rowkey column families will be read and
320 converted to string. Defaults to :data:`False`.
322 See
323 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.read_rowkey_as_string
324 """
325 return self._properties.get("readRowkeyAsString")
327 @read_rowkey_as_string.setter
328 def read_rowkey_as_string(self, value):
329 self._properties["readRowkeyAsString"] = value
331 @property
332 def column_families(self):
333 """List[:class:`~.external_config.BigtableColumnFamily`]: List of
334 column families to expose in the table schema along with their types.
336 See
337 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#BigtableOptions.FIELDS.column_families
338 """
339 prop = self._properties.get("columnFamilies", [])
340 return [BigtableColumnFamily.from_api_repr(cf) for cf in prop]
342 @column_families.setter
343 def column_families(self, value):
344 self._properties["columnFamilies"] = [cf.to_api_repr() for cf in value]
346 def to_api_repr(self) -> dict:
347 """Build an API representation of this object.
349 Returns:
350 Dict[str, Any]:
351 A dictionary in the format used by the BigQuery API.
352 """
353 return copy.deepcopy(self._properties)
355 @classmethod
356 def from_api_repr(cls, resource: dict) -> "BigtableOptions":
357 """Factory: construct a :class:`~.external_config.BigtableOptions`
358 instance given its API representation.
360 Args:
361 resource (Dict[str, Any]):
362 Definition of a :class:`~.external_config.BigtableOptions`
363 instance in the same representation as is returned from the
364 API.
366 Returns:
367 BigtableOptions: Configuration parsed from ``resource``.
368 """
369 config = cls()
370 config._properties = copy.deepcopy(resource)
371 return config
374class CSVOptions(object):
375 """Options that describe how to treat CSV files as BigQuery tables."""
377 _SOURCE_FORMAT = "CSV"
378 _RESOURCE_NAME = "csvOptions"
380 def __init__(self):
381 self._properties = {}
383 @property
384 def allow_jagged_rows(self):
385 """bool: If :data:`True`, BigQuery treats missing trailing columns as
386 null values. Defaults to :data:`False`.
388 See
389 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.allow_jagged_rows
390 """
391 return self._properties.get("allowJaggedRows")
393 @allow_jagged_rows.setter
394 def allow_jagged_rows(self, value):
395 self._properties["allowJaggedRows"] = value
397 @property
398 def allow_quoted_newlines(self):
399 """bool: If :data:`True`, quoted data sections that contain newline
400 characters in a CSV file are allowed. Defaults to :data:`False`.
402 See
403 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.allow_quoted_newlines
404 """
405 return self._properties.get("allowQuotedNewlines")
407 @allow_quoted_newlines.setter
408 def allow_quoted_newlines(self, value):
409 self._properties["allowQuotedNewlines"] = value
411 @property
412 def encoding(self):
413 """str: The character encoding of the data.
415 See
416 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.encoding
417 """
418 return self._properties.get("encoding")
420 @encoding.setter
421 def encoding(self, value):
422 self._properties["encoding"] = value
424 @property
425 def preserve_ascii_control_characters(self):
426 """bool: Indicates if the embedded ASCII control characters
427 (the first 32 characters in the ASCII-table, from '\x00' to '\x1F') are preserved.
429 See
430 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.preserve_ascii_control_characters
431 """
432 return self._properties.get("preserveAsciiControlCharacters")
434 @preserve_ascii_control_characters.setter
435 def preserve_ascii_control_characters(self, value):
436 self._properties["preserveAsciiControlCharacters"] = value
438 @property
439 def field_delimiter(self):
440 """str: The separator for fields in a CSV file. Defaults to comma (',').
442 See
443 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.field_delimiter
444 """
445 return self._properties.get("fieldDelimiter")
447 @field_delimiter.setter
448 def field_delimiter(self, value):
449 self._properties["fieldDelimiter"] = value
451 @property
452 def quote_character(self):
453 """str: The value that is used to quote data sections in a CSV file.
455 See
456 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.quote
457 """
458 return self._properties.get("quote")
460 @quote_character.setter
461 def quote_character(self, value):
462 self._properties["quote"] = value
464 @property
465 def skip_leading_rows(self):
466 """int: The number of rows at the top of a CSV file.
468 See
469 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.skip_leading_rows
470 """
471 return _int_or_none(self._properties.get("skipLeadingRows"))
473 @skip_leading_rows.setter
474 def skip_leading_rows(self, value):
475 self._properties["skipLeadingRows"] = str(value)
477 def to_api_repr(self) -> dict:
478 """Build an API representation of this object.
480 Returns:
481 Dict[str, Any]: A dictionary in the format used by the BigQuery API.
482 """
483 return copy.deepcopy(self._properties)
485 @classmethod
486 def from_api_repr(cls, resource: dict) -> "CSVOptions":
487 """Factory: construct a :class:`~.external_config.CSVOptions` instance
488 given its API representation.
490 Args:
491 resource (Dict[str, Any]):
492 Definition of a :class:`~.external_config.CSVOptions`
493 instance in the same representation as is returned from the
494 API.
496 Returns:
497 CSVOptions: Configuration parsed from ``resource``.
498 """
499 config = cls()
500 config._properties = copy.deepcopy(resource)
501 return config
504class GoogleSheetsOptions(object):
505 """Options that describe how to treat Google Sheets as BigQuery tables."""
507 _SOURCE_FORMAT = "GOOGLE_SHEETS"
508 _RESOURCE_NAME = "googleSheetsOptions"
510 def __init__(self):
511 self._properties = {}
513 @property
514 def skip_leading_rows(self):
515 """int: The number of rows at the top of a sheet that BigQuery will
516 skip when reading the data.
518 See
519 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#GoogleSheetsOptions.FIELDS.skip_leading_rows
520 """
521 return _int_or_none(self._properties.get("skipLeadingRows"))
523 @skip_leading_rows.setter
524 def skip_leading_rows(self, value):
525 self._properties["skipLeadingRows"] = str(value)
527 @property
528 def range(self):
529 """str: The range of a sheet that BigQuery will query from.
531 See
532 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#GoogleSheetsOptions.FIELDS.range
533 """
534 return _str_or_none(self._properties.get("range"))
536 @range.setter
537 def range(self, value):
538 self._properties["range"] = value
540 def to_api_repr(self) -> dict:
541 """Build an API representation of this object.
543 Returns:
544 Dict[str, Any]: A dictionary in the format used by the BigQuery API.
545 """
546 return copy.deepcopy(self._properties)
548 @classmethod
549 def from_api_repr(cls, resource: dict) -> "GoogleSheetsOptions":
550 """Factory: construct a :class:`~.external_config.GoogleSheetsOptions`
551 instance given its API representation.
553 Args:
554 resource (Dict[str, Any]):
555 Definition of a :class:`~.external_config.GoogleSheetsOptions`
556 instance in the same representation as is returned from the
557 API.
559 Returns:
560 GoogleSheetsOptions: Configuration parsed from ``resource``.
561 """
562 config = cls()
563 config._properties = copy.deepcopy(resource)
564 return config
567_OPTION_CLASSES = (
568 AvroOptions,
569 BigtableOptions,
570 CSVOptions,
571 GoogleSheetsOptions,
572 ParquetOptions,
573)
575OptionsType = Union[
576 AvroOptions,
577 BigtableOptions,
578 CSVOptions,
579 GoogleSheetsOptions,
580 ParquetOptions,
581]
584class HivePartitioningOptions(object):
585 """[Beta] Options that configure hive partitioning.
587 .. note::
588 **Experimental**. This feature is experimental and might change or
589 have limited support.
591 See
592 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions
593 """
595 def __init__(self) -> None:
596 self._properties: Dict[str, Any] = {}
598 @property
599 def mode(self):
600 """Optional[str]: When set, what mode of hive partitioning to use when reading data.
602 Two modes are supported: "AUTO" and "STRINGS".
604 See
605 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.mode
606 """
607 return self._properties.get("mode")
609 @mode.setter
610 def mode(self, value):
611 self._properties["mode"] = value
613 @property
614 def source_uri_prefix(self):
615 """Optional[str]: When hive partition detection is requested, a common prefix for
616 all source URIs is required.
618 See
619 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.source_uri_prefix
620 """
621 return self._properties.get("sourceUriPrefix")
623 @source_uri_prefix.setter
624 def source_uri_prefix(self, value):
625 self._properties["sourceUriPrefix"] = value
627 @property
628 def require_partition_filter(self):
629 """Optional[bool]: If set to true, queries over the partitioned table require a
630 partition filter that can be used for partition elimination to be
631 specified.
633 See
634 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions.FIELDS.mode
635 """
636 return self._properties.get("requirePartitionFilter")
638 @require_partition_filter.setter
639 def require_partition_filter(self, value):
640 self._properties["requirePartitionFilter"] = value
642 def to_api_repr(self) -> dict:
643 """Build an API representation of this object.
645 Returns:
646 Dict[str, Any]: A dictionary in the format used by the BigQuery API.
647 """
648 return copy.deepcopy(self._properties)
650 @classmethod
651 def from_api_repr(cls, resource: dict) -> "HivePartitioningOptions":
652 """Factory: construct a :class:`~.external_config.HivePartitioningOptions`
653 instance given its API representation.
655 Args:
656 resource (Dict[str, Any]):
657 Definition of a :class:`~.external_config.HivePartitioningOptions`
658 instance in the same representation as is returned from the
659 API.
661 Returns:
662 HivePartitioningOptions: Configuration parsed from ``resource``.
663 """
664 config = cls()
665 config._properties = copy.deepcopy(resource)
666 return config
669class ExternalConfig(object):
670 """Description of an external data source.
672 Args:
673 source_format (ExternalSourceFormat):
674 See :attr:`source_format`.
675 """
677 def __init__(self, source_format) -> None:
678 self._properties = {"sourceFormat": source_format}
680 @property
681 def source_format(self):
682 """:class:`~.external_config.ExternalSourceFormat`:
683 Format of external source.
685 See
686 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.source_format
687 """
688 return self._properties["sourceFormat"]
690 @property
691 def options(self) -> Optional[OptionsType]:
692 """Source-specific options."""
693 for optcls in _OPTION_CLASSES:
694 # The code below is too much magic for mypy to handle.
695 if self.source_format == optcls._SOURCE_FORMAT: # type: ignore
696 options: OptionsType = optcls() # type: ignore
697 options._properties = self._properties.setdefault(
698 optcls._RESOURCE_NAME, {} # type: ignore
699 )
700 return options
702 # No matching source format found.
703 return None
705 @property
706 def autodetect(self):
707 """bool: If :data:`True`, try to detect schema and format options
708 automatically.
710 See
711 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.autodetect
712 """
713 return self._properties.get("autodetect")
715 @autodetect.setter
716 def autodetect(self, value):
717 self._properties["autodetect"] = value
719 @property
720 def compression(self):
721 """str: The compression type of the data source.
723 See
724 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.compression
725 """
726 return self._properties.get("compression")
728 @compression.setter
729 def compression(self, value):
730 self._properties["compression"] = value
732 @property
733 def decimal_target_types(self) -> Optional[FrozenSet[str]]:
734 """Possible SQL data types to which the source decimal values are converted.
736 See:
737 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.decimal_target_types
739 .. versionadded:: 2.21.0
740 """
741 prop = self._properties.get("decimalTargetTypes")
742 if prop is not None:
743 prop = frozenset(prop)
744 return prop
746 @decimal_target_types.setter
747 def decimal_target_types(self, value: Optional[Iterable[str]]):
748 if value is not None:
749 self._properties["decimalTargetTypes"] = list(value)
750 else:
751 if "decimalTargetTypes" in self._properties:
752 del self._properties["decimalTargetTypes"]
754 @property
755 def hive_partitioning(self):
756 """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \
757 it configures hive partitioning support.
759 .. note::
760 **Experimental**. This feature is experimental and might change or
761 have limited support.
763 See
764 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.hive_partitioning_options
765 """
766 prop = self._properties.get("hivePartitioningOptions")
767 if prop is None:
768 return None
769 return HivePartitioningOptions.from_api_repr(prop)
771 @hive_partitioning.setter
772 def hive_partitioning(self, value):
773 prop = value.to_api_repr() if value is not None else None
774 self._properties["hivePartitioningOptions"] = prop
776 @property
777 def reference_file_schema_uri(self):
778 """Optional[str]:
779 When creating an external table, the user can provide a reference file with the
780 table schema. This is enabled for the following formats:
782 AVRO, PARQUET, ORC
783 """
784 return self._properties.get("referenceFileSchemaUri")
786 @reference_file_schema_uri.setter
787 def reference_file_schema_uri(self, value):
788 self._properties["referenceFileSchemaUri"] = value
790 @property
791 def ignore_unknown_values(self):
792 """bool: If :data:`True`, extra values that are not represented in the
793 table schema are ignored. Defaults to :data:`False`.
795 See
796 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.ignore_unknown_values
797 """
798 return self._properties.get("ignoreUnknownValues")
800 @ignore_unknown_values.setter
801 def ignore_unknown_values(self, value):
802 self._properties["ignoreUnknownValues"] = value
804 @property
805 def max_bad_records(self):
806 """int: The maximum number of bad records that BigQuery can ignore when
807 reading data.
809 See
810 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.max_bad_records
811 """
812 return self._properties.get("maxBadRecords")
814 @max_bad_records.setter
815 def max_bad_records(self, value):
816 self._properties["maxBadRecords"] = value
818 @property
819 def source_uris(self):
820 """List[str]: URIs that point to your data in Google Cloud.
822 See
823 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.source_uris
824 """
825 return self._properties.get("sourceUris", [])
827 @source_uris.setter
828 def source_uris(self, value):
829 self._properties["sourceUris"] = value
831 @property
832 def schema(self):
833 """List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema
834 for the data.
836 See
837 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.schema
838 """
839 prop: Dict[str, Any] = typing.cast(
840 Dict[str, Any], self._properties.get("schema", {})
841 )
842 return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
844 @schema.setter
845 def schema(self, value):
846 prop = value
847 if value is not None:
848 prop = {"fields": [field.to_api_repr() for field in value]}
849 self._properties["schema"] = prop
851 @property
852 def time_zone(self) -> Optional[str]:
853 """Optional[str]: Time zone used when parsing timestamp values that do not
854 have specific time zone information (e.g. 2024-04-20 12:34:56). The expected
855 format is an IANA timezone string (e.g. America/Los_Angeles).
857 See:
858 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_zone
859 """
861 result = self._properties.get("timeZone")
862 return typing.cast(str, result)
864 @time_zone.setter
865 def time_zone(self, value: Optional[str]):
866 self._properties["timeZone"] = value
868 @property
869 def connection_id(self):
870 """Optional[str]: [Experimental] ID of a BigQuery Connection API
871 resource.
873 .. WARNING::
875 This feature is experimental. Pre-GA features may have limited
876 support, and changes to pre-GA features may not be compatible with
877 other pre-GA versions.
878 """
879 return self._properties.get("connectionId")
881 @connection_id.setter
882 def connection_id(self, value):
883 self._properties["connectionId"] = value
885 @property
886 def avro_options(self) -> Optional[AvroOptions]:
887 """Additional properties to set if ``sourceFormat`` is set to AVRO.
889 See:
890 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.avro_options
891 """
892 if self.source_format == ExternalSourceFormat.AVRO:
893 self._properties.setdefault(AvroOptions._RESOURCE_NAME, {})
894 resource = self._properties.get(AvroOptions._RESOURCE_NAME)
895 if resource is None:
896 return None
897 options = AvroOptions()
898 options._properties = resource
899 return options
901 @avro_options.setter
902 def avro_options(self, value):
903 if self.source_format != ExternalSourceFormat.AVRO:
904 msg = f"Cannot set Avro options, source format is {self.source_format}"
905 raise TypeError(msg)
906 self._properties[AvroOptions._RESOURCE_NAME] = value._properties
908 @property
909 def bigtable_options(self) -> Optional[BigtableOptions]:
910 """Additional properties to set if ``sourceFormat`` is set to BIGTABLE.
912 See:
913 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.bigtable_options
914 """
915 if self.source_format == ExternalSourceFormat.BIGTABLE:
916 self._properties.setdefault(BigtableOptions._RESOURCE_NAME, {})
917 resource = self._properties.get(BigtableOptions._RESOURCE_NAME)
918 if resource is None:
919 return None
920 options = BigtableOptions()
921 options._properties = resource
922 return options
924 @bigtable_options.setter
925 def bigtable_options(self, value):
926 if self.source_format != ExternalSourceFormat.BIGTABLE:
927 msg = f"Cannot set Bigtable options, source format is {self.source_format}"
928 raise TypeError(msg)
929 self._properties[BigtableOptions._RESOURCE_NAME] = value._properties
931 @property
932 def csv_options(self) -> Optional[CSVOptions]:
933 """Additional properties to set if ``sourceFormat`` is set to CSV.
935 See:
936 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.csv_options
937 """
938 if self.source_format == ExternalSourceFormat.CSV:
939 self._properties.setdefault(CSVOptions._RESOURCE_NAME, {})
940 resource = self._properties.get(CSVOptions._RESOURCE_NAME)
941 if resource is None:
942 return None
943 options = CSVOptions()
944 options._properties = resource
945 return options
947 @csv_options.setter
948 def csv_options(self, value):
949 if self.source_format != ExternalSourceFormat.CSV:
950 msg = f"Cannot set CSV options, source format is {self.source_format}"
951 raise TypeError(msg)
952 self._properties[CSVOptions._RESOURCE_NAME] = value._properties
954 @property
955 def google_sheets_options(self) -> Optional[GoogleSheetsOptions]:
956 """Additional properties to set if ``sourceFormat`` is set to
957 GOOGLE_SHEETS.
959 See:
960 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.google_sheets_options
961 """
962 if self.source_format == ExternalSourceFormat.GOOGLE_SHEETS:
963 self._properties.setdefault(GoogleSheetsOptions._RESOURCE_NAME, {})
964 resource = self._properties.get(GoogleSheetsOptions._RESOURCE_NAME)
965 if resource is None:
966 return None
967 options = GoogleSheetsOptions()
968 options._properties = resource
969 return options
971 @google_sheets_options.setter
972 def google_sheets_options(self, value):
973 if self.source_format != ExternalSourceFormat.GOOGLE_SHEETS:
974 msg = f"Cannot set Google Sheets options, source format is {self.source_format}"
975 raise TypeError(msg)
976 self._properties[GoogleSheetsOptions._RESOURCE_NAME] = value._properties
978 @property
979 def parquet_options(self) -> Optional[ParquetOptions]:
980 """Additional properties to set if ``sourceFormat`` is set to PARQUET.
982 See:
983 https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.parquet_options
984 """
985 if self.source_format == ExternalSourceFormat.PARQUET:
986 self._properties.setdefault(ParquetOptions._RESOURCE_NAME, {})
987 resource = self._properties.get(ParquetOptions._RESOURCE_NAME)
988 if resource is None:
989 return None
990 options = ParquetOptions()
991 options._properties = resource
992 return options
994 @parquet_options.setter
995 def parquet_options(self, value):
996 if self.source_format != ExternalSourceFormat.PARQUET:
997 msg = f"Cannot set Parquet options, source format is {self.source_format}"
998 raise TypeError(msg)
999 self._properties[ParquetOptions._RESOURCE_NAME] = value._properties
1001 def to_api_repr(self) -> dict:
1002 """Build an API representation of this object.
1004 Returns:
1005 Dict[str, Any]:
1006 A dictionary in the format used by the BigQuery API.
1007 """
1008 config = copy.deepcopy(self._properties)
1009 return config
1011 @classmethod
1012 def from_api_repr(cls, resource: dict) -> "ExternalConfig":
1013 """Factory: construct an :class:`~.external_config.ExternalConfig`
1014 instance given its API representation.
1016 Args:
1017 resource (Dict[str, Any]):
1018 Definition of an :class:`~.external_config.ExternalConfig`
1019 instance in the same representation as is returned from the
1020 API.
1022 Returns:
1023 ExternalConfig: Configuration parsed from ``resource``.
1024 """
1025 config = cls(resource["sourceFormat"])
1026 config._properties = copy.deepcopy(resource)
1027 return config
1030class ExternalCatalogDatasetOptions:
1031 """Options defining open source compatible datasets living in the BigQuery catalog.
1032 Contains metadata of open source database, schema or namespace represented
1033 by the current dataset.
1035 Args:
1036 default_storage_location_uri (Optional[str]): The storage location URI for all
1037 tables in the dataset. Equivalent to hive metastore's database
1038 locationUri. Maximum length of 1024 characters. (str)
1039 parameters (Optional[dict[str, Any]]): A map of key value pairs defining the parameters
1040 and properties of the open source schema. Maximum size of 2Mib.
1041 """
1043 def __init__(
1044 self,
1045 default_storage_location_uri: Optional[str] = None,
1046 parameters: Optional[Dict[str, Any]] = None,
1047 ):
1048 self._properties: Dict[str, Any] = {}
1049 self.default_storage_location_uri = default_storage_location_uri
1050 self.parameters = parameters
1052 @property
1053 def default_storage_location_uri(self) -> Optional[str]:
1054 """Optional. The storage location URI for all tables in the dataset.
1055 Equivalent to hive metastore's database locationUri. Maximum length of
1056 1024 characters."""
1058 return self._properties.get("defaultStorageLocationUri")
1060 @default_storage_location_uri.setter
1061 def default_storage_location_uri(self, value: Optional[str]):
1062 value = _helpers._isinstance_or_raise(value, str, none_allowed=True)
1063 self._properties["defaultStorageLocationUri"] = value
1065 @property
1066 def parameters(self) -> Optional[Dict[str, Any]]:
1067 """Optional. A map of key value pairs defining the parameters and
1068 properties of the open source schema. Maximum size of 2Mib."""
1070 return self._properties.get("parameters")
1072 @parameters.setter
1073 def parameters(self, value: Optional[Dict[str, Any]]):
1074 value = _helpers._isinstance_or_raise(value, dict, none_allowed=True)
1075 self._properties["parameters"] = value
1077 def to_api_repr(self) -> dict:
1078 """Build an API representation of this object.
1080 Returns:
1081 Dict[str, Any]:
1082 A dictionary in the format used by the BigQuery API.
1083 """
1084 return self._properties
1086 @classmethod
1087 def from_api_repr(cls, api_repr: dict) -> ExternalCatalogDatasetOptions:
1088 """Factory: constructs an instance of the class (cls)
1089 given its API representation.
1091 Args:
1092 api_repr (Dict[str, Any]):
1093 API representation of the object to be instantiated.
1095 Returns:
1096 An instance of the class initialized with data from 'resource'.
1097 """
1098 config = cls()
1099 config._properties = api_repr
1100 return config
1103class ExternalCatalogTableOptions:
1104 """Metadata about open source compatible table. The fields contained in these
1105 options correspond to hive metastore's table level properties.
1107 Args:
1108 connection_id (Optional[str]): The connection specifying the credentials to be
1109 used to read external storage, such as Azure Blob, Cloud Storage, or
1110 S3. The connection is needed to read the open source table from
1111 BigQuery Engine. The connection_id can have the form `..` or
1112 `projects//locations//connections/`.
1113 parameters (Union[Dict[str, Any], None]): A map of key value pairs defining the parameters
1114 and properties of the open source table. Corresponds with hive meta
1115 store table parameters. Maximum size of 4Mib.
1116 storage_descriptor (Optional[StorageDescriptor]): A storage descriptor containing information
1117 about the physical storage of this table.
1118 """
1120 def __init__(
1121 self,
1122 connection_id: Optional[str] = None,
1123 parameters: Union[Dict[str, Any], None] = None,
1124 storage_descriptor: Optional[schema.StorageDescriptor] = None,
1125 ):
1126 self._properties: Dict[str, Any] = {}
1127 self.connection_id = connection_id
1128 self.parameters = parameters
1129 self.storage_descriptor = storage_descriptor
1131 @property
1132 def connection_id(self) -> Optional[str]:
1133 """Optional. The connection specifying the credentials to be
1134 used to read external storage, such as Azure Blob, Cloud Storage, or
1135 S3. The connection is needed to read the open source table from
1136 BigQuery Engine. The connection_id can have the form `..` or
1137 `projects//locations//connections/`.
1138 """
1140 return self._properties.get("connectionId")
1142 @connection_id.setter
1143 def connection_id(self, value: Optional[str]):
1144 value = _helpers._isinstance_or_raise(value, str, none_allowed=True)
1145 self._properties["connectionId"] = value
1147 @property
1148 def parameters(self) -> Union[Dict[str, Any], None]:
1149 """Optional. A map of key value pairs defining the parameters and
1150 properties of the open source table. Corresponds with hive meta
1151 store table parameters. Maximum size of 4Mib.
1152 """
1154 return self._properties.get("parameters")
1156 @parameters.setter
1157 def parameters(self, value: Union[Dict[str, Any], None]):
1158 value = _helpers._isinstance_or_raise(value, dict, none_allowed=True)
1159 self._properties["parameters"] = value
1161 @property
1162 def storage_descriptor(self) -> Any:
1163 """Optional. A storage descriptor containing information about the
1164 physical storage of this table."""
1166 prop = _helpers._get_sub_prop(self._properties, ["storageDescriptor"])
1168 if prop is not None:
1169 return schema.StorageDescriptor.from_api_repr(prop)
1170 return None
1172 @storage_descriptor.setter
1173 def storage_descriptor(self, value: Union[schema.StorageDescriptor, dict, None]):
1174 value = _helpers._isinstance_or_raise(
1175 value, (schema.StorageDescriptor, dict), none_allowed=True
1176 )
1177 if isinstance(value, schema.StorageDescriptor):
1178 self._properties["storageDescriptor"] = value.to_api_repr()
1179 else:
1180 self._properties["storageDescriptor"] = value
1182 def to_api_repr(self) -> dict:
1183 """Build an API representation of this object.
1185 Returns:
1186 Dict[str, Any]:
1187 A dictionary in the format used by the BigQuery API.
1188 """
1190 return self._properties
1192 @classmethod
1193 def from_api_repr(cls, api_repr: dict) -> ExternalCatalogTableOptions:
1194 """Factory: constructs an instance of the class (cls)
1195 given its API representation.
1197 Args:
1198 api_repr (Dict[str, Any]):
1199 API representation of the object to be instantiated.
1201 Returns:
1202 An instance of the class initialized with data from 'api_repr'.
1203 """
1204 config = cls()
1205 config._properties = api_repr
1206 return config