Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2015 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Classes for load jobs."""
17import typing
18from typing import FrozenSet, List, Iterable, Optional
20from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
21from google.cloud.bigquery.external_config import HivePartitioningOptions
22from google.cloud.bigquery.format_options import ParquetOptions
23from google.cloud.bigquery import _helpers
24from google.cloud.bigquery.schema import SchemaField
25from google.cloud.bigquery.schema import _to_schema_fields
26from google.cloud.bigquery.table import RangePartitioning
27from google.cloud.bigquery.table import TableReference
28from google.cloud.bigquery.table import TimePartitioning
29from google.cloud.bigquery.job.base import _AsyncJob
30from google.cloud.bigquery.job.base import _JobConfig
31from google.cloud.bigquery.job.base import _JobReference
32from google.cloud.bigquery.query import ConnectionProperty
35class ColumnNameCharacterMap:
36 """Indicates the character map used for column names.
38 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap
39 """
41 COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED = "COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED"
42 """Unspecified column name character map."""
44 STRICT = "STRICT"
45 """Support flexible column name and reject invalid column names."""
47 V1 = "V1"
48 """ Support alphanumeric + underscore characters and names must start with
49 a letter or underscore. Invalid column names will be normalized."""
51 V2 = "V2"
52 """Support flexible column name. Invalid column names will be normalized."""
55class LoadJobConfig(_JobConfig):
56 """Configuration options for load jobs.
58 Set properties on the constructed configuration by using the property name
59 as the name of a keyword argument. Values which are unset or :data:`None`
60 use the BigQuery REST API default values. See the `BigQuery REST API
61 reference documentation
62 <https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_
63 for a list of default values.
65 Required options differ based on the
66 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.
67 For example, the BigQuery API's default value for
68 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.
69 When loading a CSV file, either
70 :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or
71 :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to
72 :data:`True`.
73 """
75 def __init__(self, **kwargs) -> None:
76 super(LoadJobConfig, self).__init__("load", **kwargs)
78 @property
79 def allow_jagged_rows(self):
80 """Optional[bool]: Allow missing trailing optional columns (CSV only).
82 See:
83 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_jagged_rows
84 """
85 return self._get_sub_prop("allowJaggedRows")
87 @allow_jagged_rows.setter
88 def allow_jagged_rows(self, value):
89 self._set_sub_prop("allowJaggedRows", value)
91 @property
92 def allow_quoted_newlines(self):
93 """Optional[bool]: Allow quoted data containing newline characters (CSV only).
95 See:
96 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_quoted_newlines
97 """
98 return self._get_sub_prop("allowQuotedNewlines")
100 @allow_quoted_newlines.setter
101 def allow_quoted_newlines(self, value):
102 self._set_sub_prop("allowQuotedNewlines", value)
104 @property
105 def autodetect(self):
106 """Optional[bool]: Automatically infer the schema from a sample of the data.
108 See:
109 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.autodetect
110 """
111 return self._get_sub_prop("autodetect")
113 @autodetect.setter
114 def autodetect(self, value):
115 self._set_sub_prop("autodetect", value)
117 @property
118 def clustering_fields(self):
119 """Optional[List[str]]: Fields defining clustering for the table
121 (Defaults to :data:`None`).
123 Clustering fields are immutable after table creation.
125 .. note::
127 BigQuery supports clustering for both partitioned and
128 non-partitioned tables.
129 """
130 prop = self._get_sub_prop("clustering")
131 if prop is not None:
132 return list(prop.get("fields", ()))
134 @clustering_fields.setter
135 def clustering_fields(self, value):
136 """Optional[List[str]]: Fields defining clustering for the table
138 (Defaults to :data:`None`).
139 """
140 if value is not None:
141 self._set_sub_prop("clustering", {"fields": value})
142 else:
143 self._del_sub_prop("clustering")
145 @property
146 def connection_properties(self) -> List[ConnectionProperty]:
147 """Connection properties.
149 See
150 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.connection_properties
152 .. versionadded:: 3.7.0
153 """
154 resource = self._get_sub_prop("connectionProperties", [])
155 return [ConnectionProperty.from_api_repr(prop) for prop in resource]
157 @connection_properties.setter
158 def connection_properties(self, value: Iterable[ConnectionProperty]):
159 self._set_sub_prop(
160 "connectionProperties",
161 [prop.to_api_repr() for prop in value],
162 )
164 @property
165 def create_disposition(self):
166 """Optional[google.cloud.bigquery.job.CreateDisposition]: Specifies behavior
167 for creating tables.
169 See:
170 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition
171 """
172 return self._get_sub_prop("createDisposition")
174 @create_disposition.setter
175 def create_disposition(self, value):
176 self._set_sub_prop("createDisposition", value)
178 @property
179 def create_session(self) -> Optional[bool]:
180 """[Preview] If :data:`True`, creates a new session, where
181 :attr:`~google.cloud.bigquery.job.LoadJob.session_info` will contain a
182 random server generated session id.
184 If :data:`False`, runs load job with an existing ``session_id`` passed in
185 :attr:`~google.cloud.bigquery.job.LoadJobConfig.connection_properties`,
186 otherwise runs load job in non-session mode.
188 See
189 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_session
191 .. versionadded:: 3.7.0
192 """
193 return self._get_sub_prop("createSession")
195 @create_session.setter
196 def create_session(self, value: Optional[bool]):
197 self._set_sub_prop("createSession", value)
199 @property
200 def decimal_target_types(self) -> Optional[FrozenSet[str]]:
201 """Possible SQL data types to which the source decimal values are converted.
203 See:
204 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types
206 .. versionadded:: 2.21.0
207 """
208 prop = self._get_sub_prop("decimalTargetTypes")
209 if prop is not None:
210 prop = frozenset(prop)
211 return prop
213 @decimal_target_types.setter
214 def decimal_target_types(self, value: Optional[Iterable[str]]):
215 if value is not None:
216 self._set_sub_prop("decimalTargetTypes", list(value))
217 else:
218 self._del_sub_prop("decimalTargetTypes")
220 @property
221 def destination_encryption_configuration(self):
222 """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom
223 encryption configuration for the destination table.
225 Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
226 if using default encryption.
228 See:
229 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_encryption_configuration
230 """
231 prop = self._get_sub_prop("destinationEncryptionConfiguration")
232 if prop is not None:
233 prop = EncryptionConfiguration.from_api_repr(prop)
234 return prop
236 @destination_encryption_configuration.setter
237 def destination_encryption_configuration(self, value):
238 api_repr = value
239 if value is not None:
240 api_repr = value.to_api_repr()
241 self._set_sub_prop("destinationEncryptionConfiguration", api_repr)
242 else:
243 self._del_sub_prop("destinationEncryptionConfiguration")
245 @property
246 def destination_table_description(self):
247 """Optional[str]: Description of the destination table.
249 See:
250 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
251 """
252 prop = self._get_sub_prop("destinationTableProperties")
253 if prop is not None:
254 return prop["description"]
256 @destination_table_description.setter
257 def destination_table_description(self, value):
258 keys = [self._job_type, "destinationTableProperties", "description"]
259 if value is not None:
260 _helpers._set_sub_prop(self._properties, keys, value)
261 else:
262 _helpers._del_sub_prop(self._properties, keys)
264 @property
265 def destination_table_friendly_name(self):
266 """Optional[str]: Name given to destination table.
268 See:
269 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
270 """
271 prop = self._get_sub_prop("destinationTableProperties")
272 if prop is not None:
273 return prop["friendlyName"]
275 @destination_table_friendly_name.setter
276 def destination_table_friendly_name(self, value):
277 keys = [self._job_type, "destinationTableProperties", "friendlyName"]
278 if value is not None:
279 _helpers._set_sub_prop(self._properties, keys, value)
280 else:
281 _helpers._del_sub_prop(self._properties, keys)
283 @property
284 def encoding(self):
285 """Optional[google.cloud.bigquery.job.Encoding]: The character encoding of the
286 data.
288 See:
289 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.encoding
290 """
291 return self._get_sub_prop("encoding")
293 @encoding.setter
294 def encoding(self, value):
295 self._set_sub_prop("encoding", value)
297 @property
298 def field_delimiter(self):
299 """Optional[str]: The separator for fields in a CSV file.
301 See:
302 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.field_delimiter
303 """
304 return self._get_sub_prop("fieldDelimiter")
306 @field_delimiter.setter
307 def field_delimiter(self, value):
308 self._set_sub_prop("fieldDelimiter", value)
310 @property
311 def hive_partitioning(self):
312 """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \
313 it configures hive partitioning support.
315 .. note::
316 **Experimental**. This feature is experimental and might change or
317 have limited support.
319 See:
320 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.hive_partitioning_options
321 """
322 prop = self._get_sub_prop("hivePartitioningOptions")
323 if prop is None:
324 return None
325 return HivePartitioningOptions.from_api_repr(prop)
327 @hive_partitioning.setter
328 def hive_partitioning(self, value):
329 if value is not None:
330 if isinstance(value, HivePartitioningOptions):
331 value = value.to_api_repr()
332 else:
333 raise TypeError("Expected a HivePartitioningOptions instance or None.")
335 self._set_sub_prop("hivePartitioningOptions", value)
337 @property
338 def ignore_unknown_values(self):
339 """Optional[bool]: Ignore extra values not represented in the table schema.
341 See:
342 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.ignore_unknown_values
343 """
344 return self._get_sub_prop("ignoreUnknownValues")
346 @ignore_unknown_values.setter
347 def ignore_unknown_values(self, value):
348 self._set_sub_prop("ignoreUnknownValues", value)
350 @property
351 def json_extension(self):
352 """Optional[str]: The extension to use for writing JSON data to BigQuery. Only supports GeoJSON currently.
354 See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.json_extension
356 """
357 return self._get_sub_prop("jsonExtension")
359 @json_extension.setter
360 def json_extension(self, value):
361 self._set_sub_prop("jsonExtension", value)
363 @property
364 def max_bad_records(self):
365 """Optional[int]: Number of invalid rows to ignore.
367 See:
368 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.max_bad_records
369 """
370 return _helpers._int_or_none(self._get_sub_prop("maxBadRecords"))
372 @max_bad_records.setter
373 def max_bad_records(self, value):
374 self._set_sub_prop("maxBadRecords", value)
376 @property
377 def null_marker(self):
378 """Optional[str]: Represents a null value (CSV only).
380 See:
381 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_marker
382 """
383 return self._get_sub_prop("nullMarker")
385 @null_marker.setter
386 def null_marker(self, value):
387 self._set_sub_prop("nullMarker", value)
389 @property
390 def preserve_ascii_control_characters(self):
391 """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.
393 See:
394 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters
395 """
396 return self._get_sub_prop("preserveAsciiControlCharacters")
398 @preserve_ascii_control_characters.setter
399 def preserve_ascii_control_characters(self, value):
400 self._set_sub_prop("preserveAsciiControlCharacters", bool(value))
402 @property
403 def projection_fields(self) -> Optional[List[str]]:
404 """Optional[List[str]]: If
405 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to
406 "DATASTORE_BACKUP", indicates which entity properties to load into
407 BigQuery from a Cloud Datastore backup.
409 Property names are case sensitive and must be top-level properties. If
410 no properties are specified, BigQuery loads all properties. If any
411 named property isn't found in the Cloud Datastore backup, an invalid
412 error is returned in the job result.
414 See:
415 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields
416 """
417 return self._get_sub_prop("projectionFields")
419 @projection_fields.setter
420 def projection_fields(self, value: Optional[List[str]]):
421 self._set_sub_prop("projectionFields", value)
423 @property
424 def quote_character(self):
425 """Optional[str]: Character used to quote data sections (CSV only).
427 See:
428 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.quote
429 """
430 return self._get_sub_prop("quote")
432 @quote_character.setter
433 def quote_character(self, value):
434 self._set_sub_prop("quote", value)
436 @property
437 def range_partitioning(self):
438 """Optional[google.cloud.bigquery.table.RangePartitioning]:
439 Configures range-based partitioning for destination table.
441 .. note::
442 **Beta**. The integer range partitioning feature is in a
443 pre-release state and might change or have limited support.
445 Only specify at most one of
446 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
447 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
449 Raises:
450 ValueError:
451 If the value is not
452 :class:`~google.cloud.bigquery.table.RangePartitioning` or
453 :data:`None`.
454 """
455 resource = self._get_sub_prop("rangePartitioning")
456 if resource is not None:
457 return RangePartitioning(_properties=resource)
459 @range_partitioning.setter
460 def range_partitioning(self, value):
461 resource = value
462 if isinstance(value, RangePartitioning):
463 resource = value._properties
464 elif value is not None:
465 raise ValueError(
466 "Expected value to be RangePartitioning or None, got {}.".format(value)
467 )
468 self._set_sub_prop("rangePartitioning", resource)
470 @property
471 def reference_file_schema_uri(self):
472 """Optional[str]:
473 When creating an external table, the user can provide a reference file with the
474 table schema. This is enabled for the following formats:
476 AVRO, PARQUET, ORC
477 """
478 return self._get_sub_prop("referenceFileSchemaUri")
480 @reference_file_schema_uri.setter
481 def reference_file_schema_uri(self, value):
482 return self._set_sub_prop("referenceFileSchemaUri", value)
484 @property
485 def schema(self):
486 """Optional[Sequence[Union[ \
487 :class:`~google.cloud.bigquery.schema.SchemaField`, \
488 Mapping[str, Any] \
489 ]]]: Schema of the destination table.
491 See:
492 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema
493 """
494 schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])
495 if schema is None:
496 return
497 return [SchemaField.from_api_repr(field) for field in schema]
499 @schema.setter
500 def schema(self, value):
501 if value is None:
502 self._del_sub_prop("schema")
503 return
505 value = _to_schema_fields(value)
507 _helpers._set_sub_prop(
508 self._properties,
509 ["load", "schema", "fields"],
510 [field.to_api_repr() for field in value],
511 )
513 @property
514 def schema_update_options(self):
515 """Optional[List[google.cloud.bigquery.job.SchemaUpdateOption]]: Specifies
516 updates to the destination table schema to allow as a side effect of
517 the load job.
518 """
519 return self._get_sub_prop("schemaUpdateOptions")
521 @schema_update_options.setter
522 def schema_update_options(self, values):
523 self._set_sub_prop("schemaUpdateOptions", values)
525 @property
526 def skip_leading_rows(self):
527 """Optional[int]: Number of rows to skip when reading data (CSV only).
529 See:
530 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.skip_leading_rows
531 """
532 return _helpers._int_or_none(self._get_sub_prop("skipLeadingRows"))
534 @skip_leading_rows.setter
535 def skip_leading_rows(self, value):
536 self._set_sub_prop("skipLeadingRows", str(value))
538 @property
539 def source_format(self):
540 """Optional[google.cloud.bigquery.job.SourceFormat]: File format of the data.
542 See:
543 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_format
544 """
545 return self._get_sub_prop("sourceFormat")
547 @source_format.setter
548 def source_format(self, value):
549 self._set_sub_prop("sourceFormat", value)
551 @property
552 def date_format(self) -> Optional[str]:
553 """Optional[str]: Date format used for parsing DATE values.
555 See:
556 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.date_format
557 """
558 return self._get_sub_prop("dateFormat")
560 @date_format.setter
561 def date_format(self, value: Optional[str]):
562 self._set_sub_prop("dateFormat", value)
564 @property
565 def time_zone(self) -> Optional[str]:
566 """Optional[str]: Default time zone that will apply when parsing timestamp
567 values that have no specific time zone.
569 See:
570 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_zone
571 """
572 return self._get_sub_prop("timeZone")
574 @time_zone.setter
575 def time_zone(self, value: Optional[str]):
576 self._set_sub_prop("timeZone", value)
578 @property
579 def time_partitioning(self):
580 """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
581 partitioning for the destination table.
583 Only specify at most one of
584 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
585 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
586 """
587 prop = self._get_sub_prop("timePartitioning")
588 if prop is not None:
589 prop = TimePartitioning.from_api_repr(prop)
590 return prop
592 @time_partitioning.setter
593 def time_partitioning(self, value):
594 api_repr = value
595 if value is not None:
596 api_repr = value.to_api_repr()
597 self._set_sub_prop("timePartitioning", api_repr)
598 else:
599 self._del_sub_prop("timePartitioning")
601 @property
602 def use_avro_logical_types(self):
603 """Optional[bool]: For loads of Avro data, governs whether Avro logical types are
604 converted to their corresponding BigQuery types (e.g. TIMESTAMP) rather than
605 raw types (e.g. INTEGER).
606 """
607 return self._get_sub_prop("useAvroLogicalTypes")
609 @use_avro_logical_types.setter
610 def use_avro_logical_types(self, value):
611 self._set_sub_prop("useAvroLogicalTypes", bool(value))
613 @property
614 def write_disposition(self):
615 """Optional[google.cloud.bigquery.job.WriteDisposition]: Action that occurs if
616 the destination table already exists.
618 See:
619 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.write_disposition
620 """
621 return self._get_sub_prop("writeDisposition")
623 @write_disposition.setter
624 def write_disposition(self, value):
625 self._set_sub_prop("writeDisposition", value)
627 @property
628 def parquet_options(self):
629 """Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional
630 properties to set if ``sourceFormat`` is set to PARQUET.
632 See:
633 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.parquet_options
634 """
635 prop = self._get_sub_prop("parquetOptions")
636 if prop is not None:
637 prop = ParquetOptions.from_api_repr(prop)
638 return prop
640 @parquet_options.setter
641 def parquet_options(self, value):
642 if value is not None:
643 self._set_sub_prop("parquetOptions", value.to_api_repr())
644 else:
645 self._del_sub_prop("parquetOptions")
647 @property
648 def column_name_character_map(self) -> str:
649 """Optional[google.cloud.bigquery.job.ColumnNameCharacterMap]:
650 Character map supported for column names in CSV/Parquet loads. Defaults
651 to STRICT and can be overridden by Project Config Service. Using this
652 option with unsupported load formats will result in an error.
654 See
655 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.column_name_character_map
656 """
657 return self._get_sub_prop(
658 "columnNameCharacterMap",
659 ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,
660 )
662 @column_name_character_map.setter
663 def column_name_character_map(self, value: Optional[str]):
664 if value is None:
665 value = ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED
666 self._set_sub_prop("columnNameCharacterMap", value)
669class LoadJob(_AsyncJob):
670 """Asynchronous job for loading data into a table.
672 Can load from Google Cloud Storage URIs or from a file.
674 Args:
675 job_id (str): the job's ID
677 source_uris (Optional[Sequence[str]]):
678 URIs of one or more data files to be loaded. See
679 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
680 for supported URI formats. Pass None for jobs that load from a file.
682 destination (google.cloud.bigquery.table.TableReference): reference to table into which data is to be loaded.
684 client (google.cloud.bigquery.client.Client):
685 A client which holds credentials and project configuration
686 for the dataset (which requires a project).
687 """
689 _JOB_TYPE = "load"
690 _CONFIG_CLASS = LoadJobConfig
692 def __init__(self, job_id, source_uris, destination, client, job_config=None):
693 super(LoadJob, self).__init__(job_id, client)
695 if job_config is not None:
696 self._properties["configuration"] = job_config._properties
698 if source_uris is not None:
699 _helpers._set_sub_prop(
700 self._properties, ["configuration", "load", "sourceUris"], source_uris
701 )
703 if destination is not None:
704 _helpers._set_sub_prop(
705 self._properties,
706 ["configuration", "load", "destinationTable"],
707 destination.to_api_repr(),
708 )
710 @property
711 def configuration(self) -> LoadJobConfig:
712 """The configuration for this load job."""
713 return typing.cast(LoadJobConfig, super().configuration)
715 @property
716 def destination(self):
717 """google.cloud.bigquery.table.TableReference: table where loaded rows are written
719 See:
720 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_table
721 """
722 dest_config = _helpers._get_sub_prop(
723 self._properties, ["configuration", "load", "destinationTable"]
724 )
725 return TableReference.from_api_repr(dest_config)
727 @property
728 def source_uris(self):
729 """Optional[Sequence[str]]: URIs of data files to be loaded. See
730 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris
731 for supported URI formats. None for jobs that load from a file.
732 """
733 return _helpers._get_sub_prop(
734 self._properties, ["configuration", "load", "sourceUris"]
735 )
737 @property
738 def allow_jagged_rows(self):
739 """See
740 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.
741 """
742 return self.configuration.allow_jagged_rows
744 @property
745 def allow_quoted_newlines(self):
746 """See
747 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.
748 """
749 return self.configuration.allow_quoted_newlines
751 @property
752 def autodetect(self):
753 """See
754 :attr:`google.cloud.bigquery.job.LoadJobConfig.autodetect`.
755 """
756 return self.configuration.autodetect
758 @property
759 def connection_properties(self) -> List[ConnectionProperty]:
760 """See
761 :attr:`google.cloud.bigquery.job.LoadJobConfig.connection_properties`.
763 .. versionadded:: 3.7.0
764 """
765 return self.configuration.connection_properties
767 @property
768 def create_disposition(self):
769 """See
770 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_disposition`.
771 """
772 return self.configuration.create_disposition
774 @property
775 def create_session(self) -> Optional[bool]:
776 """See
777 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_session`.
779 .. versionadded:: 3.7.0
780 """
781 return self.configuration.create_session
783 @property
784 def encoding(self):
785 """See
786 :attr:`google.cloud.bigquery.job.LoadJobConfig.encoding`.
787 """
788 return self.configuration.encoding
790 @property
791 def field_delimiter(self):
792 """See
793 :attr:`google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.
794 """
795 return self.configuration.field_delimiter
797 @property
798 def ignore_unknown_values(self):
799 """See
800 :attr:`google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.
801 """
802 return self.configuration.ignore_unknown_values
804 @property
805 def max_bad_records(self):
806 """See
807 :attr:`google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.
808 """
809 return self.configuration.max_bad_records
811 @property
812 def null_marker(self):
813 """See
814 :attr:`google.cloud.bigquery.job.LoadJobConfig.null_marker`.
815 """
816 return self.configuration.null_marker
818 @property
819 def quote_character(self):
820 """See
821 :attr:`google.cloud.bigquery.job.LoadJobConfig.quote_character`.
822 """
823 return self.configuration.quote_character
825 @property
826 def reference_file_schema_uri(self):
827 """See:
828 attr:`google.cloud.bigquery.job.LoadJobConfig.reference_file_schema_uri`.
829 """
830 return self.configuration.reference_file_schema_uri
832 @property
833 def skip_leading_rows(self):
834 """See
835 :attr:`google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.
836 """
837 return self.configuration.skip_leading_rows
839 @property
840 def source_format(self):
841 """See
842 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format`.
843 """
844 return self.configuration.source_format
846 @property
847 def write_disposition(self):
848 """See
849 :attr:`google.cloud.bigquery.job.LoadJobConfig.write_disposition`.
850 """
851 return self.configuration.write_disposition
853 @property
854 def schema(self):
855 """See
856 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema`.
857 """
858 return self.configuration.schema
860 @property
861 def destination_encryption_configuration(self):
862 """google.cloud.bigquery.encryption_configuration.EncryptionConfiguration: Custom
863 encryption configuration for the destination table.
865 Custom encryption configuration (e.g., Cloud KMS keys)
866 or :data:`None` if using default encryption.
868 See
869 :attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.
870 """
871 return self.configuration.destination_encryption_configuration
873 @property
874 def destination_table_description(self):
875 """Optional[str] name given to destination table.
877 See:
878 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description
879 """
880 return self.configuration.destination_table_description
882 @property
883 def destination_table_friendly_name(self):
884 """Optional[str] name given to destination table.
886 See:
887 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name
888 """
889 return self.configuration.destination_table_friendly_name
891 @property
892 def range_partitioning(self):
893 """See
894 :attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
895 """
896 return self.configuration.range_partitioning
898 @property
899 def time_partitioning(self):
900 """See
901 :attr:`google.cloud.bigquery.job.LoadJobConfig.time_partitioning`.
902 """
903 return self.configuration.time_partitioning
905 @property
906 def use_avro_logical_types(self):
907 """See
908 :attr:`google.cloud.bigquery.job.LoadJobConfig.use_avro_logical_types`.
909 """
910 return self.configuration.use_avro_logical_types
912 @property
913 def clustering_fields(self):
914 """See
915 :attr:`google.cloud.bigquery.job.LoadJobConfig.clustering_fields`.
916 """
917 return self.configuration.clustering_fields
919 @property
920 def date_format(self):
921 """See
922 :attr:`google.cloud.bigquery.job.LoadJobConfig.date_format`.
923 """
924 return self.configuration.date_format
926 @property
927 def time_zone(self):
928 """See
929 :attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.
930 """
931 return self.configuration.time_zone
933 @property
934 def schema_update_options(self):
935 """See
936 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.
937 """
938 return self.configuration.schema_update_options
940 @property
941 def input_file_bytes(self):
942 """Count of bytes loaded from source files.
944 Returns:
945 Optional[int]: the count (None until set from the server).
947 Raises:
948 ValueError: for invalid value types.
949 """
950 return _helpers._int_or_none(
951 _helpers._get_sub_prop(
952 self._properties, ["statistics", "load", "inputFileBytes"]
953 )
954 )
956 @property
957 def input_files(self):
958 """Count of source files.
960 Returns:
961 Optional[int]: the count (None until set from the server).
962 """
963 return _helpers._int_or_none(
964 _helpers._get_sub_prop(
965 self._properties, ["statistics", "load", "inputFiles"]
966 )
967 )
969 @property
970 def output_bytes(self):
971 """Count of bytes saved to destination table.
973 Returns:
974 Optional[int]: the count (None until set from the server).
975 """
976 return _helpers._int_or_none(
977 _helpers._get_sub_prop(
978 self._properties, ["statistics", "load", "outputBytes"]
979 )
980 )
982 @property
983 def output_rows(self):
984 """Count of rows saved to destination table.
986 Returns:
987 Optional[int]: the count (None until set from the server).
988 """
989 return _helpers._int_or_none(
990 _helpers._get_sub_prop(
991 self._properties, ["statistics", "load", "outputRows"]
992 )
993 )
995 def to_api_repr(self):
996 """Generate a resource for :meth:`_begin`."""
997 # Exclude statistics, if set.
998 return {
999 "jobReference": self._properties["jobReference"],
1000 "configuration": self._properties["configuration"],
1001 }
1003 @classmethod
1004 def from_api_repr(cls, resource: dict, client) -> "LoadJob":
1005 """Factory: construct a job given its API representation
1007 .. note::
1009 This method assumes that the project found in the resource matches
1010 the client's project.
1012 Args:
1013 resource (Dict): dataset job representation returned from the API
1015 client (google.cloud.bigquery.client.Client):
1016 Client which holds credentials and project
1017 configuration for the dataset.
1019 Returns:
1020 google.cloud.bigquery.job.LoadJob: Job parsed from ``resource``.
1021 """
1022 cls._check_resource_config(resource)
1023 job_ref = _JobReference._from_api_repr(resource["jobReference"])
1024 job = cls(job_ref, None, None, client)
1025 job._set_properties(resource)
1026 return job