Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/google/cloud/bigquery/job/load.py: 56%

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

15"""Classes for load jobs."""

17import typing

18from typing import FrozenSet, List, Iterable, Optional

20from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration

21from google.cloud.bigquery.external_config import HivePartitioningOptions

22from google.cloud.bigquery.format_options import ParquetOptions

23from google.cloud.bigquery import _helpers

24from google.cloud.bigquery.schema import SchemaField

25from google.cloud.bigquery.schema import _to_schema_fields

26from google.cloud.bigquery.table import RangePartitioning

27from google.cloud.bigquery.table import TableReference

28from google.cloud.bigquery.table import TimePartitioning

29from google.cloud.bigquery.job.base import _AsyncJob

30from google.cloud.bigquery.job.base import _JobConfig

31from google.cloud.bigquery.job.base import _JobReference

32from google.cloud.bigquery.query import ConnectionProperty

35class ColumnNameCharacterMap:

36 """Indicates the character map used for column names.

38 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap

39 """

41 COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED = "COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED"

42 """Unspecified column name character map."""

44 STRICT = "STRICT"

45 """Support flexible column name and reject invalid column names."""

47 V1 = "V1"

48 """ Support alphanumeric + underscore characters and names must start with

49 a letter or underscore. Invalid column names will be normalized."""

51 V2 = "V2"

52 """Support flexible column name. Invalid column names will be normalized."""

55class LoadJobConfig(_JobConfig):

56 """Configuration options for load jobs.

58 Set properties on the constructed configuration by using the property name

59 as the name of a keyword argument. Values which are unset or :data:`None`

60 use the BigQuery REST API default values. See the `BigQuery REST API

61 reference documentation

62 <https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_

63 for a list of default values.

65 Required options differ based on the

66 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.

67 For example, the BigQuery API's default value for

68 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.

69 When loading a CSV file, either

70 :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or

71 :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to

72 :data:`True`.

73 """

75 def __init__(self, **kwargs) -> None:

76 super(LoadJobConfig, self).__init__("load", **kwargs)

78 @property

79 def allow_jagged_rows(self):

80 """Optional[bool]: Allow missing trailing optional columns (CSV only).

82 See:

83 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_jagged_rows

84 """

85 return self._get_sub_prop("allowJaggedRows")

87 @allow_jagged_rows.setter

88 def allow_jagged_rows(self, value):

89 self._set_sub_prop("allowJaggedRows", value)

91 @property

92 def allow_quoted_newlines(self):

93 """Optional[bool]: Allow quoted data containing newline characters (CSV only).

95 See:

96 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_quoted_newlines

97 """

98 return self._get_sub_prop("allowQuotedNewlines")

100 @allow_quoted_newlines.setter

101 def allow_quoted_newlines(self, value):

102 self._set_sub_prop("allowQuotedNewlines", value)

103

104 @property

105 def autodetect(self):

106 """Optional[bool]: Automatically infer the schema from a sample of the data.

107

108 See:

109 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.autodetect

110 """

111 return self._get_sub_prop("autodetect")

112

113 @autodetect.setter

114 def autodetect(self, value):

115 self._set_sub_prop("autodetect", value)

116

117 @property

118 def clustering_fields(self):

119 """Optional[List[str]]: Fields defining clustering for the table

120

121 (Defaults to :data:`None`).

122

123 Clustering fields are immutable after table creation.

124

125 .. note::

126

127 BigQuery supports clustering for both partitioned and

128 non-partitioned tables.

129 """

130 prop = self._get_sub_prop("clustering")

131 if prop is not None:

132 return list(prop.get("fields", ()))

133

134 @clustering_fields.setter

135 def clustering_fields(self, value):

136 """Optional[List[str]]: Fields defining clustering for the table

137

138 (Defaults to :data:`None`).

139 """

140 if value is not None:

141 self._set_sub_prop("clustering", {"fields": value})

142 else:

143 self._del_sub_prop("clustering")

144

145 @property

146 def connection_properties(self) -> List[ConnectionProperty]:

147 """Connection properties.

148

149 See

150 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.connection_properties

151

152 .. versionadded:: 3.7.0

153 """

154 resource = self._get_sub_prop("connectionProperties", [])

155 return [ConnectionProperty.from_api_repr(prop) for prop in resource]

156

157 @connection_properties.setter

158 def connection_properties(self, value: Iterable[ConnectionProperty]):

159 self._set_sub_prop(

160 "connectionProperties",

161 [prop.to_api_repr() for prop in value],

162 )

163

164 @property

165 def create_disposition(self):

166 """Optional[google.cloud.bigquery.job.CreateDisposition]: Specifies behavior

167 for creating tables.

168

169 See:

170 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition

171 """

172 return self._get_sub_prop("createDisposition")

173

174 @create_disposition.setter

175 def create_disposition(self, value):

176 self._set_sub_prop("createDisposition", value)

177

178 @property

179 def create_session(self) -> Optional[bool]:

180 """[Preview] If :data:`True`, creates a new session, where

181 :attr:`~google.cloud.bigquery.job.LoadJob.session_info` will contain a

182 random server generated session id.

183

184 If :data:`False`, runs load job with an existing ``session_id`` passed in

185 :attr:`~google.cloud.bigquery.job.LoadJobConfig.connection_properties`,

186 otherwise runs load job in non-session mode.

187

188 See

189 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_session

190

191 .. versionadded:: 3.7.0

192 """

193 return self._get_sub_prop("createSession")

194

195 @create_session.setter

196 def create_session(self, value: Optional[bool]):

197 self._set_sub_prop("createSession", value)

198

199 @property

200 def decimal_target_types(self) -> Optional[FrozenSet[str]]:

201 """Possible SQL data types to which the source decimal values are converted.

202

203 See:

204 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types

205

206 .. versionadded:: 2.21.0

207 """

208 prop = self._get_sub_prop("decimalTargetTypes")

209 if prop is not None:

210 prop = frozenset(prop)

211 return prop

212

213 @decimal_target_types.setter

214 def decimal_target_types(self, value: Optional[Iterable[str]]):

215 if value is not None:

216 self._set_sub_prop("decimalTargetTypes", list(value))

217 else:

218 self._del_sub_prop("decimalTargetTypes")

219

220 @property

221 def destination_encryption_configuration(self):

222 """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom

223 encryption configuration for the destination table.

224

225 Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`

226 if using default encryption.

227

228 See:

229 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_encryption_configuration

230 """

231 prop = self._get_sub_prop("destinationEncryptionConfiguration")

232 if prop is not None:

233 prop = EncryptionConfiguration.from_api_repr(prop)

234 return prop

235

236 @destination_encryption_configuration.setter

237 def destination_encryption_configuration(self, value):

238 api_repr = value

239 if value is not None:

240 api_repr = value.to_api_repr()

241 self._set_sub_prop("destinationEncryptionConfiguration", api_repr)

242 else:

243 self._del_sub_prop("destinationEncryptionConfiguration")

244

245 @property

246 def destination_table_description(self):

247 """Optional[str]: Description of the destination table.

248

249 See:

250 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description

251 """

252 prop = self._get_sub_prop("destinationTableProperties")

253 if prop is not None:

254 return prop["description"]

255

256 @destination_table_description.setter

257 def destination_table_description(self, value):

258 keys = [self._job_type, "destinationTableProperties", "description"]

259 if value is not None:

260 _helpers._set_sub_prop(self._properties, keys, value)

261 else:

262 _helpers._del_sub_prop(self._properties, keys)

263

264 @property

265 def destination_table_friendly_name(self):

266 """Optional[str]: Name given to destination table.

267

268 See:

269 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name

270 """

271 prop = self._get_sub_prop("destinationTableProperties")

272 if prop is not None:

273 return prop["friendlyName"]

274

275 @destination_table_friendly_name.setter

276 def destination_table_friendly_name(self, value):

277 keys = [self._job_type, "destinationTableProperties", "friendlyName"]

278 if value is not None:

279 _helpers._set_sub_prop(self._properties, keys, value)

280 else:

281 _helpers._del_sub_prop(self._properties, keys)

282

283 @property

284 def encoding(self):

285 """Optional[google.cloud.bigquery.job.Encoding]: The character encoding of the

286 data.

287

288 See:

289 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.encoding

290 """

291 return self._get_sub_prop("encoding")

292

293 @encoding.setter

294 def encoding(self, value):

295 self._set_sub_prop("encoding", value)

296

297 @property

298 def field_delimiter(self):

299 """Optional[str]: The separator for fields in a CSV file.

300

301 See:

302 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.field_delimiter

303 """

304 return self._get_sub_prop("fieldDelimiter")

305

306 @field_delimiter.setter

307 def field_delimiter(self, value):

308 self._set_sub_prop("fieldDelimiter", value)

309

310 @property

311 def hive_partitioning(self):

312 """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \

313 it configures hive partitioning support.

314

315 .. note::

316 **Experimental**. This feature is experimental and might change or

317 have limited support.

318

319 See:

320 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.hive_partitioning_options

321 """

322 prop = self._get_sub_prop("hivePartitioningOptions")

323 if prop is None:

324 return None

325 return HivePartitioningOptions.from_api_repr(prop)

326

327 @hive_partitioning.setter

328 def hive_partitioning(self, value):

329 if value is not None:

330 if isinstance(value, HivePartitioningOptions):

331 value = value.to_api_repr()

332 else:

333 raise TypeError("Expected a HivePartitioningOptions instance or None.")

334

335 self._set_sub_prop("hivePartitioningOptions", value)

336

337 @property

338 def ignore_unknown_values(self):

339 """Optional[bool]: Ignore extra values not represented in the table schema.

340

341 See:

342 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.ignore_unknown_values

343 """

344 return self._get_sub_prop("ignoreUnknownValues")

345

346 @ignore_unknown_values.setter

347 def ignore_unknown_values(self, value):

348 self._set_sub_prop("ignoreUnknownValues", value)

349

350 @property

351 def json_extension(self):

352 """Optional[str]: The extension to use for writing JSON data to BigQuery. Only supports GeoJSON currently.

353

354 See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.json_extension

355

356 """

357 return self._get_sub_prop("jsonExtension")

358

359 @json_extension.setter

360 def json_extension(self, value):

361 self._set_sub_prop("jsonExtension", value)

362

363 @property

364 def max_bad_records(self):

365 """Optional[int]: Number of invalid rows to ignore.

366

367 See:

368 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.max_bad_records

369 """

370 return _helpers._int_or_none(self._get_sub_prop("maxBadRecords"))

371

372 @max_bad_records.setter

373 def max_bad_records(self, value):

374 self._set_sub_prop("maxBadRecords", value)

375

376 @property

377 def null_marker(self):

378 """Optional[str]: Represents a null value (CSV only).

379

380 See:

381 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_marker

382 """

383 return self._get_sub_prop("nullMarker")

384

385 @null_marker.setter

386 def null_marker(self, value):

387 self._set_sub_prop("nullMarker", value)

388

389 @property

390 def preserve_ascii_control_characters(self):

391 """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.

392

393 See:

394 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters

395 """

396 return self._get_sub_prop("preserveAsciiControlCharacters")

397

398 @preserve_ascii_control_characters.setter

399 def preserve_ascii_control_characters(self, value):

400 self._set_sub_prop("preserveAsciiControlCharacters", bool(value))

401

402 @property

403 def projection_fields(self) -> Optional[List[str]]:

404 """Optional[List[str]]: If

405 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to

406 "DATASTORE_BACKUP", indicates which entity properties to load into

407 BigQuery from a Cloud Datastore backup.

408

409 Property names are case sensitive and must be top-level properties. If

410 no properties are specified, BigQuery loads all properties. If any

411 named property isn't found in the Cloud Datastore backup, an invalid

412 error is returned in the job result.

413

414 See:

415 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields

416 """

417 return self._get_sub_prop("projectionFields")

418

419 @projection_fields.setter

420 def projection_fields(self, value: Optional[List[str]]):

421 self._set_sub_prop("projectionFields", value)

422

423 @property

424 def quote_character(self):

425 """Optional[str]: Character used to quote data sections (CSV only).

426

427 See:

428 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.quote

429 """

430 return self._get_sub_prop("quote")

431

432 @quote_character.setter

433 def quote_character(self, value):

434 self._set_sub_prop("quote", value)

435

436 @property

437 def range_partitioning(self):

438 """Optional[google.cloud.bigquery.table.RangePartitioning]:

439 Configures range-based partitioning for destination table.

440

441 .. note::

442 **Beta**. The integer range partitioning feature is in a

443 pre-release state and might change or have limited support.

444

445 Only specify at most one of

446 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or

447 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

448

449 Raises:

450 ValueError:

451 If the value is not

452 :class:`~google.cloud.bigquery.table.RangePartitioning` or

453 :data:`None`.

454 """

455 resource = self._get_sub_prop("rangePartitioning")

456 if resource is not None:

457 return RangePartitioning(_properties=resource)

458

459 @range_partitioning.setter

460 def range_partitioning(self, value):

461 resource = value

462 if isinstance(value, RangePartitioning):

463 resource = value._properties

464 elif value is not None:

465 raise ValueError(

466 "Expected value to be RangePartitioning or None, got {}.".format(value)

467 )

468 self._set_sub_prop("rangePartitioning", resource)

469

470 @property

471 def reference_file_schema_uri(self):

472 """Optional[str]:

473 When creating an external table, the user can provide a reference file with the

474 table schema. This is enabled for the following formats:

475

476 AVRO, PARQUET, ORC

477 """

478 return self._get_sub_prop("referenceFileSchemaUri")

479

480 @reference_file_schema_uri.setter

481 def reference_file_schema_uri(self, value):

482 return self._set_sub_prop("referenceFileSchemaUri", value)

483

484 @property

485 def schema(self):

486 """Optional[Sequence[Union[ \

487 :class:`~google.cloud.bigquery.schema.SchemaField`, \

488 Mapping[str, Any] \

489 ]]]: Schema of the destination table.

490

491 See:

492 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema

493 """

494 schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])

495 if schema is None:

496 return

497 return [SchemaField.from_api_repr(field) for field in schema]

498

499 @schema.setter

500 def schema(self, value):

501 if value is None:

502 self._del_sub_prop("schema")

503 return

504

505 value = _to_schema_fields(value)

506

507 _helpers._set_sub_prop(

508 self._properties,

509 ["load", "schema", "fields"],

510 [field.to_api_repr() for field in value],

511 )

512

513 @property

514 def schema_update_options(self):

515 """Optional[List[google.cloud.bigquery.job.SchemaUpdateOption]]: Specifies

516 updates to the destination table schema to allow as a side effect of

517 the load job.

518 """

519 return self._get_sub_prop("schemaUpdateOptions")

520

521 @schema_update_options.setter

522 def schema_update_options(self, values):

523 self._set_sub_prop("schemaUpdateOptions", values)

524

525 @property

526 def skip_leading_rows(self):

527 """Optional[int]: Number of rows to skip when reading data (CSV only).

528

529 See:

530 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.skip_leading_rows

531 """

532 return _helpers._int_or_none(self._get_sub_prop("skipLeadingRows"))

533

534 @skip_leading_rows.setter

535 def skip_leading_rows(self, value):

536 self._set_sub_prop("skipLeadingRows", str(value))

537

538 @property

539 def source_format(self):

540 """Optional[google.cloud.bigquery.job.SourceFormat]: File format of the data.

541

542 See:

543 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_format

544 """

545 return self._get_sub_prop("sourceFormat")

546

547 @source_format.setter

548 def source_format(self, value):

549 self._set_sub_prop("sourceFormat", value)

550

551 @property

552 def time_partitioning(self):

553 """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based

554 partitioning for the destination table.

555

556 Only specify at most one of

557 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or

558 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

559 """

560 prop = self._get_sub_prop("timePartitioning")

561 if prop is not None:

562 prop = TimePartitioning.from_api_repr(prop)

563 return prop

564

565 @time_partitioning.setter

566 def time_partitioning(self, value):

567 api_repr = value

568 if value is not None:

569 api_repr = value.to_api_repr()

570 self._set_sub_prop("timePartitioning", api_repr)

571 else:

572 self._del_sub_prop("timePartitioning")

573

574 @property

575 def use_avro_logical_types(self):

576 """Optional[bool]: For loads of Avro data, governs whether Avro logical types are

577 converted to their corresponding BigQuery types (e.g. TIMESTAMP) rather than

578 raw types (e.g. INTEGER).

579 """

580 return self._get_sub_prop("useAvroLogicalTypes")

581

582 @use_avro_logical_types.setter

583 def use_avro_logical_types(self, value):

584 self._set_sub_prop("useAvroLogicalTypes", bool(value))

585

586 @property

587 def write_disposition(self):

588 """Optional[google.cloud.bigquery.job.WriteDisposition]: Action that occurs if

589 the destination table already exists.

590

591 See:

592 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.write_disposition

593 """

594 return self._get_sub_prop("writeDisposition")

595

596 @write_disposition.setter

597 def write_disposition(self, value):

598 self._set_sub_prop("writeDisposition", value)

599

600 @property

601 def parquet_options(self):

602 """Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional

603 properties to set if ``sourceFormat`` is set to PARQUET.

604

605 See:

606 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.parquet_options

607 """

608 prop = self._get_sub_prop("parquetOptions")

609 if prop is not None:

610 prop = ParquetOptions.from_api_repr(prop)

611 return prop

612

613 @parquet_options.setter

614 def parquet_options(self, value):

615 if value is not None:

616 self._set_sub_prop("parquetOptions", value.to_api_repr())

617 else:

618 self._del_sub_prop("parquetOptions")

619

620 @property

621 def column_name_character_map(self) -> str:

622 """Optional[google.cloud.bigquery.job.ColumnNameCharacterMap]:

623 Character map supported for column names in CSV/Parquet loads. Defaults

624 to STRICT and can be overridden by Project Config Service. Using this

625 option with unsupported load formats will result in an error.

626

627 See

628 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.column_name_character_map

629 """

630 return self._get_sub_prop(

631 "columnNameCharacterMap",

632 ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,

633 )

634

635 @column_name_character_map.setter

636 def column_name_character_map(self, value: Optional[str]):

637 if value is None:

638 value = ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED

639 self._set_sub_prop("columnNameCharacterMap", value)

640

641

642class LoadJob(_AsyncJob):

643 """Asynchronous job for loading data into a table.

644

645 Can load from Google Cloud Storage URIs or from a file.

646

647 Args:

648 job_id (str): the job's ID

649

650 source_uris (Optional[Sequence[str]]):

651 URIs of one or more data files to be loaded. See

652 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris

653 for supported URI formats. Pass None for jobs that load from a file.

654

655 destination (google.cloud.bigquery.table.TableReference): reference to table into which data is to be loaded.

656

657 client (google.cloud.bigquery.client.Client):

658 A client which holds credentials and project configuration

659 for the dataset (which requires a project).

660 """

661

662 _JOB_TYPE = "load"

663 _CONFIG_CLASS = LoadJobConfig

664

665 def __init__(self, job_id, source_uris, destination, client, job_config=None):

666 super(LoadJob, self).__init__(job_id, client)

667

668 if job_config is not None:

669 self._properties["configuration"] = job_config._properties

670

671 if source_uris is not None:

672 _helpers._set_sub_prop(

673 self._properties, ["configuration", "load", "sourceUris"], source_uris

674 )

675

676 if destination is not None:

677 _helpers._set_sub_prop(

678 self._properties,

679 ["configuration", "load", "destinationTable"],

680 destination.to_api_repr(),

681 )

682

683 @property

684 def configuration(self) -> LoadJobConfig:

685 """The configuration for this load job."""

686 return typing.cast(LoadJobConfig, super().configuration)

687

688 @property

689 def destination(self):

690 """google.cloud.bigquery.table.TableReference: table where loaded rows are written

691

692 See:

693 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_table

694 """

695 dest_config = _helpers._get_sub_prop(

696 self._properties, ["configuration", "load", "destinationTable"]

697 )

698 return TableReference.from_api_repr(dest_config)

699

700 @property

701 def source_uris(self):

702 """Optional[Sequence[str]]: URIs of data files to be loaded. See

703 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris

704 for supported URI formats. None for jobs that load from a file.

705 """

706 return _helpers._get_sub_prop(

707 self._properties, ["configuration", "load", "sourceUris"]

708 )

709

710 @property

711 def allow_jagged_rows(self):

712 """See

713 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.

714 """

715 return self.configuration.allow_jagged_rows

716

717 @property

718 def allow_quoted_newlines(self):

719 """See

720 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.

721 """

722 return self.configuration.allow_quoted_newlines

723

724 @property

725 def autodetect(self):

726 """See

727 :attr:`google.cloud.bigquery.job.LoadJobConfig.autodetect`.

728 """

729 return self.configuration.autodetect

730

731 @property

732 def connection_properties(self) -> List[ConnectionProperty]:

733 """See

734 :attr:`google.cloud.bigquery.job.LoadJobConfig.connection_properties`.

735

736 .. versionadded:: 3.7.0

737 """

738 return self.configuration.connection_properties

739

740 @property

741 def create_disposition(self):

742 """See

743 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_disposition`.

744 """

745 return self.configuration.create_disposition

746

747 @property

748 def create_session(self) -> Optional[bool]:

749 """See

750 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_session`.

751

752 .. versionadded:: 3.7.0

753 """

754 return self.configuration.create_session

755

756 @property

757 def encoding(self):

758 """See

759 :attr:`google.cloud.bigquery.job.LoadJobConfig.encoding`.

760 """

761 return self.configuration.encoding

762

763 @property

764 def field_delimiter(self):

765 """See

766 :attr:`google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.

767 """

768 return self.configuration.field_delimiter

769

770 @property

771 def ignore_unknown_values(self):

772 """See

773 :attr:`google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.

774 """

775 return self.configuration.ignore_unknown_values

776

777 @property

778 def max_bad_records(self):

779 """See

780 :attr:`google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.

781 """

782 return self.configuration.max_bad_records

783

784 @property

785 def null_marker(self):

786 """See

787 :attr:`google.cloud.bigquery.job.LoadJobConfig.null_marker`.

788 """

789 return self.configuration.null_marker

790

791 @property

792 def quote_character(self):

793 """See

794 :attr:`google.cloud.bigquery.job.LoadJobConfig.quote_character`.

795 """

796 return self.configuration.quote_character

797

798 @property

799 def reference_file_schema_uri(self):

800 """See:

801 attr:`google.cloud.bigquery.job.LoadJobConfig.reference_file_schema_uri`.

802 """

803 return self.configuration.reference_file_schema_uri

804

805 @property

806 def skip_leading_rows(self):

807 """See

808 :attr:`google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.

809 """

810 return self.configuration.skip_leading_rows

811

812 @property

813 def source_format(self):

814 """See

815 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format`.

816 """

817 return self.configuration.source_format

818

819 @property

820 def write_disposition(self):

821 """See

822 :attr:`google.cloud.bigquery.job.LoadJobConfig.write_disposition`.

823 """

824 return self.configuration.write_disposition

825

826 @property

827 def schema(self):

828 """See

829 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema`.

830 """

831 return self.configuration.schema

832

833 @property

834 def destination_encryption_configuration(self):

835 """google.cloud.bigquery.encryption_configuration.EncryptionConfiguration: Custom

836 encryption configuration for the destination table.

837

838 Custom encryption configuration (e.g., Cloud KMS keys)

839 or :data:`None` if using default encryption.

840

841 See

842 :attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.

843 """

844 return self.configuration.destination_encryption_configuration

845

846 @property

847 def destination_table_description(self):

848 """Optional[str] name given to destination table.

849

850 See:

851 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description

852 """

853 return self.configuration.destination_table_description

854

855 @property

856 def destination_table_friendly_name(self):

857 """Optional[str] name given to destination table.

858

859 See:

860 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name

861 """

862 return self.configuration.destination_table_friendly_name

863

864 @property

865 def range_partitioning(self):

866 """See

867 :attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

868 """

869 return self.configuration.range_partitioning

870

871 @property

872 def time_partitioning(self):

873 """See

874 :attr:`google.cloud.bigquery.job.LoadJobConfig.time_partitioning`.

875 """

876 return self.configuration.time_partitioning

877

878 @property

879 def use_avro_logical_types(self):

880 """See

881 :attr:`google.cloud.bigquery.job.LoadJobConfig.use_avro_logical_types`.

882 """

883 return self.configuration.use_avro_logical_types

884

885 @property

886 def clustering_fields(self):

887 """See

888 :attr:`google.cloud.bigquery.job.LoadJobConfig.clustering_fields`.

889 """

890 return self.configuration.clustering_fields

891

892 @property

893 def schema_update_options(self):

894 """See

895 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.

896 """

897 return self.configuration.schema_update_options

898

899 @property

900 def input_file_bytes(self):

901 """Count of bytes loaded from source files.

902

903 Returns:

904 Optional[int]: the count (None until set from the server).

905

906 Raises:

907 ValueError: for invalid value types.

908 """

909 return _helpers._int_or_none(

910 _helpers._get_sub_prop(

911 self._properties, ["statistics", "load", "inputFileBytes"]

912 )

913 )

914

915 @property

916 def input_files(self):

917 """Count of source files.

918

919 Returns:

920 Optional[int]: the count (None until set from the server).

921 """

922 return _helpers._int_or_none(

923 _helpers._get_sub_prop(

924 self._properties, ["statistics", "load", "inputFiles"]

925 )

926 )

927

928 @property

929 def output_bytes(self):

930 """Count of bytes saved to destination table.

931

932 Returns:

933 Optional[int]: the count (None until set from the server).

934 """

935 return _helpers._int_or_none(

936 _helpers._get_sub_prop(

937 self._properties, ["statistics", "load", "outputBytes"]

938 )

939 )

940

941 @property

942 def output_rows(self):

943 """Count of rows saved to destination table.

944

945 Returns:

946 Optional[int]: the count (None until set from the server).

947 """

948 return _helpers._int_or_none(

949 _helpers._get_sub_prop(

950 self._properties, ["statistics", "load", "outputRows"]

951 )

952 )

953

954 def to_api_repr(self):

955 """Generate a resource for :meth:`_begin`."""

956 # Exclude statistics, if set.

957 return {

958 "jobReference": self._properties["jobReference"],

959 "configuration": self._properties["configuration"],

960 }

961

962 @classmethod

963 def from_api_repr(cls, resource: dict, client) -> "LoadJob":

964 """Factory: construct a job given its API representation

965

966 .. note::

967

968 This method assumes that the project found in the resource matches

969 the client's project.

970

971 Args:

972 resource (Dict): dataset job representation returned from the API

973

974 client (google.cloud.bigquery.client.Client):

975 Client which holds credentials and project

976 configuration for the dataset.

977

978 Returns:

979 google.cloud.bigquery.job.LoadJob: Job parsed from ``resource``.

980 """

981 cls._check_resource_config(resource)

982 job_ref = _JobReference._from_api_repr(resource["jobReference"])

983 job = cls(job_ref, None, None, client)

984 job._set_properties(resource)

985 return job