Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/google/cloud/bigquery/job/load.py: 57%

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

15"""Classes for load jobs."""

17import typing

18from typing import FrozenSet, List, Iterable, Optional

20from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration

21from google.cloud.bigquery.external_config import HivePartitioningOptions

22from google.cloud.bigquery.format_options import ParquetOptions

23from google.cloud.bigquery import _helpers

24from google.cloud.bigquery.schema import SchemaField

25from google.cloud.bigquery.schema import _to_schema_fields

26from google.cloud.bigquery.table import RangePartitioning

27from google.cloud.bigquery.table import TableReference

28from google.cloud.bigquery.table import TimePartitioning

29from google.cloud.bigquery.job.base import _AsyncJob

30from google.cloud.bigquery.job.base import _JobConfig

31from google.cloud.bigquery.job.base import _JobReference

32from google.cloud.bigquery.query import ConnectionProperty

35class ColumnNameCharacterMap:

36 """Indicates the character map used for column names.

38 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap

39 """

41 COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED = "COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED"

42 """Unspecified column name character map."""

44 STRICT = "STRICT"

45 """Support flexible column name and reject invalid column names."""

47 V1 = "V1"

48 """ Support alphanumeric + underscore characters and names must start with

49 a letter or underscore. Invalid column names will be normalized."""

51 V2 = "V2"

52 """Support flexible column name. Invalid column names will be normalized."""

55class LoadJobConfig(_JobConfig):

56 """Configuration options for load jobs.

58 Set properties on the constructed configuration by using the property name

59 as the name of a keyword argument. Values which are unset or :data:`None`

60 use the BigQuery REST API default values. See the `BigQuery REST API

61 reference documentation

62 <https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_

63 for a list of default values.

65 Required options differ based on the

66 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.

67 For example, the BigQuery API's default value for

68 :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.

69 When loading a CSV file, either

70 :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or

71 :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to

72 :data:`True`.

73 """

75 def __init__(self, **kwargs) -> None:

76 super(LoadJobConfig, self).__init__("load", **kwargs)

78 @property

79 def allow_jagged_rows(self):

80 """Optional[bool]: Allow missing trailing optional columns (CSV only).

82 See:

83 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_jagged_rows

84 """

85 return self._get_sub_prop("allowJaggedRows")

87 @allow_jagged_rows.setter

88 def allow_jagged_rows(self, value):

89 self._set_sub_prop("allowJaggedRows", value)

91 @property

92 def allow_quoted_newlines(self):

93 """Optional[bool]: Allow quoted data containing newline characters (CSV only).

95 See:

96 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.allow_quoted_newlines

97 """

98 return self._get_sub_prop("allowQuotedNewlines")

100 @allow_quoted_newlines.setter

101 def allow_quoted_newlines(self, value):

102 self._set_sub_prop("allowQuotedNewlines", value)

103

104 @property

105 def autodetect(self):

106 """Optional[bool]: Automatically infer the schema from a sample of the data.

107

108 See:

109 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.autodetect

110 """

111 return self._get_sub_prop("autodetect")

112

113 @autodetect.setter

114 def autodetect(self, value):

115 self._set_sub_prop("autodetect", value)

116

117 @property

118 def clustering_fields(self):

119 """Optional[List[str]]: Fields defining clustering for the table

120

121 (Defaults to :data:`None`).

122

123 Clustering fields are immutable after table creation.

124

125 .. note::

126

127 BigQuery supports clustering for both partitioned and

128 non-partitioned tables.

129 """

130 prop = self._get_sub_prop("clustering")

131 if prop is not None:

132 return list(prop.get("fields", ()))

133

134 @clustering_fields.setter

135 def clustering_fields(self, value):

136 """Optional[List[str]]: Fields defining clustering for the table

137

138 (Defaults to :data:`None`).

139 """

140 if value is not None:

141 self._set_sub_prop("clustering", {"fields": value})

142 else:

143 self._del_sub_prop("clustering")

144

145 @property

146 def connection_properties(self) -> List[ConnectionProperty]:

147 """Connection properties.

148

149 See

150 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.connection_properties

151

152 .. versionadded:: 3.7.0

153 """

154 resource = self._get_sub_prop("connectionProperties", [])

155 return [ConnectionProperty.from_api_repr(prop) for prop in resource]

156

157 @connection_properties.setter

158 def connection_properties(self, value: Iterable[ConnectionProperty]):

159 self._set_sub_prop(

160 "connectionProperties",

161 [prop.to_api_repr() for prop in value],

162 )

163

164 @property

165 def create_disposition(self):

166 """Optional[google.cloud.bigquery.job.CreateDisposition]: Specifies behavior

167 for creating tables.

168

169 See:

170 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition

171 """

172 return self._get_sub_prop("createDisposition")

173

174 @create_disposition.setter

175 def create_disposition(self, value):

176 self._set_sub_prop("createDisposition", value)

177

178 @property

179 def create_session(self) -> Optional[bool]:

180 """[Preview] If :data:`True`, creates a new session, where

181 :attr:`~google.cloud.bigquery.job.LoadJob.session_info` will contain a

182 random server generated session id.

183

184 If :data:`False`, runs load job with an existing ``session_id`` passed in

185 :attr:`~google.cloud.bigquery.job.LoadJobConfig.connection_properties`,

186 otherwise runs load job in non-session mode.

187

188 See

189 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_session

190

191 .. versionadded:: 3.7.0

192 """

193 return self._get_sub_prop("createSession")

194

195 @create_session.setter

196 def create_session(self, value: Optional[bool]):

197 self._set_sub_prop("createSession", value)

198

199 @property

200 def decimal_target_types(self) -> Optional[FrozenSet[str]]:

201 """Possible SQL data types to which the source decimal values are converted.

202

203 See:

204 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types

205

206 .. versionadded:: 2.21.0

207 """

208 prop = self._get_sub_prop("decimalTargetTypes")

209 if prop is not None:

210 prop = frozenset(prop)

211 return prop

212

213 @decimal_target_types.setter

214 def decimal_target_types(self, value: Optional[Iterable[str]]):

215 if value is not None:

216 self._set_sub_prop("decimalTargetTypes", list(value))

217 else:

218 self._del_sub_prop("decimalTargetTypes")

219

220 @property

221 def destination_encryption_configuration(self):

222 """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom

223 encryption configuration for the destination table.

224

225 Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`

226 if using default encryption.

227

228 See:

229 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_encryption_configuration

230 """

231 prop = self._get_sub_prop("destinationEncryptionConfiguration")

232 if prop is not None:

233 prop = EncryptionConfiguration.from_api_repr(prop)

234 return prop

235

236 @destination_encryption_configuration.setter

237 def destination_encryption_configuration(self, value):

238 api_repr = value

239 if value is not None:

240 api_repr = value.to_api_repr()

241 self._set_sub_prop("destinationEncryptionConfiguration", api_repr)

242 else:

243 self._del_sub_prop("destinationEncryptionConfiguration")

244

245 @property

246 def destination_table_description(self):

247 """Optional[str]: Description of the destination table.

248

249 See:

250 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description

251 """

252 prop = self._get_sub_prop("destinationTableProperties")

253 if prop is not None:

254 return prop["description"]

255

256 @destination_table_description.setter

257 def destination_table_description(self, value):

258 keys = [self._job_type, "destinationTableProperties", "description"]

259 if value is not None:

260 _helpers._set_sub_prop(self._properties, keys, value)

261 else:

262 _helpers._del_sub_prop(self._properties, keys)

263

264 @property

265 def destination_table_friendly_name(self):

266 """Optional[str]: Name given to destination table.

267

268 See:

269 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name

270 """

271 prop = self._get_sub_prop("destinationTableProperties")

272 if prop is not None:

273 return prop["friendlyName"]

274

275 @destination_table_friendly_name.setter

276 def destination_table_friendly_name(self, value):

277 keys = [self._job_type, "destinationTableProperties", "friendlyName"]

278 if value is not None:

279 _helpers._set_sub_prop(self._properties, keys, value)

280 else:

281 _helpers._del_sub_prop(self._properties, keys)

282

283 @property

284 def encoding(self):

285 """Optional[google.cloud.bigquery.job.Encoding]: The character encoding of the

286 data.

287

288 See:

289 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.encoding

290 """

291 return self._get_sub_prop("encoding")

292

293 @encoding.setter

294 def encoding(self, value):

295 self._set_sub_prop("encoding", value)

296

297 @property

298 def field_delimiter(self):

299 """Optional[str]: The separator for fields in a CSV file.

300

301 See:

302 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.field_delimiter

303 """

304 return self._get_sub_prop("fieldDelimiter")

305

306 @field_delimiter.setter

307 def field_delimiter(self, value):

308 self._set_sub_prop("fieldDelimiter", value)

309

310 @property

311 def hive_partitioning(self):

312 """Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set, \

313 it configures hive partitioning support.

314

315 .. note::

316 **Experimental**. This feature is experimental and might change or

317 have limited support.

318

319 See:

320 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.hive_partitioning_options

321 """

322 prop = self._get_sub_prop("hivePartitioningOptions")

323 if prop is None:

324 return None

325 return HivePartitioningOptions.from_api_repr(prop)

326

327 @hive_partitioning.setter

328 def hive_partitioning(self, value):

329 if value is not None:

330 if isinstance(value, HivePartitioningOptions):

331 value = value.to_api_repr()

332 else:

333 raise TypeError("Expected a HivePartitioningOptions instance or None.")

334

335 self._set_sub_prop("hivePartitioningOptions", value)

336

337 @property

338 def ignore_unknown_values(self):

339 """Optional[bool]: Ignore extra values not represented in the table schema.

340

341 See:

342 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.ignore_unknown_values

343 """

344 return self._get_sub_prop("ignoreUnknownValues")

345

346 @ignore_unknown_values.setter

347 def ignore_unknown_values(self, value):

348 self._set_sub_prop("ignoreUnknownValues", value)

349

350 @property

351 def json_extension(self):

352 """Optional[str]: The extension to use for writing JSON data to BigQuery. Only supports GeoJSON currently.

353

354 See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.json_extension

355

356 """

357 return self._get_sub_prop("jsonExtension")

358

359 @json_extension.setter

360 def json_extension(self, value):

361 self._set_sub_prop("jsonExtension", value)

362

363 @property

364 def max_bad_records(self):

365 """Optional[int]: Number of invalid rows to ignore.

366

367 See:

368 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.max_bad_records

369 """

370 return _helpers._int_or_none(self._get_sub_prop("maxBadRecords"))

371

372 @max_bad_records.setter

373 def max_bad_records(self, value):

374 self._set_sub_prop("maxBadRecords", value)

375

376 @property

377 def null_marker(self):

378 """Optional[str]: Represents a null value (CSV only).

379

380 See:

381 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_marker

382 """

383 return self._get_sub_prop("nullMarker")

384

385 @null_marker.setter

386 def null_marker(self, value):

387 self._set_sub_prop("nullMarker", value)

388

389 @property

390 def preserve_ascii_control_characters(self):

391 """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.

392

393 See:

394 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.preserve_ascii_control_characters

395 """

396 return self._get_sub_prop("preserveAsciiControlCharacters")

397

398 @preserve_ascii_control_characters.setter

399 def preserve_ascii_control_characters(self, value):

400 self._set_sub_prop("preserveAsciiControlCharacters", bool(value))

401

402 @property

403 def projection_fields(self) -> Optional[List[str]]:

404 """Optional[List[str]]: If

405 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to

406 "DATASTORE_BACKUP", indicates which entity properties to load into

407 BigQuery from a Cloud Datastore backup.

408

409 Property names are case sensitive and must be top-level properties. If

410 no properties are specified, BigQuery loads all properties. If any

411 named property isn't found in the Cloud Datastore backup, an invalid

412 error is returned in the job result.

413

414 See:

415 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields

416 """

417 return self._get_sub_prop("projectionFields")

418

419 @projection_fields.setter

420 def projection_fields(self, value: Optional[List[str]]):

421 self._set_sub_prop("projectionFields", value)

422

423 @property

424 def quote_character(self):

425 """Optional[str]: Character used to quote data sections (CSV only).

426

427 See:

428 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.quote

429 """

430 return self._get_sub_prop("quote")

431

432 @quote_character.setter

433 def quote_character(self, value):

434 self._set_sub_prop("quote", value)

435

436 @property

437 def range_partitioning(self):

438 """Optional[google.cloud.bigquery.table.RangePartitioning]:

439 Configures range-based partitioning for destination table.

440

441 .. note::

442 **Beta**. The integer range partitioning feature is in a

443 pre-release state and might change or have limited support.

444

445 Only specify at most one of

446 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or

447 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

448

449 Raises:

450 ValueError:

451 If the value is not

452 :class:`~google.cloud.bigquery.table.RangePartitioning` or

453 :data:`None`.

454 """

455 resource = self._get_sub_prop("rangePartitioning")

456 if resource is not None:

457 return RangePartitioning(_properties=resource)

458

459 @range_partitioning.setter

460 def range_partitioning(self, value):

461 resource = value

462 if isinstance(value, RangePartitioning):

463 resource = value._properties

464 elif value is not None:

465 raise ValueError(

466 "Expected value to be RangePartitioning or None, got {}.".format(value)

467 )

468 self._set_sub_prop("rangePartitioning", resource)

469

470 @property

471 def reference_file_schema_uri(self):

472 """Optional[str]:

473 When creating an external table, the user can provide a reference file with the

474 table schema. This is enabled for the following formats:

475

476 AVRO, PARQUET, ORC

477 """

478 return self._get_sub_prop("referenceFileSchemaUri")

479

480 @reference_file_schema_uri.setter

481 def reference_file_schema_uri(self, value):

482 return self._set_sub_prop("referenceFileSchemaUri", value)

483

484 @property

485 def schema(self):

486 """Optional[Sequence[Union[ \

487 :class:`~google.cloud.bigquery.schema.SchemaField`, \

488 Mapping[str, Any] \

489 ]]]: Schema of the destination table.

490

491 See:

492 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema

493 """

494 schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])

495 if schema is None:

496 return

497 return [SchemaField.from_api_repr(field) for field in schema]

498

499 @schema.setter

500 def schema(self, value):

501 if value is None:

502 self._del_sub_prop("schema")

503 return

504

505 value = _to_schema_fields(value)

506

507 _helpers._set_sub_prop(

508 self._properties,

509 ["load", "schema", "fields"],

510 [field.to_api_repr() for field in value],

511 )

512

513 @property

514 def schema_update_options(self):

515 """Optional[List[google.cloud.bigquery.job.SchemaUpdateOption]]: Specifies

516 updates to the destination table schema to allow as a side effect of

517 the load job.

518 """

519 return self._get_sub_prop("schemaUpdateOptions")

520

521 @schema_update_options.setter

522 def schema_update_options(self, values):

523 self._set_sub_prop("schemaUpdateOptions", values)

524

525 @property

526 def skip_leading_rows(self):

527 """Optional[int]: Number of rows to skip when reading data (CSV only).

528

529 See:

530 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.skip_leading_rows

531 """

532 return _helpers._int_or_none(self._get_sub_prop("skipLeadingRows"))

533

534 @skip_leading_rows.setter

535 def skip_leading_rows(self, value):

536 self._set_sub_prop("skipLeadingRows", str(value))

537

538 @property

539 def source_format(self):

540 """Optional[google.cloud.bigquery.job.SourceFormat]: File format of the data.

541

542 See:

543 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_format

544 """

545 return self._get_sub_prop("sourceFormat")

546

547 @source_format.setter

548 def source_format(self, value):

549 self._set_sub_prop("sourceFormat", value)

550

551 @property

552 def time_zone(self) -> Optional[str]:

553 """Optional[str]: Default time zone that will apply when parsing timestamp

554 values that have no specific time zone.

555

556 See:

557 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_zone

558 """

559 return self._get_sub_prop("timeZone")

560

561 @time_zone.setter

562 def time_zone(self, value: Optional[str]):

563 self._set_sub_prop("timeZone", value)

564

565 @property

566 def time_partitioning(self):

567 """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based

568 partitioning for the destination table.

569

570 Only specify at most one of

571 :attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or

572 :attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

573 """

574 prop = self._get_sub_prop("timePartitioning")

575 if prop is not None:

576 prop = TimePartitioning.from_api_repr(prop)

577 return prop

578

579 @time_partitioning.setter

580 def time_partitioning(self, value):

581 api_repr = value

582 if value is not None:

583 api_repr = value.to_api_repr()

584 self._set_sub_prop("timePartitioning", api_repr)

585 else:

586 self._del_sub_prop("timePartitioning")

587

588 @property

589 def use_avro_logical_types(self):

590 """Optional[bool]: For loads of Avro data, governs whether Avro logical types are

591 converted to their corresponding BigQuery types (e.g. TIMESTAMP) rather than

592 raw types (e.g. INTEGER).

593 """

594 return self._get_sub_prop("useAvroLogicalTypes")

595

596 @use_avro_logical_types.setter

597 def use_avro_logical_types(self, value):

598 self._set_sub_prop("useAvroLogicalTypes", bool(value))

599

600 @property

601 def write_disposition(self):

602 """Optional[google.cloud.bigquery.job.WriteDisposition]: Action that occurs if

603 the destination table already exists.

604

605 See:

606 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.write_disposition

607 """

608 return self._get_sub_prop("writeDisposition")

609

610 @write_disposition.setter

611 def write_disposition(self, value):

612 self._set_sub_prop("writeDisposition", value)

613

614 @property

615 def parquet_options(self):

616 """Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional

617 properties to set if ``sourceFormat`` is set to PARQUET.

618

619 See:

620 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.parquet_options

621 """

622 prop = self._get_sub_prop("parquetOptions")

623 if prop is not None:

624 prop = ParquetOptions.from_api_repr(prop)

625 return prop

626

627 @parquet_options.setter

628 def parquet_options(self, value):

629 if value is not None:

630 self._set_sub_prop("parquetOptions", value.to_api_repr())

631 else:

632 self._del_sub_prop("parquetOptions")

633

634 @property

635 def column_name_character_map(self) -> str:

636 """Optional[google.cloud.bigquery.job.ColumnNameCharacterMap]:

637 Character map supported for column names in CSV/Parquet loads. Defaults

638 to STRICT and can be overridden by Project Config Service. Using this

639 option with unsupported load formats will result in an error.

640

641 See

642 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.column_name_character_map

643 """

644 return self._get_sub_prop(

645 "columnNameCharacterMap",

646 ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,

647 )

648

649 @column_name_character_map.setter

650 def column_name_character_map(self, value: Optional[str]):

651 if value is None:

652 value = ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED

653 self._set_sub_prop("columnNameCharacterMap", value)

654

655

656class LoadJob(_AsyncJob):

657 """Asynchronous job for loading data into a table.

658

659 Can load from Google Cloud Storage URIs or from a file.

660

661 Args:

662 job_id (str): the job's ID

663

664 source_uris (Optional[Sequence[str]]):

665 URIs of one or more data files to be loaded. See

666 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris

667 for supported URI formats. Pass None for jobs that load from a file.

668

669 destination (google.cloud.bigquery.table.TableReference): reference to table into which data is to be loaded.

670

671 client (google.cloud.bigquery.client.Client):

672 A client which holds credentials and project configuration

673 for the dataset (which requires a project).

674 """

675

676 _JOB_TYPE = "load"

677 _CONFIG_CLASS = LoadJobConfig

678

679 def __init__(self, job_id, source_uris, destination, client, job_config=None):

680 super(LoadJob, self).__init__(job_id, client)

681

682 if job_config is not None:

683 self._properties["configuration"] = job_config._properties

684

685 if source_uris is not None:

686 _helpers._set_sub_prop(

687 self._properties, ["configuration", "load", "sourceUris"], source_uris

688 )

689

690 if destination is not None:

691 _helpers._set_sub_prop(

692 self._properties,

693 ["configuration", "load", "destinationTable"],

694 destination.to_api_repr(),

695 )

696

697 @property

698 def configuration(self) -> LoadJobConfig:

699 """The configuration for this load job."""

700 return typing.cast(LoadJobConfig, super().configuration)

701

702 @property

703 def destination(self):

704 """google.cloud.bigquery.table.TableReference: table where loaded rows are written

705

706 See:

707 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.destination_table

708 """

709 dest_config = _helpers._get_sub_prop(

710 self._properties, ["configuration", "load", "destinationTable"]

711 )

712 return TableReference.from_api_repr(dest_config)

713

714 @property

715 def source_uris(self):

716 """Optional[Sequence[str]]: URIs of data files to be loaded. See

717 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_uris

718 for supported URI formats. None for jobs that load from a file.

719 """

720 return _helpers._get_sub_prop(

721 self._properties, ["configuration", "load", "sourceUris"]

722 )

723

724 @property

725 def allow_jagged_rows(self):

726 """See

727 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.

728 """

729 return self.configuration.allow_jagged_rows

730

731 @property

732 def allow_quoted_newlines(self):

733 """See

734 :attr:`google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.

735 """

736 return self.configuration.allow_quoted_newlines

737

738 @property

739 def autodetect(self):

740 """See

741 :attr:`google.cloud.bigquery.job.LoadJobConfig.autodetect`.

742 """

743 return self.configuration.autodetect

744

745 @property

746 def connection_properties(self) -> List[ConnectionProperty]:

747 """See

748 :attr:`google.cloud.bigquery.job.LoadJobConfig.connection_properties`.

749

750 .. versionadded:: 3.7.0

751 """

752 return self.configuration.connection_properties

753

754 @property

755 def create_disposition(self):

756 """See

757 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_disposition`.

758 """

759 return self.configuration.create_disposition

760

761 @property

762 def create_session(self) -> Optional[bool]:

763 """See

764 :attr:`google.cloud.bigquery.job.LoadJobConfig.create_session`.

765

766 .. versionadded:: 3.7.0

767 """

768 return self.configuration.create_session

769

770 @property

771 def encoding(self):

772 """See

773 :attr:`google.cloud.bigquery.job.LoadJobConfig.encoding`.

774 """

775 return self.configuration.encoding

776

777 @property

778 def field_delimiter(self):

779 """See

780 :attr:`google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.

781 """

782 return self.configuration.field_delimiter

783

784 @property

785 def ignore_unknown_values(self):

786 """See

787 :attr:`google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.

788 """

789 return self.configuration.ignore_unknown_values

790

791 @property

792 def max_bad_records(self):

793 """See

794 :attr:`google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.

795 """

796 return self.configuration.max_bad_records

797

798 @property

799 def null_marker(self):

800 """See

801 :attr:`google.cloud.bigquery.job.LoadJobConfig.null_marker`.

802 """

803 return self.configuration.null_marker

804

805 @property

806 def quote_character(self):

807 """See

808 :attr:`google.cloud.bigquery.job.LoadJobConfig.quote_character`.

809 """

810 return self.configuration.quote_character

811

812 @property

813 def reference_file_schema_uri(self):

814 """See:

815 attr:`google.cloud.bigquery.job.LoadJobConfig.reference_file_schema_uri`.

816 """

817 return self.configuration.reference_file_schema_uri

818

819 @property

820 def skip_leading_rows(self):

821 """See

822 :attr:`google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.

823 """

824 return self.configuration.skip_leading_rows

825

826 @property

827 def source_format(self):

828 """See

829 :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format`.

830 """

831 return self.configuration.source_format

832

833 @property

834 def write_disposition(self):

835 """See

836 :attr:`google.cloud.bigquery.job.LoadJobConfig.write_disposition`.

837 """

838 return self.configuration.write_disposition

839

840 @property

841 def schema(self):

842 """See

843 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema`.

844 """

845 return self.configuration.schema

846

847 @property

848 def destination_encryption_configuration(self):

849 """google.cloud.bigquery.encryption_configuration.EncryptionConfiguration: Custom

850 encryption configuration for the destination table.

851

852 Custom encryption configuration (e.g., Cloud KMS keys)

853 or :data:`None` if using default encryption.

854

855 See

856 :attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.

857 """

858 return self.configuration.destination_encryption_configuration

859

860 @property

861 def destination_table_description(self):

862 """Optional[str] name given to destination table.

863

864 See:

865 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.description

866 """

867 return self.configuration.destination_table_description

868

869 @property

870 def destination_table_friendly_name(self):

871 """Optional[str] name given to destination table.

872

873 See:

874 https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#DestinationTableProperties.FIELDS.friendly_name

875 """

876 return self.configuration.destination_table_friendly_name

877

878 @property

879 def range_partitioning(self):

880 """See

881 :attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

882 """

883 return self.configuration.range_partitioning

884

885 @property

886 def time_partitioning(self):

887 """See

888 :attr:`google.cloud.bigquery.job.LoadJobConfig.time_partitioning`.

889 """

890 return self.configuration.time_partitioning

891

892 @property

893 def use_avro_logical_types(self):

894 """See

895 :attr:`google.cloud.bigquery.job.LoadJobConfig.use_avro_logical_types`.

896 """

897 return self.configuration.use_avro_logical_types

898

899 @property

900 def clustering_fields(self):

901 """See

902 :attr:`google.cloud.bigquery.job.LoadJobConfig.clustering_fields`.

903 """

904 return self.configuration.clustering_fields

905

906 @property

907 def time_zone(self):

908 """See

909 :attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.

910 """

911 return self.configuration.time_zone

912

913 @property

914 def schema_update_options(self):

915 """See

916 :attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.

917 """

918 return self.configuration.schema_update_options

919

920 @property

921 def input_file_bytes(self):

922 """Count of bytes loaded from source files.

923

924 Returns:

925 Optional[int]: the count (None until set from the server).

926

927 Raises:

928 ValueError: for invalid value types.

929 """

930 return _helpers._int_or_none(

931 _helpers._get_sub_prop(

932 self._properties, ["statistics", "load", "inputFileBytes"]

933 )

934 )

935

936 @property

937 def input_files(self):

938 """Count of source files.

939

940 Returns:

941 Optional[int]: the count (None until set from the server).

942 """

943 return _helpers._int_or_none(

944 _helpers._get_sub_prop(

945 self._properties, ["statistics", "load", "inputFiles"]

946 )

947 )

948

949 @property

950 def output_bytes(self):

951 """Count of bytes saved to destination table.

952

953 Returns:

954 Optional[int]: the count (None until set from the server).

955 """

956 return _helpers._int_or_none(

957 _helpers._get_sub_prop(

958 self._properties, ["statistics", "load", "outputBytes"]

959 )

960 )

961

962 @property

963 def output_rows(self):

964 """Count of rows saved to destination table.

965

966 Returns:

967 Optional[int]: the count (None until set from the server).

968 """

969 return _helpers._int_or_none(

970 _helpers._get_sub_prop(

971 self._properties, ["statistics", "load", "outputRows"]

972 )

973 )

974

975 def to_api_repr(self):

976 """Generate a resource for :meth:`_begin`."""

977 # Exclude statistics, if set.

978 return {

979 "jobReference": self._properties["jobReference"],

980 "configuration": self._properties["configuration"],

981 }

982

983 @classmethod

984 def from_api_repr(cls, resource: dict, client) -> "LoadJob":

985 """Factory: construct a job given its API representation

986

987 .. note::

988

989 This method assumes that the project found in the resource matches

990 the client's project.

991

992 Args:

993 resource (Dict): dataset job representation returned from the API

994

995 client (google.cloud.bigquery.client.Client):

996 Client which holds credentials and project

997 configuration for the dataset.

998

999 Returns:

1000 google.cloud.bigquery.job.LoadJob: Job parsed from ``resource``.

1001 """

1002 cls._check_resource_config(resource)

1003 job_ref = _JobReference._from_api_repr(resource["jobReference"])

1004 job = cls(job_ref, None, None, client)

1005 job._set_properties(resource)

1006 return job