Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

15"""Helper functions for Cloud Storage utility classes.

17These are *not* part of the API.

18"""

20import base64

21from hashlib import md5

22import os

23from urllib.parse import urlsplit

24from uuid import uuid4

26from google import resumable_media

27from google.auth import environment_vars

28from google.cloud.storage.constants import _DEFAULT_TIMEOUT

29from google.cloud.storage.retry import DEFAULT_RETRY

30from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED

33STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"

34"""Environment variable defining host for Storage emulator."""

36_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"

37"""This is an experimental configuration variable. Use api_endpoint instead."""

39_API_VERSION_OVERRIDE_ENV_VAR = "API_VERSION_OVERRIDE"

40"""This is an experimental configuration variable used for internal testing."""

42_DEFAULT_STORAGE_HOST = os.getenv(

43 _API_ENDPOINT_OVERRIDE_ENV_VAR, "https://storage.googleapis.com"

44)

45"""Default storage host for JSON API."""

47_API_VERSION = os.getenv(_API_VERSION_OVERRIDE_ENV_VAR, "v1")

48"""API version of the default storage host"""

50# etag match parameters in snake case and equivalent header

51_ETAG_MATCH_PARAMETERS = (

52 ("if_etag_match", "If-Match"),

53 ("if_etag_not_match", "If-None-Match"),

54)

56# generation match parameters in camel and snake cases

57_GENERATION_MATCH_PARAMETERS = (

58 ("if_generation_match", "ifGenerationMatch"),

59 ("if_generation_not_match", "ifGenerationNotMatch"),

60 ("if_metageneration_match", "ifMetagenerationMatch"),

61 ("if_metageneration_not_match", "ifMetagenerationNotMatch"),

62 ("if_source_generation_match", "ifSourceGenerationMatch"),

63 ("if_source_generation_not_match", "ifSourceGenerationNotMatch"),

64 ("if_source_metageneration_match", "ifSourceMetagenerationMatch"),

65 ("if_source_metageneration_not_match", "ifSourceMetagenerationNotMatch"),

66)

68_NUM_RETRIES_MESSAGE = (

69 "`num_retries` has been deprecated and will be removed in a future "

70 "release. Use the `retry` argument with a Retry or ConditionalRetryPolicy "

71 "object, or None, instead."

72)

75def _get_storage_host():

76 return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)

79def _get_environ_project():

80 return os.getenv(

81 environment_vars.PROJECT,

82 os.getenv(environment_vars.LEGACY_PROJECT),

83 )

86def _validate_name(name):

87 """Pre-flight ``Bucket`` name validation.

89 :type name: str or :data:`NoneType`

90 :param name: Proposed bucket name.

92 :rtype: str or :data:`NoneType`

93 :returns: ``name`` if valid.

94 """

95 if name is None:

96 return

98 # The first and last characters must be alphanumeric.

99 if not all([name[0].isalnum(), name[-1].isalnum()]):

100 raise ValueError("Bucket names must start and end with a number or letter.")

101 return name

102

103

104class _PropertyMixin(object):

105 """Abstract mixin for cloud storage classes with associated properties.

106

107 Non-abstract subclasses should implement:

108 - path

109 - client

110 - user_project

111

112 :type name: str

113 :param name: The name of the object. Bucket names must start and end with a

114 number or letter.

115 """

116

117 def __init__(self, name=None):

118 self.name = name

119 self._properties = {}

120 self._changes = set()

121

122 @property

123 def path(self):

124 """Abstract getter for the object path."""

125 raise NotImplementedError

126

127 @property

128 def client(self):

129 """Abstract getter for the object client."""

130 raise NotImplementedError

131

132 @property

133 def user_project(self):

134 """Abstract getter for the object user_project."""

135 raise NotImplementedError

136

137 def _require_client(self, client):

138 """Check client or verify over-ride.

139

140 :type client: :class:`~google.cloud.storage.client.Client` or

141 ``NoneType``

142 :param client: the client to use. If not passed, falls back to the

143 ``client`` stored on the current object.

144

145 :rtype: :class:`google.cloud.storage.client.Client`

146 :returns: The client passed in or the currently bound client.

147 """

148 if client is None:

149 client = self.client

150 return client

151

152 def _encryption_headers(self):

153 """Return any encryption headers needed to fetch the object.

154

155 .. note::

156 Defined here because :meth:`reload` calls it, but this method is

157 really only relevant for :class:`~google.cloud.storage.blob.Blob`.

158

159 :rtype: dict

160 :returns: a mapping of encryption-related headers.

161 """

162 return {}

163

164 @property

165 def _query_params(self):

166 """Default query parameters."""

167 params = {}

168 if self.user_project is not None:

169 params["userProject"] = self.user_project

170 return params

171

172 def reload(

173 self,

174 client=None,

175 projection="noAcl",

176 if_etag_match=None,

177 if_etag_not_match=None,

178 if_generation_match=None,

179 if_generation_not_match=None,

180 if_metageneration_match=None,

181 if_metageneration_not_match=None,

182 timeout=_DEFAULT_TIMEOUT,

183 retry=DEFAULT_RETRY,

184 ):

185 """Reload properties from Cloud Storage.

186

187 If :attr:`user_project` is set, bills the API request to that project.

188

189 :type client: :class:`~google.cloud.storage.client.Client` or

190 ``NoneType``

191 :param client: the client to use. If not passed, falls back to the

192 ``client`` stored on the current object.

193

194 :type projection: str

195 :param projection: (Optional) If used, must be 'full' or 'noAcl'.

196 Defaults to ``'noAcl'``. Specifies the set of

197 properties to return.

198

199 :type if_etag_match: Union[str, Set[str]]

200 :param if_etag_match: (Optional) See :ref:`using-if-etag-match`

201

202 :type if_etag_not_match: Union[str, Set[str]])

203 :param if_etag_not_match: (Optional) See :ref:`using-if-etag-not-match`

204

205 :type if_generation_match: long

206 :param if_generation_match:

207 (Optional) See :ref:`using-if-generation-match`

208

209 :type if_generation_not_match: long

210 :param if_generation_not_match:

211 (Optional) See :ref:`using-if-generation-not-match`

212

213 :type if_metageneration_match: long

214 :param if_metageneration_match:

215 (Optional) See :ref:`using-if-metageneration-match`

216

217 :type if_metageneration_not_match: long

218 :param if_metageneration_not_match:

219 (Optional) See :ref:`using-if-metageneration-not-match`

220

221 :type timeout: float or tuple

222 :param timeout:

223 (Optional) The amount of time, in seconds, to wait

224 for the server response. See: :ref:`configuring_timeouts`

225

226 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

227 :param retry:

228 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

229 """

230 client = self._require_client(client)

231 query_params = self._query_params

232 # Pass only '?projection=noAcl' here because 'acl' and related

233 # are handled via custom endpoints.

234 query_params["projection"] = projection

235 _add_generation_match_parameters(

236 query_params,

237 if_generation_match=if_generation_match,

238 if_generation_not_match=if_generation_not_match,

239 if_metageneration_match=if_metageneration_match,

240 if_metageneration_not_match=if_metageneration_not_match,

241 )

242 headers = self._encryption_headers()

243 _add_etag_match_headers(

244 headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match

245 )

246 api_response = client._get_resource(

247 self.path,

248 query_params=query_params,

249 headers=headers,

250 timeout=timeout,

251 retry=retry,

252 _target_object=self,

253 )

254 self._set_properties(api_response)

255

256 def _patch_property(self, name, value):

257 """Update field of this object's properties.

258

259 This method will only update the field provided and will not

260 touch the other fields.

261

262 It **will not** reload the properties from the server. The behavior is

263 local only and syncing occurs via :meth:`patch`.

264

265 :type name: str

266 :param name: The field name to update.

267

268 :type value: object

269 :param value: The value being updated.

270 """

271 self._changes.add(name)

272 self._properties[name] = value

273

274 def _set_properties(self, value):

275 """Set the properties for the current object.

276

277 :type value: dict or :class:`google.cloud.storage.batch._FutureDict`

278 :param value: The properties to be set.

279 """

280 self._properties = value

281 # If the values are reset, the changes must as well.

282 self._changes = set()

283

284 def patch(

285 self,

286 client=None,

287 if_generation_match=None,

288 if_generation_not_match=None,

289 if_metageneration_match=None,

290 if_metageneration_not_match=None,

291 timeout=_DEFAULT_TIMEOUT,

292 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,

293 ):

294 """Sends all changed properties in a PATCH request.

295

296 Updates the ``_properties`` with the response from the backend.

297

298 If :attr:`user_project` is set, bills the API request to that project.

299

300 :type client: :class:`~google.cloud.storage.client.Client` or

301 ``NoneType``

302 :param client: the client to use. If not passed, falls back to the

303 ``client`` stored on the current object.

304

305 :type if_generation_match: long

306 :param if_generation_match:

307 (Optional) See :ref:`using-if-generation-match`

308

309 :type if_generation_not_match: long

310 :param if_generation_not_match:

311 (Optional) See :ref:`using-if-generation-not-match`

312

313 :type if_metageneration_match: long

314 :param if_metageneration_match:

315 (Optional) See :ref:`using-if-metageneration-match`

316

317 :type if_metageneration_not_match: long

318 :param if_metageneration_not_match:

319 (Optional) See :ref:`using-if-metageneration-not-match`

320

321 :type timeout: float or tuple

322 :param timeout:

323 (Optional) The amount of time, in seconds, to wait

324 for the server response. See: :ref:`configuring_timeouts`

325

326 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

327 :param retry:

328 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

329 """

330 client = self._require_client(client)

331 query_params = self._query_params

332 # Pass '?projection=full' here because 'PATCH' documented not

333 # to work properly w/ 'noAcl'.

334 query_params["projection"] = "full"

335 _add_generation_match_parameters(

336 query_params,

337 if_generation_match=if_generation_match,

338 if_generation_not_match=if_generation_not_match,

339 if_metageneration_match=if_metageneration_match,

340 if_metageneration_not_match=if_metageneration_not_match,

341 )

342 update_properties = {key: self._properties[key] for key in self._changes}

343

344 # Make the API call.

345 api_response = client._patch_resource(

346 self.path,

347 update_properties,

348 query_params=query_params,

349 _target_object=self,

350 timeout=timeout,

351 retry=retry,

352 )

353 self._set_properties(api_response)

354

355 def update(

356 self,

357 client=None,

358 if_generation_match=None,

359 if_generation_not_match=None,

360 if_metageneration_match=None,

361 if_metageneration_not_match=None,

362 timeout=_DEFAULT_TIMEOUT,

363 retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,

364 ):

365 """Sends all properties in a PUT request.

366

367 Updates the ``_properties`` with the response from the backend.

368

369 If :attr:`user_project` is set, bills the API request to that project.

370

371 :type client: :class:`~google.cloud.storage.client.Client` or

372 ``NoneType``

373 :param client: the client to use. If not passed, falls back to the

374 ``client`` stored on the current object.

375

376 :type if_generation_match: long

377 :param if_generation_match:

378 (Optional) See :ref:`using-if-generation-match`

379

380 :type if_generation_not_match: long

381 :param if_generation_not_match:

382 (Optional) See :ref:`using-if-generation-not-match`

383

384 :type if_metageneration_match: long

385 :param if_metageneration_match:

386 (Optional) See :ref:`using-if-metageneration-match`

387

388 :type if_metageneration_not_match: long

389 :param if_metageneration_not_match:

390 (Optional) See :ref:`using-if-metageneration-not-match`

391

392 :type timeout: float or tuple

393 :param timeout:

394 (Optional) The amount of time, in seconds, to wait

395 for the server response. See: :ref:`configuring_timeouts`

396

397 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy

398 :param retry:

399 (Optional) How to retry the RPC. See: :ref:`configuring_retries`

400 """

401 client = self._require_client(client)

402

403 query_params = self._query_params

404 query_params["projection"] = "full"

405 _add_generation_match_parameters(

406 query_params,

407 if_generation_match=if_generation_match,

408 if_generation_not_match=if_generation_not_match,

409 if_metageneration_match=if_metageneration_match,

410 if_metageneration_not_match=if_metageneration_not_match,

411 )

412

413 api_response = client._put_resource(

414 self.path,

415 self._properties,

416 query_params=query_params,

417 timeout=timeout,

418 retry=retry,

419 _target_object=self,

420 )

421 self._set_properties(api_response)

422

423

424def _scalar_property(fieldname):

425 """Create a property descriptor around the :class:`_PropertyMixin` helpers."""

426

427 def _getter(self):

428 """Scalar property getter."""

429 return self._properties.get(fieldname)

430

431 def _setter(self, value):

432 """Scalar property setter."""

433 self._patch_property(fieldname, value)

434

435 return property(_getter, _setter)

436

437

438def _write_buffer_to_hash(buffer_object, hash_obj, digest_block_size=8192):

439 """Read blocks from a buffer and update a hash with them.

440

441 :type buffer_object: bytes buffer

442 :param buffer_object: Buffer containing bytes used to update a hash object.

443

444 :type hash_obj: object that implements update

445 :param hash_obj: A hash object (MD5 or CRC32-C).

446

447 :type digest_block_size: int

448 :param digest_block_size: The block size to write to the hash.

449 Defaults to 8192.

450 """

451 block = buffer_object.read(digest_block_size)

452

453 while len(block) > 0:

454 hash_obj.update(block)

455 # Update the block for the next iteration.

456 block = buffer_object.read(digest_block_size)

457

458

459def _base64_md5hash(buffer_object):

460 """Get MD5 hash of bytes (as base64).

461

462 :type buffer_object: bytes buffer

463 :param buffer_object: Buffer containing bytes used to compute an MD5

464 hash (as base64).

465

466 :rtype: str

467 :returns: A base64 encoded digest of the MD5 hash.

468 """

469 hash_obj = md5()

470 _write_buffer_to_hash(buffer_object, hash_obj)

471 digest_bytes = hash_obj.digest()

472 return base64.b64encode(digest_bytes)

473

474

475def _add_etag_match_headers(headers, **match_parameters):

476 """Add generation match parameters into the given parameters list.

477

478 :type headers: dict

479 :param headers: Headers dict.

480

481 :type match_parameters: dict

482 :param match_parameters: if*etag*match parameters to add.

483 """

484 for snakecase_name, header_name in _ETAG_MATCH_PARAMETERS:

485 value = match_parameters.get(snakecase_name)

486

487 if value is not None:

488 if isinstance(value, str):

489 value = [value]

490 headers[header_name] = ", ".join(value)

491

492

493def _add_generation_match_parameters(parameters, **match_parameters):

494 """Add generation match parameters into the given parameters list.

495

496 :type parameters: list or dict

497 :param parameters: Parameters list or dict.

498

499 :type match_parameters: dict

500 :param match_parameters: if*generation*match parameters to add.

501

502 :raises: :exc:`ValueError` if ``parameters`` is not a ``list()``

503 or a ``dict()``.

504 """

505 for snakecase_name, camelcase_name in _GENERATION_MATCH_PARAMETERS:

506 value = match_parameters.get(snakecase_name)

507

508 if value is not None:

509 if isinstance(parameters, list):

510 parameters.append((camelcase_name, value))

511

512 elif isinstance(parameters, dict):

513 parameters[camelcase_name] = value

514

515 else:

516 raise ValueError(

517 "`parameters` argument should be a dict() or a list()."

518 )

519

520

521def _raise_if_more_than_one_set(**kwargs):

522 """Raise ``ValueError`` exception if more than one parameter was set.

523

524 :type error: :exc:`ValueError`

525 :param error: Description of which fields were set

526

527 :raises: :class:`~ValueError` containing the fields that were set

528 """

529 if sum(arg is not None for arg in kwargs.values()) > 1:

530 escaped_keys = [f"'{name}'" for name in kwargs.keys()]

531

532 keys_but_last = ", ".join(escaped_keys[:-1])

533 last_key = escaped_keys[-1]

534

535 msg = f"Pass at most one of {keys_but_last} and {last_key}"

536

537 raise ValueError(msg)

538

539

540def _bucket_bound_hostname_url(host, scheme=None):

541 """Helper to build bucket bound hostname URL.

542

543 :type host: str

544 :param host: Host name.

545

546 :type scheme: str

547 :param scheme: (Optional) Web scheme. If passed, use it

548 as a scheme in the result URL.

549

550 :rtype: str

551 :returns: A bucket bound hostname URL.

552 """

553 url_parts = urlsplit(host)

554 if url_parts.scheme and url_parts.netloc:

555 return host

556

557 return f"{scheme}://{host}"

558

559

560def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):

561 """Convert google.api.core.Retry to google.resumable_media.RetryStrategy.

562

563 Custom predicates are not translated.

564

565 :type retry: google.api_core.Retry

566 :param retry: (Optional) The google.api_core.Retry object to translate.

567

568 :type num_retries: int

569 :param num_retries: (Optional) The number of retries desired. This is

570 supported for backwards compatibility and is mutually exclusive with

571 `retry`.

572

573 :rtype: google.resumable_media.RetryStrategy

574 :returns: A RetryStrategy with all applicable attributes copied from input,

575 or a RetryStrategy with max_retries set to 0 if None was input.

576 """

577

578 if retry is not None and num_retries is not None:

579 raise ValueError("num_retries and retry arguments are mutually exclusive")

580

581 elif retry is not None:

582 return resumable_media.RetryStrategy(

583 max_sleep=retry._maximum,

584 max_cumulative_retry=retry._deadline,

585 initial_delay=retry._initial,

586 multiplier=retry._multiplier,

587 )

588 elif num_retries is not None:

589 return resumable_media.RetryStrategy(max_retries=num_retries)

590 else:

591 return resumable_media.RetryStrategy(max_retries=0)

592

593

594def _get_invocation_id():

595 return "gccl-invocation-id/" + str(uuid4())

596

597

598def _get_default_headers(

599 user_agent,

600 content_type="application/json; charset=UTF-8",

601 x_upload_content_type=None,

602):

603 """Get the headers for a request.

604

605 Args:

606 user_agent (str): The user-agent for requests.

607 Returns:

608 Dict: The headers to be used for the request.

609 """

610 return {

611 "Accept": "application/json",

612 "Accept-Encoding": "gzip, deflate",

613 "User-Agent": user_agent,

614 "X-Goog-API-Client": f"{user_agent} {_get_invocation_id()}",

615 "content-type": content_type,

616 "x-upload-content-type": x_upload_content_type or content_type,

617 }

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/google/cloud/storage/_helpers.py: 37%

141 statements