Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/botocore/handlers.py: 22%

1# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"). You 

4# may not use this file except in compliance with the License. A copy of 

5# the License is located at 

6# 

7# http://aws.amazon.com/apache2.0/ 

8# 

9# or in the "license" file accompanying this file. This file is 

10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 

11# ANY KIND, either express or implied. See the License for the specific 

12# language governing permissions and limitations under the License. 

13 

14"""Builtin event handlers. 

15 

16This module contains builtin handlers for events emitted by botocore. 

17""" 

18 

19import base64 

20import copy 

21import logging 

22import os 

23import re 

24import uuid 

25import warnings 

26from io import BytesIO 

27 

28import botocore 

29import botocore.auth 

30from botocore import ( 

31 retryhandler, # noqa: F401 

32 translate, # noqa: F401 

33 utils, 

34) 

35from botocore.args import ClientConfigString 

36from botocore.compat import ( 

37 MD5_AVAILABLE, # noqa: F401 

38 ETree, 

39 OrderedDict, 

40 XMLParseError, 

41 ensure_bytes, 

42 get_md5, 

43 json, 

44 quote, 

45 unquote, 

46 unquote_str, 

47 urlsplit, 

48 urlunsplit, 

49) 

50from botocore.docs.utils import ( 

51 AppendParamDocumentation, 

52 AutoPopulatedParam, 

53 HideParamFromOperations, 

54) 

55from botocore.endpoint_provider import VALID_HOST_LABEL_RE 

56from botocore.exceptions import ( 

57 AliasConflictParameterError, 

58 MissingServiceIdError, # noqa: F401 

59 ParamValidationError, 

60 UnsupportedTLSVersionWarning, 

61) 

62from botocore.regions import EndpointResolverBuiltins 

63from botocore.signers import ( 

64 add_dsql_generate_db_auth_token_methods, 

65 add_generate_db_auth_token, 

66 add_generate_presigned_post, 

67 add_generate_presigned_url, 

68) 

69from botocore.useragent import register_feature_id 

70from botocore.utils import ( 

71 SAFE_CHARS, 

72 SERVICE_NAME_ALIASES, # noqa: F401 

73 ArnParser, 

74 get_token_from_environment, 

75 hyphenize_service_id, # noqa: F401 

76 is_global_accesspoint, # noqa: F401 

77 percent_encode, 

78 switch_host_with_param, 

79) 

80 

81logger = logging.getLogger(__name__) 

82 

83REGISTER_FIRST = object() 

84REGISTER_LAST = object() 

85# From the S3 docs: 

86# The rules for bucket names in the US Standard region allow bucket names 

87# to be as long as 255 characters, and bucket names can contain any 

88# combination of uppercase letters, lowercase letters, numbers, periods 

89# (.), hyphens (-), and underscores (_). 

90VALID_BUCKET = re.compile(r'^[a-zA-Z0-9.\-_]{1,255}$') 

91_ACCESSPOINT_ARN = ( 

92 r'^arn:(aws).*:(s3|s3-object-lambda):[a-z\-0-9]*:[0-9]{12}:accesspoint[/:]' 

93 r'[a-zA-Z0-9\-.]{1,63}$' 

94) 

95_OUTPOST_ARN = ( 

96 r'^arn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]' 

97 r'[a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63}$' 

98) 

99VALID_S3_ARN = re.compile('|'.join([_ACCESSPOINT_ARN, _OUTPOST_ARN])) 

100# signing names used for the services s3 and s3-control, for example in 

101# botocore/data/s3/2006-03-01/endpoints-rule-set-1.json 

102S3_SIGNING_NAMES = ('s3', 's3-outposts', 's3-object-lambda', 's3express') 

103VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$') 

104 

105 

106def handle_service_name_alias(service_name, **kwargs): 

107 return SERVICE_NAME_ALIASES.get(service_name, service_name) 

108 

109 

110def add_recursion_detection_header(params, **kwargs): 

111 has_lambda_name = 'AWS_LAMBDA_FUNCTION_NAME' in os.environ 

112 trace_id = os.environ.get('_X_AMZN_TRACE_ID') 

113 if has_lambda_name and trace_id: 

114 headers = params['headers'] 

115 if 'X-Amzn-Trace-Id' not in headers: 

116 headers['X-Amzn-Trace-Id'] = quote(trace_id, safe='-=;:+&[]{}"\',') 

117 

118 

119def escape_xml_payload(params, **kwargs): 

120 # Replace \r and \n with the escaped sequence over the whole XML document 

121 # to avoid linebreak normalization modifying customer input when the 

122 # document is parsed. Ideally, we would do this in ElementTree.tostring, 

123 # but it doesn't allow us to override entity escaping for text fields. For 

124 # this operation \r and \n can only appear in the XML document if they were 

125 # passed as part of the customer input. 

126 body = params['body'] 

127 if b'\r' in body: 

128 body = body.replace(b'\r', b'
') 

129 if b'\n' in body: 

130 body = body.replace(b'\n', b'
') 

131 

132 params['body'] = body 

133 

134 

135def check_for_200_error(response, **kwargs): 

136 """This function has been deprecated, but is kept for backwards compatibility.""" 

137 # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html 

138 # There are two opportunities for a copy request to return an error. One 

139 # can occur when Amazon S3 receives the copy request and the other can 

140 # occur while Amazon S3 is copying the files. If the error occurs before 

141 # the copy operation starts, you receive a standard Amazon S3 error. If the 

142 # error occurs during the copy operation, the error response is embedded in 

143 # the 200 OK response. This means that a 200 OK response can contain either 

144 # a success or an error. Make sure to design your application to parse the 

145 # contents of the response and handle it appropriately. 

146 # 

147 # So this handler checks for this case. Even though the server sends a 

148 # 200 response, conceptually this should be handled exactly like a 

149 # 500 response (with respect to raising exceptions, retries, etc.) 

150 # We're connected *before* all the other retry logic handlers, so as long 

151 # as we switch the error code to 500, we'll retry the error as expected. 

152 if response is None: 

153 # A None response can happen if an exception is raised while 

154 # trying to retrieve the response. See Endpoint._get_response(). 

155 return 

156 http_response, parsed = response 

157 if _looks_like_special_case_error( 

158 http_response.status_code, http_response.content 

159 ): 

160 logger.debug( 

161 "Error found for response with 200 status code, " 

162 "errors: %s, changing status code to " 

163 "500.", 

164 parsed, 

165 ) 

166 http_response.status_code = 500 

167 

168 

169def _looks_like_special_case_error(status_code, body): 

170 if status_code == 200 and body: 

171 try: 

172 parser = ETree.XMLParser( 

173 target=ETree.TreeBuilder(), encoding='utf-8' 

174 ) 

175 parser.feed(body) 

176 root = parser.close() 

177 except XMLParseError: 

178 # In cases of network disruptions, we may end up with a partial 

179 # streamed response from S3. We need to treat these cases as 

180 # 500 Service Errors and try again. 

181 return True 

182 if root.tag == 'Error': 

183 return True 

184 return False 

185 

186 

187def set_operation_specific_signer(context, signing_name, **kwargs): 

188 """Choose the operation-specific signer. 

189 

190 Individual operations may have a different auth type than the service as a 

191 whole. This will most often manifest as operations that should not be 

192 authenticated at all, but can include other auth modes such as sigv4 

193 without body signing. 

194 """ 

195 auth_type = context.get('auth_type') 

196 

197 # Auth type will be None if the operation doesn't have a configured auth 

198 # type. 

199 if not auth_type: 

200 return 

201 

202 # Auth type will be the string value 'none' if the operation should not 

203 # be signed at all. 

204 if auth_type == 'none': 

205 return botocore.UNSIGNED 

206 

207 if auth_type == 'bearer': 

208 return 'bearer' 

209 

210 # If the operation needs an unsigned body, we set additional context 

211 # allowing the signer to be aware of this. 

212 if context.get('unsigned_payload') or auth_type == 'v4-unsigned-body': 

213 context['payload_signing_enabled'] = False 

214 

215 if auth_type.startswith('v4'): 

216 if auth_type == 'v4-s3express': 

217 return auth_type 

218 

219 if auth_type == 'v4a': 

220 # If sigv4a is chosen, we must add additional signing config for 

221 # global signature. 

222 region = _resolve_sigv4a_region(context) 

223 signing = {'region': region, 'signing_name': signing_name} 

224 if 'signing' in context: 

225 context['signing'].update(signing) 

226 else: 

227 context['signing'] = signing 

228 signature_version = 'v4a' 

229 else: 

230 signature_version = 'v4' 

231 

232 # Signing names used by s3 and s3-control use customized signers "s3v4" 

233 # and "s3v4a". 

234 if signing_name in S3_SIGNING_NAMES: 

235 signature_version = f's3{signature_version}' 

236 

237 return signature_version 

238 

239 

240def _handle_sqs_compatible_error(parsed, context, **kwargs): 

241 """ 

242 Ensures backward compatibility for SQS errors. 

243 

244 SQS's migration from the Query protocol to JSON was done prior to SDKs allowing a 

245 service to support multiple protocols. Because of this, SQS is missing the "error" 

246 key from its modeled exceptions, which is used by most query compatible services 

247 to map error codes to the proper exception. Instead, SQS uses the error's shape name, 

248 which is preserved in the QueryErrorCode key. 

249 """ 

250 parsed_error = parsed.get("Error", {}) 

251 if not parsed_error: 

252 return 

253 

254 if query_code := parsed_error.get("QueryErrorCode"): 

255 context['error_code_override'] = query_code 

256 

257 

258def _resolve_sigv4a_region(context): 

259 region = None 

260 if 'client_config' in context: 

261 region = context['client_config'].sigv4a_signing_region_set 

262 if not region and context.get('signing', {}).get('region'): 

263 region = context['signing']['region'] 

264 return region or '*' 

265 

266 

267def decode_console_output(parsed, **kwargs): 

268 if 'Output' in parsed: 

269 try: 

270 # We're using 'replace' for errors because it is 

271 # possible that console output contains non string 

272 # chars we can't utf-8 decode. 

273 value = base64.b64decode( 

274 bytes(parsed['Output'], 'latin-1') 

275 ).decode('utf-8', 'replace') 

276 parsed['Output'] = value 

277 except (ValueError, TypeError, AttributeError): 

278 logger.debug('Error decoding base64', exc_info=True) 

279 

280 

281def generate_idempotent_uuid(params, model, **kwargs): 

282 for name in model.idempotent_members: 

283 if name not in params: 

284 params[name] = str(uuid.uuid4()) 

285 logger.debug( 

286 "injecting idempotency token (%s) into param '%s'.", 

287 params[name], 

288 name, 

289 ) 

290 

291 

292def decode_quoted_jsondoc(value): 

293 try: 

294 value = json.loads(unquote(value)) 

295 except (ValueError, TypeError): 

296 logger.debug('Error loading quoted JSON', exc_info=True) 

297 return value 

298 

299 

300def json_decode_template_body(parsed, **kwargs): 

301 if 'TemplateBody' in parsed: 

302 try: 

303 value = json.loads( 

304 parsed['TemplateBody'], object_pairs_hook=OrderedDict 

305 ) 

306 parsed['TemplateBody'] = value 

307 except (ValueError, TypeError): 

308 logger.debug('error loading JSON', exc_info=True) 

309 

310 

311def validate_bucket_name(params, **kwargs): 

312 if 'Bucket' not in params: 

313 return 

314 bucket = params['Bucket'] 

315 if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket): 

316 error_msg = ( 

317 f'Invalid bucket name "{bucket}": Bucket name must match ' 

318 f'the regex "{VALID_BUCKET.pattern}" or be an ARN matching ' 

319 f'the regex "{VALID_S3_ARN.pattern}"' 

320 ) 

321 raise ParamValidationError(report=error_msg) 

322 

323 

324def sse_md5(params, **kwargs): 

325 """ 

326 S3 server-side encryption requires the encryption key to be sent to the 

327 server base64 encoded, as well as a base64-encoded MD5 hash of the 

328 encryption key. This handler does both if the MD5 has not been set by 

329 the caller. 

330 """ 

331 _sse_md5(params, 'SSECustomer') 

332 

333 

334def copy_source_sse_md5(params, **kwargs): 

335 """ 

336 S3 server-side encryption requires the encryption key to be sent to the 

337 server base64 encoded, as well as a base64-encoded MD5 hash of the 

338 encryption key. This handler does both if the MD5 has not been set by 

339 the caller specifically if the parameter is for the copy-source sse-c key. 

340 """ 

341 _sse_md5(params, 'CopySourceSSECustomer') 

342 

343 

344def _sse_md5(params, sse_member_prefix='SSECustomer'): 

345 if not _needs_s3_sse_customization(params, sse_member_prefix): 

346 return 

347 

348 sse_key_member = sse_member_prefix + 'Key' 

349 sse_md5_member = sse_member_prefix + 'KeyMD5' 

350 key_as_bytes = params[sse_key_member] 

351 if isinstance(key_as_bytes, str): 

352 key_as_bytes = key_as_bytes.encode('utf-8') 

353 md5_val = get_md5(key_as_bytes, usedforsecurity=False).digest() 

354 key_md5_str = base64.b64encode(md5_val).decode('utf-8') 

355 key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8') 

356 params[sse_key_member] = key_b64_encoded 

357 params[sse_md5_member] = key_md5_str 

358 

359 

360def _needs_s3_sse_customization(params, sse_member_prefix): 

361 return ( 

362 params.get(sse_member_prefix + 'Key') is not None 

363 and sse_member_prefix + 'KeyMD5' not in params 

364 ) 

365 

366 

367def disable_signing(**kwargs): 

368 """ 

369 This handler disables request signing by setting the signer 

370 name to a special sentinel value. 

371 """ 

372 return botocore.UNSIGNED 

373 

374 

375def add_expect_header(model, params, **kwargs): 

376 if model.http.get('method', '') not in ['PUT', 'POST']: 

377 return 

378 if 'body' in params: 

379 body = params['body'] 

380 if hasattr(body, 'read'): 

381 check_body = utils.ensure_boolean( 

382 os.environ.get( 

383 'BOTO_EXPERIMENTAL__NO_EMPTY_CONTINUE', 

384 False, 

385 ) 

386 ) 

387 if check_body and utils.determine_content_length(body) == 0: 

388 return 

389 # Any file like object will use an expect 100-continue 

390 # header regardless of size. 

391 logger.debug("Adding expect 100 continue header to request.") 

392 params['headers']['Expect'] = '100-continue' 

393 

394 

395class DeprecatedServiceDocumenter: 

396 def __init__(self, replacement_service_name): 

397 self._replacement_service_name = replacement_service_name 

398 

399 def inject_deprecation_notice(self, section, event_name, **kwargs): 

400 section.style.start_important() 

401 section.write('This service client is deprecated. Please use ') 

402 section.style.ref( 

403 self._replacement_service_name, 

404 self._replacement_service_name, 

405 ) 

406 section.write(' instead.') 

407 section.style.end_important() 

408 

409 

410def document_copy_source_form(section, event_name, **kwargs): 

411 if 'request-example' in event_name: 

412 parent = section.get_section('structure-value') 

413 param_line = parent.get_section('CopySource') 

414 value_portion = param_line.get_section('member-value') 

415 value_portion.clear_text() 

416 value_portion.write( 

417 "'string' or {'Bucket': 'string', " 

418 "'Key': 'string', 'VersionId': 'string'}" 

419 ) 

420 elif 'request-params' in event_name: 

421 param_section = section.get_section('CopySource') 

422 type_section = param_section.get_section('param-type') 

423 type_section.clear_text() 

424 type_section.write(':type CopySource: str or dict') 

425 doc_section = param_section.get_section('param-documentation') 

426 doc_section.clear_text() 

427 doc_section.write( 

428 "The name of the source bucket, key name of the source object, " 

429 "and optional version ID of the source object. You can either " 

430 "provide this value as a string or a dictionary. The " 

431 "string form is {bucket}/{key} or " 

432 "{bucket}/{key}?versionId={versionId} if you want to copy a " 

433 "specific version. You can also provide this value as a " 

434 "dictionary. The dictionary format is recommended over " 

435 "the string format because it is more explicit. The dictionary " 

436 "format is: {'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}." 

437 " Note that the VersionId key is optional and may be omitted." 

438 " To specify an S3 access point, provide the access point" 

439 " ARN for the ``Bucket`` key in the copy source dictionary. If you" 

440 " want to provide the copy source for an S3 access point as a" 

441 " string instead of a dictionary, the ARN provided must be the" 

442 " full S3 access point object ARN" 

443 " (i.e. {accesspoint_arn}/object/{key})" 

444 ) 

445 

446 

447def handle_copy_source_param(params, **kwargs): 

448 """Convert CopySource param for CopyObject/UploadPartCopy. 

449 

450 This handler will deal with two cases: 

451 

452 * CopySource provided as a string. We'll make a best effort 

453 to URL encode the key name as required. This will require 

454 parsing the bucket and version id from the CopySource value 

455 and only encoding the key. 

456 * CopySource provided as a dict. In this case we're 

457 explicitly given the Bucket, Key, and VersionId so we're 

458 able to encode the key and ensure this value is serialized 

459 and correctly sent to S3. 

460 

461 """ 

462 source = params.get('CopySource') 

463 if source is None: 

464 # The call will eventually fail but we'll let the 

465 # param validator take care of this. It will 

466 # give a better error message. 

467 return 

468 if isinstance(source, str): 

469 params['CopySource'] = _quote_source_header(source) 

470 elif isinstance(source, dict): 

471 params['CopySource'] = _quote_source_header_from_dict(source) 

472 

473 

474def _quote_source_header_from_dict(source_dict): 

475 try: 

476 bucket = source_dict['Bucket'] 

477 key = source_dict['Key'] 

478 version_id = source_dict.get('VersionId') 

479 if VALID_S3_ARN.search(bucket): 

480 final = f'{bucket}/object/{key}' 

481 else: 

482 final = f'{bucket}/{key}' 

483 except KeyError as e: 

484 raise ParamValidationError( 

485 report=f'Missing required parameter: {str(e)}' 

486 ) 

487 final = percent_encode(final, safe=SAFE_CHARS + '/') 

488 if version_id is not None: 

489 final += f'?versionId={version_id}' 

490 return final 

491 

492 

493def _quote_source_header(value): 

494 result = VERSION_ID_SUFFIX.search(value) 

495 if result is None: 

496 return percent_encode(value, safe=SAFE_CHARS + '/') 

497 else: 

498 first, version_id = value[: result.start()], value[result.start() :] 

499 return percent_encode(first, safe=SAFE_CHARS + '/') + version_id 

500 

501 

502def _get_cross_region_presigned_url( 

503 request_signer, request_dict, model, source_region, destination_region 

504): 

505 # The better way to do this is to actually get the 

506 # endpoint_resolver and get the endpoint_url given the 

507 # source region. In this specific case, we know that 

508 # we can safely replace the dest region with the source 

509 # region because of the supported EC2 regions, but in 

510 # general this is not a safe assumption to make. 

511 # I think eventually we should try to plumb through something 

512 # that allows us to resolve endpoints from regions. 

513 request_dict_copy = copy.deepcopy(request_dict) 

514 request_dict_copy['body']['DestinationRegion'] = destination_region 

515 request_dict_copy['url'] = request_dict['url'].replace( 

516 destination_region, source_region 

517 ) 

518 request_dict_copy['method'] = 'GET' 

519 request_dict_copy['headers'] = {} 

520 return request_signer.generate_presigned_url( 

521 request_dict_copy, region_name=source_region, operation_name=model.name 

522 ) 

523 

524 

525def _get_presigned_url_source_and_destination_regions(request_signer, params): 

526 # Gets the source and destination regions to be used 

527 destination_region = request_signer._region_name 

528 source_region = params.get('SourceRegion') 

529 return source_region, destination_region 

530 

531 

532def inject_presigned_url_ec2(params, request_signer, model, **kwargs): 

533 # The customer can still provide this, so we should pass if they do. 

534 if 'PresignedUrl' in params['body']: 

535 return 

536 src, dest = _get_presigned_url_source_and_destination_regions( 

537 request_signer, params['body'] 

538 ) 

539 url = _get_cross_region_presigned_url( 

540 request_signer, params, model, src, dest 

541 ) 

542 params['body']['PresignedUrl'] = url 

543 # EC2 Requires that the destination region be sent over the wire in 

544 # addition to the source region. 

545 params['body']['DestinationRegion'] = dest 

546 

547 

548def inject_presigned_url_rds(params, request_signer, model, **kwargs): 

549 # SourceRegion is not required for RDS operations, so it's possible that 

550 # it isn't set. In that case it's probably a local copy so we don't need 

551 # to do anything else. 

552 if 'SourceRegion' not in params['body']: 

553 return 

554 

555 src, dest = _get_presigned_url_source_and_destination_regions( 

556 request_signer, params['body'] 

557 ) 

558 

559 # Since SourceRegion isn't actually modeled for RDS, it needs to be 

560 # removed from the request params before we send the actual request. 

561 del params['body']['SourceRegion'] 

562 

563 if 'PreSignedUrl' in params['body']: 

564 return 

565 

566 url = _get_cross_region_presigned_url( 

567 request_signer, params, model, src, dest 

568 ) 

569 params['body']['PreSignedUrl'] = url 

570 

571 

572def json_decode_policies(parsed, model, **kwargs): 

573 # Any time an IAM operation returns a policy document 

574 # it is a string that is json that has been urlencoded, 

575 # i.e urlencode(json.dumps(policy_document)). 

576 # To give users something more useful, we will urldecode 

577 # this value and json.loads() the result so that they have 

578 # the policy document as a dictionary. 

579 output_shape = model.output_shape 

580 if output_shape is not None: 

581 _decode_policy_types(parsed, model.output_shape) 

582 

583 

584def _decode_policy_types(parsed, shape): 

585 # IAM consistently uses the policyDocumentType shape to indicate 

586 # strings that have policy documents. 

587 shape_name = 'policyDocumentType' 

588 if shape.type_name == 'structure': 

589 for member_name, member_shape in shape.members.items(): 

590 if ( 

591 member_shape.type_name == 'string' 

592 and member_shape.name == shape_name 

593 and member_name in parsed 

594 ): 

595 parsed[member_name] = decode_quoted_jsondoc( 

596 parsed[member_name] 

597 ) 

598 elif member_name in parsed: 

599 _decode_policy_types(parsed[member_name], member_shape) 

600 if shape.type_name == 'list': 

601 shape_member = shape.member 

602 for item in parsed: 

603 _decode_policy_types(item, shape_member) 

604 

605 

606def parse_get_bucket_location(parsed, http_response, **kwargs): 

607 # s3.GetBucketLocation cannot be modeled properly. To 

608 # account for this we just manually parse the XML document. 

609 # The "parsed" passed in only has the ResponseMetadata 

610 # filled out. This handler will fill in the LocationConstraint 

611 # value. 

612 if http_response.raw is None: 

613 return 

614 response_body = http_response.content 

615 parser = ETree.XMLParser(target=ETree.TreeBuilder(), encoding='utf-8') 

616 parser.feed(response_body) 

617 root = parser.close() 

618 region = root.text 

619 parsed['LocationConstraint'] = region 

620 

621 

622def base64_encode_user_data(params, **kwargs): 

623 if 'UserData' in params: 

624 if isinstance(params['UserData'], str): 

625 # Encode it to bytes if it is text. 

626 params['UserData'] = params['UserData'].encode('utf-8') 

627 params['UserData'] = base64.b64encode(params['UserData']).decode( 

628 'utf-8' 

629 ) 

630 

631 

632def document_base64_encoding(param): 

633 description = ( 

634 '**This value will be base64 encoded automatically. Do ' 

635 'not base64 encode this value prior to performing the ' 

636 'operation.**' 

637 ) 

638 append = AppendParamDocumentation(param, description) 

639 return append.append_documentation 

640 

641 

642def validate_ascii_metadata(params, **kwargs): 

643 """Verify S3 Metadata only contains ascii characters. 

644 

645 From: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 

646 

647 "Amazon S3 stores user-defined metadata in lowercase. Each name, value pair 

648 must conform to US-ASCII when using REST and UTF-8 when using SOAP or 

649 browser-based uploads via POST." 

650 

651 """ 

652 metadata = params.get('Metadata') 

653 if not metadata or not isinstance(metadata, dict): 

654 # We have to at least type check the metadata as a dict type 

655 # because this handler is called before param validation. 

656 # We'll go ahead and return because the param validator will 

657 # give a descriptive error message for us. 

658 # We might need a post-param validation event. 

659 return 

660 for key, value in metadata.items(): 

661 try: 

662 key.encode('ascii') 

663 value.encode('ascii') 

664 except UnicodeEncodeError: 

665 error_msg = ( 

666 'Non ascii characters found in S3 metadata ' 

667 f'for key "{key}", value: "{value}". \nS3 metadata can only ' 

668 'contain ASCII characters. ' 

669 ) 

670 raise ParamValidationError(report=error_msg) 

671 

672 

673def fix_route53_ids(params, model, **kwargs): 

674 """ 

675 Check for and split apart Route53 resource IDs, setting 

676 only the last piece. This allows the output of one operation 

677 (e.g. ``'foo/1234'``) to be used as input in another 

678 operation (e.g. it expects just ``'1234'``). 

679 """ 

680 input_shape = model.input_shape 

681 if not input_shape or not hasattr(input_shape, 'members'): 

682 return 

683 

684 members = [ 

685 name 

686 for (name, shape) in input_shape.members.items() 

687 if shape.name in ['ResourceId', 'DelegationSetId', 'ChangeId'] 

688 ] 

689 

690 for name in members: 

691 if name in params: 

692 orig_value = params[name] 

693 params[name] = orig_value.split('/')[-1] 

694 logger.debug('%s %s -> %s', name, orig_value, params[name]) 

695 

696 

697def inject_account_id(params, **kwargs): 

698 if params.get('accountId') is None: 

699 # Glacier requires accountId, but allows you 

700 # to specify '-' for the current owners account. 

701 # We add this default value if the user does not 

702 # provide the accountId as a convenience. 

703 params['accountId'] = '-' 

704 

705 

706def add_glacier_version(model, params, **kwargs): 

707 request_dict = params 

708 request_dict['headers']['x-amz-glacier-version'] = model.metadata[ 

709 'apiVersion' 

710 ] 

711 

712 

713def add_accept_header(model, params, **kwargs): 

714 if params['headers'].get('Accept', None) is None: 

715 request_dict = params 

716 request_dict['headers']['Accept'] = 'application/json' 

717 

718 

719def add_glacier_checksums(params, **kwargs): 

720 """Add glacier checksums to the http request. 

721 

722 This will add two headers to the http request: 

723 

724 * x-amz-content-sha256 

725 * x-amz-sha256-tree-hash 

726 

727 These values will only be added if they are not present 

728 in the HTTP request. 

729 

730 """ 

731 request_dict = params 

732 headers = request_dict['headers'] 

733 body = request_dict['body'] 

734 if isinstance(body, bytes): 

735 # If the user provided a bytes type instead of a file 

736 # like object, we're temporarily create a BytesIO object 

737 # so we can use the util functions to calculate the 

738 # checksums which assume file like objects. Note that 

739 # we're not actually changing the body in the request_dict. 

740 body = BytesIO(body) 

741 starting_position = body.tell() 

742 if 'x-amz-content-sha256' not in headers: 

743 headers['x-amz-content-sha256'] = utils.calculate_sha256( 

744 body, as_hex=True 

745 ) 

746 body.seek(starting_position) 

747 if 'x-amz-sha256-tree-hash' not in headers: 

748 headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body) 

749 body.seek(starting_position) 

750 

751 

752def document_glacier_tree_hash_checksum(): 

753 doc = ''' 

754 This is a required field. 

755 

756 Ideally you will want to compute this value with checksums from 

757 previous uploaded parts, using the algorithm described in 

758 `Glacier documentation <http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html>`_. 

759 

760 But if you prefer, you can also use botocore.utils.calculate_tree_hash() 

761 to compute it from raw file by:: 

762 

763 checksum = calculate_tree_hash(open('your_file.txt', 'rb')) 

764 

765 ''' 

766 return AppendParamDocumentation('checksum', doc).append_documentation 

767 

768 

769def document_cloudformation_get_template_return_type( 

770 section, event_name, **kwargs 

771): 

772 if 'response-params' in event_name: 

773 template_body_section = section.get_section('TemplateBody') 

774 type_section = template_body_section.get_section('param-type') 

775 type_section.clear_text() 

776 type_section.write('(*dict*) --') 

777 elif 'response-example' in event_name: 

778 parent = section.get_section('structure-value') 

779 param_line = parent.get_section('TemplateBody') 

780 value_portion = param_line.get_section('member-value') 

781 value_portion.clear_text() 

782 value_portion.write('{}') 

783 

784 

785def switch_host_machinelearning(request, **kwargs): 

786 switch_host_with_param(request, 'PredictEndpoint') 

787 

788 

789def check_openssl_supports_tls_version_1_2(**kwargs): 

790 import ssl 

791 

792 try: 

793 openssl_version_tuple = ssl.OPENSSL_VERSION_INFO 

794 if openssl_version_tuple < (1, 0, 1): 

795 warnings.warn( 

796 f'Currently installed openssl version: {ssl.OPENSSL_VERSION} does not ' 

797 'support TLS 1.2, which is required for use of iot-data. ' 

798 'Please use python installed with openssl version 1.0.1 or ' 

799 'higher.', 

800 UnsupportedTLSVersionWarning, 

801 ) 

802 # We cannot check the openssl version on python2.6, so we should just 

803 # pass on this conveniency check. 

804 except AttributeError: 

805 pass 

806 

807 

808def change_get_to_post(request, **kwargs): 

809 # This is useful when we need to change a potentially large GET request 

810 # into a POST with x-www-form-urlencoded encoding. 

811 if request.method == 'GET' and '?' in request.url: 

812 request.headers['Content-Type'] = 'application/x-www-form-urlencoded' 

813 request.method = 'POST' 

814 request.url, request.data = request.url.split('?', 1) 

815 

816 

817def set_list_objects_encoding_type_url(params, context, **kwargs): 

818 if 'EncodingType' not in params: 

819 # We set this context so that we know it wasn't the customer that 

820 # requested the encoding. 

821 context['encoding_type_auto_set'] = True 

822 params['EncodingType'] = 'url' 

823 

824 

825def decode_list_object(parsed, context, **kwargs): 

826 # This is needed because we are passing url as the encoding type. Since the 

827 # paginator is based on the key, we need to handle it before it can be 

828 # round tripped. 

829 # 

830 # From the documentation: If you specify encoding-type request parameter, 

831 # Amazon S3 includes this element in the response, and returns encoded key 

832 # name values in the following response elements: 

833 # Delimiter, Marker, Prefix, NextMarker, Key. 

834 _decode_list_object( 

835 top_level_keys=['Delimiter', 'Marker', 'NextMarker'], 

836 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

837 parsed=parsed, 

838 context=context, 

839 ) 

840 

841 

842def decode_list_object_v2(parsed, context, **kwargs): 

843 # From the documentation: If you specify encoding-type request parameter, 

844 # Amazon S3 includes this element in the response, and returns encoded key 

845 # name values in the following response elements: 

846 # Delimiter, Prefix, ContinuationToken, Key, and StartAfter. 

847 _decode_list_object( 

848 top_level_keys=['Delimiter', 'Prefix', 'StartAfter'], 

849 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

850 parsed=parsed, 

851 context=context, 

852 ) 

853 

854 

855def decode_list_object_versions(parsed, context, **kwargs): 

856 # From the documentation: If you specify encoding-type request parameter, 

857 # Amazon S3 includes this element in the response, and returns encoded key 

858 # name values in the following response elements: 

859 # KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter. 

860 _decode_list_object( 

861 top_level_keys=[ 

862 'KeyMarker', 

863 'NextKeyMarker', 

864 'Prefix', 

865 'Delimiter', 

866 ], 

867 nested_keys=[ 

868 ('Versions', 'Key'), 

869 ('DeleteMarkers', 'Key'), 

870 ('CommonPrefixes', 'Prefix'), 

871 ], 

872 parsed=parsed, 

873 context=context, 

874 ) 

875 

876 

877def _decode_list_object(top_level_keys, nested_keys, parsed, context): 

878 if parsed.get('EncodingType') == 'url' and context.get( 

879 'encoding_type_auto_set' 

880 ): 

881 # URL decode top-level keys in the response if present. 

882 for key in top_level_keys: 

883 if key in parsed: 

884 parsed[key] = unquote_str(parsed[key]) 

885 # URL decode nested keys from the response if present. 

886 for top_key, child_key in nested_keys: 

887 if top_key in parsed: 

888 for member in parsed[top_key]: 

889 member[child_key] = unquote_str(member[child_key]) 

890 

891 

892def convert_body_to_file_like_object(params, **kwargs): 

893 if 'Body' in params: 

894 if isinstance(params['Body'], str): 

895 params['Body'] = BytesIO(ensure_bytes(params['Body'])) 

896 elif isinstance(params['Body'], bytes): 

897 params['Body'] = BytesIO(params['Body']) 

898 

899 

900def _add_parameter_aliases(handler_list): 

901 # Mapping of original parameter to parameter alias. 

902 # The key is <service>.<operation>.parameter 

903 # The first part of the key is used for event registration. 

904 # The last part is the original parameter name and the value is the 

905 # alias to expose in documentation. 

906 aliases = { 

907 'ec2.*.Filter': 'Filters', 

908 'logs.CreateExportTask.from': 'fromTime', 

909 'cloudsearchdomain.Search.return': 'returnFields', 

910 } 

911 

912 for original, new_name in aliases.items(): 

913 event_portion, original_name = original.rsplit('.', 1) 

914 parameter_alias = ParameterAlias(original_name, new_name) 

915 

916 # Add the handlers to the list of handlers. 

917 # One handler is to handle when users provide the alias. 

918 # The other handler is to update the documentation to show only 

919 # the alias. 

920 parameter_build_event_handler_tuple = ( 

921 'before-parameter-build.' + event_portion, 

922 parameter_alias.alias_parameter_in_call, 

923 REGISTER_FIRST, 

924 ) 

925 docs_event_handler_tuple = ( 

926 'docs.*.' + event_portion + '.complete-section', 

927 parameter_alias.alias_parameter_in_documentation, 

928 ) 

929 handler_list.append(parameter_build_event_handler_tuple)