Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/handlers.py: 24%

1# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"). You 

4# may not use this file except in compliance with the License. A copy of 

5# the License is located at 

6# 

7# http://aws.amazon.com/apache2.0/ 

8# 

9# or in the "license" file accompanying this file. This file is 

10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 

11# ANY KIND, either express or implied. See the License for the specific 

12# language governing permissions and limitations under the License. 

13 

14"""Builtin event handlers. 

15 

16This module contains builtin handlers for events emitted by botocore. 

17""" 

18 

19import base64 

20import copy 

21import logging 

22import os 

23import re 

24import uuid 

25import warnings 

26from io import BytesIO 

27 

28import botocore 

29import botocore.auth 

30from botocore import utils 

31from botocore.compat import ( 

32 ETree, 

33 OrderedDict, 

34 XMLParseError, 

35 ensure_bytes, 

36 get_md5, 

37 json, 

38 quote, 

39 unquote, 

40 unquote_str, 

41 urlsplit, 

42 urlunsplit, 

43) 

44from botocore.docs.utils import ( 

45 AppendParamDocumentation, 

46 AutoPopulatedParam, 

47 HideParamFromOperations, 

48) 

49from botocore.endpoint_provider import VALID_HOST_LABEL_RE 

50from botocore.exceptions import ( 

51 AliasConflictParameterError, 

52 ParamValidationError, 

53 UnsupportedTLSVersionWarning, 

54) 

55from botocore.regions import EndpointResolverBuiltins 

56from botocore.signers import ( 

57 add_generate_db_auth_token, 

58 add_generate_presigned_post, 

59 add_generate_presigned_url, 

60) 

61from botocore.utils import ( 

62 SAFE_CHARS, 

63 ArnParser, 

64 conditionally_calculate_checksum, 

65 conditionally_calculate_md5, 

66 percent_encode, 

67 switch_host_with_param, 

68) 

69 

70# Keep these imported. There's pre-existing code that uses them. 

71from botocore import retryhandler # noqa 

72from botocore import translate # noqa 

73from botocore.compat import MD5_AVAILABLE # noqa 

74from botocore.exceptions import MissingServiceIdError # noqa 

75from botocore.utils import hyphenize_service_id # noqa 

76from botocore.utils import is_global_accesspoint # noqa 

77from botocore.utils import SERVICE_NAME_ALIASES # noqa 

78 

79 

80logger = logging.getLogger(__name__) 

81 

82REGISTER_FIRST = object() 

83REGISTER_LAST = object() 

84# From the S3 docs: 

85# The rules for bucket names in the US Standard region allow bucket names 

86# to be as long as 255 characters, and bucket names can contain any 

87# combination of uppercase letters, lowercase letters, numbers, periods 

88# (.), hyphens (-), and underscores (_). 

89VALID_BUCKET = re.compile(r'^[a-zA-Z0-9.\-_]{1,255}$') 

90_ACCESSPOINT_ARN = ( 

91 r'^arn:(aws).*:(s3|s3-object-lambda):[a-z\-0-9]*:[0-9]{12}:accesspoint[/:]' 

92 r'[a-zA-Z0-9\-.]{1,63}$' 

93) 

94_OUTPOST_ARN = ( 

95 r'^arn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]' 

96 r'[a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63}$' 

97) 

98VALID_S3_ARN = re.compile('|'.join([_ACCESSPOINT_ARN, _OUTPOST_ARN])) 

99# signing names used for the services s3 and s3-control, for example in 

100# botocore/data/s3/2006-03-01/endpoints-rule-set-1.json 

101S3_SIGNING_NAMES = ('s3', 's3-outposts', 's3-object-lambda', 's3express') 

102VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$') 

103 

104 

105def handle_service_name_alias(service_name, **kwargs): 

106 return SERVICE_NAME_ALIASES.get(service_name, service_name) 

107 

108 

109def add_recursion_detection_header(params, **kwargs): 

110 has_lambda_name = 'AWS_LAMBDA_FUNCTION_NAME' in os.environ 

111 trace_id = os.environ.get('_X_AMZN_TRACE_ID') 

112 if has_lambda_name and trace_id: 

113 headers = params['headers'] 

114 if 'X-Amzn-Trace-Id' not in headers: 

115 headers['X-Amzn-Trace-Id'] = quote(trace_id, safe='-=;:+&[]{}"\',') 

116 

117 

118def escape_xml_payload(params, **kwargs): 

119 # Replace \r and \n with the escaped sequence over the whole XML document 

120 # to avoid linebreak normalization modifying customer input when the 

121 # document is parsed. Ideally, we would do this in ElementTree.tostring, 

122 # but it doesn't allow us to override entity escaping for text fields. For 

123 # this operation \r and \n can only appear in the XML document if they were 

124 # passed as part of the customer input. 

125 body = params['body'] 

126 if b'\r' in body: 

127 body = body.replace(b'\r', b'
') 

128 if b'\n' in body: 

129 body = body.replace(b'\n', b'
') 

130 

131 params['body'] = body 

132 

133 

134def check_for_200_error(response, **kwargs): 

135 # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html 

136 # There are two opportunities for a copy request to return an error. One 

137 # can occur when Amazon S3 receives the copy request and the other can 

138 # occur while Amazon S3 is copying the files. If the error occurs before 

139 # the copy operation starts, you receive a standard Amazon S3 error. If the 

140 # error occurs during the copy operation, the error response is embedded in 

141 # the 200 OK response. This means that a 200 OK response can contain either 

142 # a success or an error. Make sure to design your application to parse the 

143 # contents of the response and handle it appropriately. 

144 # 

145 # So this handler checks for this case. Even though the server sends a 

146 # 200 response, conceptually this should be handled exactly like a 

147 # 500 response (with respect to raising exceptions, retries, etc.) 

148 # We're connected *before* all the other retry logic handlers, so as long 

149 # as we switch the error code to 500, we'll retry the error as expected. 

150 if response is None: 

151 # A None response can happen if an exception is raised while 

152 # trying to retrieve the response. See Endpoint._get_response(). 

153 return 

154 http_response, parsed = response 

155 if _looks_like_special_case_error(http_response): 

156 logger.debug( 

157 "Error found for response with 200 status code, " 

158 "errors: %s, changing status code to " 

159 "500.", 

160 parsed, 

161 ) 

162 http_response.status_code = 500 

163 

164 

165def _looks_like_special_case_error(http_response): 

166 if http_response.status_code == 200: 

167 try: 

168 parser = ETree.XMLParser( 

169 target=ETree.TreeBuilder(), encoding='utf-8' 

170 ) 

171 parser.feed(http_response.content) 

172 root = parser.close() 

173 except XMLParseError: 

174 # In cases of network disruptions, we may end up with a partial 

175 # streamed response from S3. We need to treat these cases as 

176 # 500 Service Errors and try again. 

177 return True 

178 if root.tag == 'Error': 

179 return True 

180 return False 

181 

182 

183def set_operation_specific_signer(context, signing_name, **kwargs): 

184 """Choose the operation-specific signer. 

185 

186 Individual operations may have a different auth type than the service as a 

187 whole. This will most often manifest as operations that should not be 

188 authenticated at all, but can include other auth modes such as sigv4 

189 without body signing. 

190 """ 

191 auth_type = context.get('auth_type') 

192 

193 # Auth type will be None if the operation doesn't have a configured auth 

194 # type. 

195 if not auth_type: 

196 return 

197 

198 # Auth type will be the string value 'none' if the operation should not 

199 # be signed at all. 

200 if auth_type == 'none': 

201 return botocore.UNSIGNED 

202 

203 if auth_type == 'bearer': 

204 return 'bearer' 

205 

206 # If the operation needs an unsigned body, we set additional context 

207 # allowing the signer to be aware of this. 

208 if context.get('unsigned_payload') or auth_type == 'v4-unsigned-body': 

209 context['payload_signing_enabled'] = False 

210 

211 if auth_type.startswith('v4'): 

212 if auth_type == 'v4-s3express': 

213 return auth_type 

214 

215 if auth_type == 'v4a': 

216 # If sigv4a is chosen, we must add additional signing config for 

217 # global signature. 

218 region = _resolve_sigv4a_region(context) 

219 signing = {'region': region, 'signing_name': signing_name} 

220 if 'signing' in context: 

221 context['signing'].update(signing) 

222 else: 

223 context['signing'] = signing 

224 signature_version = 'v4a' 

225 else: 

226 signature_version = 'v4' 

227 

228 # Signing names used by s3 and s3-control use customized signers "s3v4" 

229 # and "s3v4a". 

230 if signing_name in S3_SIGNING_NAMES: 

231 signature_version = f's3{signature_version}' 

232 

233 return signature_version 

234 

235 

236def _resolve_sigv4a_region(context): 

237 region = None 

238 if 'client_config' in context: 

239 region = context['client_config'].sigv4a_signing_region_set 

240 if not region and context.get('signing', {}).get('region'): 

241 region = context['signing']['region'] 

242 return region or '*' 

243 

244 

245def decode_console_output(parsed, **kwargs): 

246 if 'Output' in parsed: 

247 try: 

248 # We're using 'replace' for errors because it is 

249 # possible that console output contains non string 

250 # chars we can't utf-8 decode. 

251 value = base64.b64decode( 

252 bytes(parsed['Output'], 'latin-1') 

253 ).decode('utf-8', 'replace') 

254 parsed['Output'] = value 

255 except (ValueError, TypeError, AttributeError): 

256 logger.debug('Error decoding base64', exc_info=True) 

257 

258 

259def generate_idempotent_uuid(params, model, **kwargs): 

260 for name in model.idempotent_members: 

261 if name not in params: 

262 params[name] = str(uuid.uuid4()) 

263 logger.debug( 

264 f"injecting idempotency token ({params[name]}) into param '{name}'." 

265 ) 

266 

267 

268def decode_quoted_jsondoc(value): 

269 try: 

270 value = json.loads(unquote(value)) 

271 except (ValueError, TypeError): 

272 logger.debug('Error loading quoted JSON', exc_info=True) 

273 return value 

274 

275 

276def json_decode_template_body(parsed, **kwargs): 

277 if 'TemplateBody' in parsed: 

278 try: 

279 value = json.loads( 

280 parsed['TemplateBody'], object_pairs_hook=OrderedDict 

281 ) 

282 parsed['TemplateBody'] = value 

283 except (ValueError, TypeError): 

284 logger.debug('error loading JSON', exc_info=True) 

285 

286 

287def validate_bucket_name(params, **kwargs): 

288 if 'Bucket' not in params: 

289 return 

290 bucket = params['Bucket'] 

291 if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket): 

292 error_msg = ( 

293 f'Invalid bucket name "{bucket}": Bucket name must match ' 

294 f'the regex "{VALID_BUCKET.pattern}" or be an ARN matching ' 

295 f'the regex "{VALID_S3_ARN.pattern}"' 

296 ) 

297 raise ParamValidationError(report=error_msg) 

298 

299 

300def sse_md5(params, **kwargs): 

301 """ 

302 S3 server-side encryption requires the encryption key to be sent to the 

303 server base64 encoded, as well as a base64-encoded MD5 hash of the 

304 encryption key. This handler does both if the MD5 has not been set by 

305 the caller. 

306 """ 

307 _sse_md5(params, 'SSECustomer') 

308 

309 

310def copy_source_sse_md5(params, **kwargs): 

311 """ 

312 S3 server-side encryption requires the encryption key to be sent to the 

313 server base64 encoded, as well as a base64-encoded MD5 hash of the 

314 encryption key. This handler does both if the MD5 has not been set by 

315 the caller specifically if the parameter is for the copy-source sse-c key. 

316 """ 

317 _sse_md5(params, 'CopySourceSSECustomer') 

318 

319 

320def _sse_md5(params, sse_member_prefix='SSECustomer'): 

321 if not _needs_s3_sse_customization(params, sse_member_prefix): 

322 return 

323 

324 sse_key_member = sse_member_prefix + 'Key' 

325 sse_md5_member = sse_member_prefix + 'KeyMD5' 

326 key_as_bytes = params[sse_key_member] 

327 if isinstance(key_as_bytes, str): 

328 key_as_bytes = key_as_bytes.encode('utf-8') 

329 key_md5_str = base64.b64encode(get_md5(key_as_bytes).digest()).decode( 

330 'utf-8' 

331 ) 

332 key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8') 

333 params[sse_key_member] = key_b64_encoded 

334 params[sse_md5_member] = key_md5_str 

335 

336 

337def _needs_s3_sse_customization(params, sse_member_prefix): 

338 return ( 

339 params.get(sse_member_prefix + 'Key') is not None 

340 and sse_member_prefix + 'KeyMD5' not in params 

341 ) 

342 

343 

344def disable_signing(**kwargs): 

345 """ 

346 This handler disables request signing by setting the signer 

347 name to a special sentinel value. 

348 """ 

349 return botocore.UNSIGNED 

350 

351 

352def add_expect_header(model, params, **kwargs): 

353 if model.http.get('method', '') not in ['PUT', 'POST']: 

354 return 

355 if 'body' in params: 

356 body = params['body'] 

357 if hasattr(body, 'read'): 

358 check_body = utils.ensure_boolean( 

359 os.environ.get( 

360 'BOTO_EXPERIMENTAL__NO_EMPTY_CONTINUE', 

361 False, 

362 ) 

363 ) 

364 if check_body and utils.determine_content_length(body) == 0: 

365 return 

366 # Any file like object will use an expect 100-continue 

367 # header regardless of size. 

368 logger.debug("Adding expect 100 continue header to request.") 

369 params['headers']['Expect'] = '100-continue' 

370 

371 

372class DeprecatedServiceDocumenter: 

373 def __init__(self, replacement_service_name): 

374 self._replacement_service_name = replacement_service_name 

375 

376 def inject_deprecation_notice(self, section, event_name, **kwargs): 

377 section.style.start_important() 

378 section.write('This service client is deprecated. Please use ') 

379 section.style.ref( 

380 self._replacement_service_name, 

381 self._replacement_service_name, 

382 ) 

383 section.write(' instead.') 

384 section.style.end_important() 

385 

386 

387def document_copy_source_form(section, event_name, **kwargs): 

388 if 'request-example' in event_name: 

389 parent = section.get_section('structure-value') 

390 param_line = parent.get_section('CopySource') 

391 value_portion = param_line.get_section('member-value') 

392 value_portion.clear_text() 

393 value_portion.write( 

394 "'string' or {'Bucket': 'string', " 

395 "'Key': 'string', 'VersionId': 'string'}" 

396 ) 

397 elif 'request-params' in event_name: 

398 param_section = section.get_section('CopySource') 

399 type_section = param_section.get_section('param-type') 

400 type_section.clear_text() 

401 type_section.write(':type CopySource: str or dict') 

402 doc_section = param_section.get_section('param-documentation') 

403 doc_section.clear_text() 

404 doc_section.write( 

405 "The name of the source bucket, key name of the source object, " 

406 "and optional version ID of the source object. You can either " 

407 "provide this value as a string or a dictionary. The " 

408 "string form is {bucket}/{key} or " 

409 "{bucket}/{key}?versionId={versionId} if you want to copy a " 

410 "specific version. You can also provide this value as a " 

411 "dictionary. The dictionary format is recommended over " 

412 "the string format because it is more explicit. The dictionary " 

413 "format is: {'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}." 

414 " Note that the VersionId key is optional and may be omitted." 

415 " To specify an S3 access point, provide the access point" 

416 " ARN for the ``Bucket`` key in the copy source dictionary. If you" 

417 " want to provide the copy source for an S3 access point as a" 

418 " string instead of a dictionary, the ARN provided must be the" 

419 " full S3 access point object ARN" 

420 " (i.e. {accesspoint_arn}/object/{key})" 

421 ) 

422 

423 

424def handle_copy_source_param(params, **kwargs): 

425 """Convert CopySource param for CopyObject/UploadPartCopy. 

426 

427 This handler will deal with two cases: 

428 

429 * CopySource provided as a string. We'll make a best effort 

430 to URL encode the key name as required. This will require 

431 parsing the bucket and version id from the CopySource value 

432 and only encoding the key. 

433 * CopySource provided as a dict. In this case we're 

434 explicitly given the Bucket, Key, and VersionId so we're 

435 able to encode the key and ensure this value is serialized 

436 and correctly sent to S3. 

437 

438 """ 

439 source = params.get('CopySource') 

440 if source is None: 

441 # The call will eventually fail but we'll let the 

442 # param validator take care of this. It will 

443 # give a better error message. 

444 return 

445 if isinstance(source, str): 

446 params['CopySource'] = _quote_source_header(source) 

447 elif isinstance(source, dict): 

448 params['CopySource'] = _quote_source_header_from_dict(source) 

449 

450 

451def _quote_source_header_from_dict(source_dict): 

452 try: 

453 bucket = source_dict['Bucket'] 

454 key = source_dict['Key'] 

455 version_id = source_dict.get('VersionId') 

456 if VALID_S3_ARN.search(bucket): 

457 final = f'{bucket}/object/{key}' 

458 else: 

459 final = f'{bucket}/{key}' 

460 except KeyError as e: 

461 raise ParamValidationError( 

462 report=f'Missing required parameter: {str(e)}' 

463 ) 

464 final = percent_encode(final, safe=SAFE_CHARS + '/') 

465 if version_id is not None: 

466 final += f'?versionId={version_id}' 

467 return final 

468 

469 

470def _quote_source_header(value): 

471 result = VERSION_ID_SUFFIX.search(value) 

472 if result is None: 

473 return percent_encode(value, safe=SAFE_CHARS + '/') 

474 else: 

475 first, version_id = value[: result.start()], value[result.start() :] 

476 return percent_encode(first, safe=SAFE_CHARS + '/') + version_id 

477 

478 

479def _get_cross_region_presigned_url( 

480 request_signer, request_dict, model, source_region, destination_region 

481): 

482 # The better way to do this is to actually get the 

483 # endpoint_resolver and get the endpoint_url given the 

484 # source region. In this specific case, we know that 

485 # we can safely replace the dest region with the source 

486 # region because of the supported EC2 regions, but in 

487 # general this is not a safe assumption to make. 

488 # I think eventually we should try to plumb through something 

489 # that allows us to resolve endpoints from regions. 

490 request_dict_copy = copy.deepcopy(request_dict) 

491 request_dict_copy['body']['DestinationRegion'] = destination_region 

492 request_dict_copy['url'] = request_dict['url'].replace( 

493 destination_region, source_region 

494 ) 

495 request_dict_copy['method'] = 'GET' 

496 request_dict_copy['headers'] = {} 

497 return request_signer.generate_presigned_url( 

498 request_dict_copy, region_name=source_region, operation_name=model.name 

499 ) 

500 

501 

502def _get_presigned_url_source_and_destination_regions(request_signer, params): 

503 # Gets the source and destination regions to be used 

504 destination_region = request_signer._region_name 

505 source_region = params.get('SourceRegion') 

506 return source_region, destination_region 

507 

508 

509def inject_presigned_url_ec2(params, request_signer, model, **kwargs): 

510 # The customer can still provide this, so we should pass if they do. 

511 if 'PresignedUrl' in params['body']: 

512 return 

513 src, dest = _get_presigned_url_source_and_destination_regions( 

514 request_signer, params['body'] 

515 ) 

516 url = _get_cross_region_presigned_url( 

517 request_signer, params, model, src, dest 

518 ) 

519 params['body']['PresignedUrl'] = url 

520 # EC2 Requires that the destination region be sent over the wire in 

521 # addition to the source region. 

522 params['body']['DestinationRegion'] = dest 

523 

524 

525def inject_presigned_url_rds(params, request_signer, model, **kwargs): 

526 # SourceRegion is not required for RDS operations, so it's possible that 

527 # it isn't set. In that case it's probably a local copy so we don't need 

528 # to do anything else. 

529 if 'SourceRegion' not in params['body']: 

530 return 

531 

532 src, dest = _get_presigned_url_source_and_destination_regions( 

533 request_signer, params['body'] 

534 ) 

535 

536 # Since SourceRegion isn't actually modeled for RDS, it needs to be 

537 # removed from the request params before we send the actual request. 

538 del params['body']['SourceRegion'] 

539 

540 if 'PreSignedUrl' in params['body']: 

541 return 

542 

543 url = _get_cross_region_presigned_url( 

544 request_signer, params, model, src, dest 

545 ) 

546 params['body']['PreSignedUrl'] = url 

547 

548 

549def json_decode_policies(parsed, model, **kwargs): 

550 # Any time an IAM operation returns a policy document 

551 # it is a string that is json that has been urlencoded, 

552 # i.e urlencode(json.dumps(policy_document)). 

553 # To give users something more useful, we will urldecode 

554 # this value and json.loads() the result so that they have 

555 # the policy document as a dictionary. 

556 output_shape = model.output_shape 

557 if output_shape is not None: 

558 _decode_policy_types(parsed, model.output_shape) 

559 

560 

561def _decode_policy_types(parsed, shape): 

562 # IAM consistently uses the policyDocumentType shape to indicate 

563 # strings that have policy documents. 

564 shape_name = 'policyDocumentType' 

565 if shape.type_name == 'structure': 

566 for member_name, member_shape in shape.members.items(): 

567 if ( 

568 member_shape.type_name == 'string' 

569 and member_shape.name == shape_name 

570 and member_name in parsed 

571 ): 

572 parsed[member_name] = decode_quoted_jsondoc( 

573 parsed[member_name] 

574 ) 

575 elif member_name in parsed: 

576 _decode_policy_types(parsed[member_name], member_shape) 

577 if shape.type_name == 'list': 

578 shape_member = shape.member 

579 for item in parsed: 

580 _decode_policy_types(item, shape_member) 

581 

582 

583def parse_get_bucket_location(parsed, http_response, **kwargs): 

584 # s3.GetBucketLocation cannot be modeled properly. To 

585 # account for this we just manually parse the XML document. 

586 # The "parsed" passed in only has the ResponseMetadata 

587 # filled out. This handler will fill in the LocationConstraint 

588 # value. 

589 if http_response.raw is None: 

590 return 

591 response_body = http_response.content 

592 parser = ETree.XMLParser(target=ETree.TreeBuilder(), encoding='utf-8') 

593 parser.feed(response_body) 

594 root = parser.close() 

595 region = root.text 

596 parsed['LocationConstraint'] = region 

597 

598 

599def base64_encode_user_data(params, **kwargs): 

600 if 'UserData' in params: 

601 if isinstance(params['UserData'], str): 

602 # Encode it to bytes if it is text. 

603 params['UserData'] = params['UserData'].encode('utf-8') 

604 params['UserData'] = base64.b64encode(params['UserData']).decode( 

605 'utf-8' 

606 ) 

607 

608 

609def document_base64_encoding(param): 

610 description = ( 

611 '**This value will be base64 encoded automatically. Do ' 

612 'not base64 encode this value prior to performing the ' 

613 'operation.**' 

614 ) 

615 append = AppendParamDocumentation(param, description) 

616 return append.append_documentation 

617 

618 

619def validate_ascii_metadata(params, **kwargs): 

620 """Verify S3 Metadata only contains ascii characters. 

621 

622 From: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 

623 

624 "Amazon S3 stores user-defined metadata in lowercase. Each name, value pair 

625 must conform to US-ASCII when using REST and UTF-8 when using SOAP or 

626 browser-based uploads via POST." 

627 

628 """ 

629 metadata = params.get('Metadata') 

630 if not metadata or not isinstance(metadata, dict): 

631 # We have to at least type check the metadata as a dict type 

632 # because this handler is called before param validation. 

633 # We'll go ahead and return because the param validator will 

634 # give a descriptive error message for us. 

635 # We might need a post-param validation event. 

636 return 

637 for key, value in metadata.items(): 

638 try: 

639 key.encode('ascii') 

640 value.encode('ascii') 

641 except UnicodeEncodeError: 

642 error_msg = ( 

643 'Non ascii characters found in S3 metadata ' 

644 f'for key "{key}", value: "{value}". \nS3 metadata can only ' 

645 'contain ASCII characters. ' 

646 ) 

647 raise ParamValidationError(report=error_msg) 

648 

649 

650def fix_route53_ids(params, model, **kwargs): 

651 """ 

652 Check for and split apart Route53 resource IDs, setting 

653 only the last piece. This allows the output of one operation 

654 (e.g. ``'foo/1234'``) to be used as input in another 

655 operation (e.g. it expects just ``'1234'``). 

656 """ 

657 input_shape = model.input_shape 

658 if not input_shape or not hasattr(input_shape, 'members'): 

659 return 

660 

661 members = [ 

662 name 

663 for (name, shape) in input_shape.members.items() 

664 if shape.name in ['ResourceId', 'DelegationSetId', 'ChangeId'] 

665 ] 

666 

667 for name in members: 

668 if name in params: 

669 orig_value = params[name] 

670 params[name] = orig_value.split('/')[-1] 

671 logger.debug('%s %s -> %s', name, orig_value, params[name]) 

672 

673 

674def inject_account_id(params, **kwargs): 

675 if params.get('accountId') is None: 

676 # Glacier requires accountId, but allows you 

677 # to specify '-' for the current owners account. 

678 # We add this default value if the user does not 

679 # provide the accountId as a convenience. 

680 params['accountId'] = '-' 

681 

682 

683def add_glacier_version(model, params, **kwargs): 

684 request_dict = params 

685 request_dict['headers']['x-amz-glacier-version'] = model.metadata[ 

686 'apiVersion' 

687 ] 

688 

689 

690def add_accept_header(model, params, **kwargs): 

691 if params['headers'].get('Accept', None) is None: 

692 request_dict = params 

693 request_dict['headers']['Accept'] = 'application/json' 

694 

695 

696def add_glacier_checksums(params, **kwargs): 

697 """Add glacier checksums to the http request. 

698 

699 This will add two headers to the http request: 

700 

701 * x-amz-content-sha256 

702 * x-amz-sha256-tree-hash 

703 

704 These values will only be added if they are not present 

705 in the HTTP request. 

706 

707 """ 

708 request_dict = params 

709 headers = request_dict['headers'] 

710 body = request_dict['body'] 

711 if isinstance(body, bytes): 

712 # If the user provided a bytes type instead of a file 

713 # like object, we're temporarily create a BytesIO object 

714 # so we can use the util functions to calculate the 

715 # checksums which assume file like objects. Note that 

716 # we're not actually changing the body in the request_dict. 

717 body = BytesIO(body) 

718 starting_position = body.tell() 

719 if 'x-amz-content-sha256' not in headers: 

720 headers['x-amz-content-sha256'] = utils.calculate_sha256( 

721 body, as_hex=True 

722 ) 

723 body.seek(starting_position) 

724 if 'x-amz-sha256-tree-hash' not in headers: 

725 headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body) 

726 body.seek(starting_position) 

727 

728 

729def document_glacier_tree_hash_checksum(): 

730 doc = ''' 

731 This is a required field. 

732 

733 Ideally you will want to compute this value with checksums from 

734 previous uploaded parts, using the algorithm described in 

735 `Glacier documentation <http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html>`_. 

736 

737 But if you prefer, you can also use botocore.utils.calculate_tree_hash() 

738 to compute it from raw file by:: 

739 

740 checksum = calculate_tree_hash(open('your_file.txt', 'rb')) 

741 

742 ''' 

743 return AppendParamDocumentation('checksum', doc).append_documentation 

744 

745 

746def document_cloudformation_get_template_return_type( 

747 section, event_name, **kwargs 

748): 

749 if 'response-params' in event_name: 

750 template_body_section = section.get_section('TemplateBody') 

751 type_section = template_body_section.get_section('param-type') 

752 type_section.clear_text() 

753 type_section.write('(*dict*) --') 

754 elif 'response-example' in event_name: 

755 parent = section.get_section('structure-value') 

756 param_line = parent.get_section('TemplateBody') 

757 value_portion = param_line.get_section('member-value') 

758 value_portion.clear_text() 

759 value_portion.write('{}') 

760 

761 

762def switch_host_machinelearning(request, **kwargs): 

763 switch_host_with_param(request, 'PredictEndpoint') 

764 

765 

766def check_openssl_supports_tls_version_1_2(**kwargs): 

767 import ssl 

768 

769 try: 

770 openssl_version_tuple = ssl.OPENSSL_VERSION_INFO 

771 if openssl_version_tuple < (1, 0, 1): 

772 warnings.warn( 

773 f'Currently installed openssl version: {ssl.OPENSSL_VERSION} does not ' 

774 'support TLS 1.2, which is required for use of iot-data. ' 

775 'Please use python installed with openssl version 1.0.1 or ' 

776 'higher.', 

777 UnsupportedTLSVersionWarning, 

778 ) 

779 # We cannot check the openssl version on python2.6, so we should just 

780 # pass on this conveniency check. 

781 except AttributeError: 

782 pass 

783 

784 

785def change_get_to_post(request, **kwargs): 

786 # This is useful when we need to change a potentially large GET request 

787 # into a POST with x-www-form-urlencoded encoding. 

788 if request.method == 'GET' and '?' in request.url: 

789 request.headers['Content-Type'] = 'application/x-www-form-urlencoded' 

790 request.method = 'POST' 

791 request.url, request.data = request.url.split('?', 1) 

792 

793 

794def set_list_objects_encoding_type_url(params, context, **kwargs): 

795 if 'EncodingType' not in params: 

796 # We set this context so that we know it wasn't the customer that 

797 # requested the encoding. 

798 context['encoding_type_auto_set'] = True 

799 params['EncodingType'] = 'url' 

800 

801 

802def decode_list_object(parsed, context, **kwargs): 

803 # This is needed because we are passing url as the encoding type. Since the 

804 # paginator is based on the key, we need to handle it before it can be 

805 # round tripped. 

806 # 

807 # From the documentation: If you specify encoding-type request parameter, 

808 # Amazon S3 includes this element in the response, and returns encoded key 

809 # name values in the following response elements: 

810 # Delimiter, Marker, Prefix, NextMarker, Key. 

811 _decode_list_object( 

812 top_level_keys=['Delimiter', 'Marker', 'NextMarker'], 

813 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

814 parsed=parsed, 

815 context=context, 

816 ) 

817 

818 

819def decode_list_object_v2(parsed, context, **kwargs): 

820 # From the documentation: If you specify encoding-type request parameter, 

821 # Amazon S3 includes this element in the response, and returns encoded key 

822 # name values in the following response elements: 

823 # Delimiter, Prefix, ContinuationToken, Key, and StartAfter. 

824 _decode_list_object( 

825 top_level_keys=['Delimiter', 'Prefix', 'StartAfter'], 

826 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

827 parsed=parsed, 

828 context=context, 

829 ) 

830 

831 

832def decode_list_object_versions(parsed, context, **kwargs): 

833 # From the documentation: If you specify encoding-type request parameter, 

834 # Amazon S3 includes this element in the response, and returns encoded key 

835 # name values in the following response elements: 

836 # KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter. 

837 _decode_list_object( 

838 top_level_keys=[ 

839 'KeyMarker', 

840 'NextKeyMarker', 

841 'Prefix', 

842 'Delimiter', 

843 ], 

844 nested_keys=[ 

845 ('Versions', 'Key'), 

846 ('DeleteMarkers', 'Key'), 

847 ('CommonPrefixes', 'Prefix'), 

848 ], 

849 parsed=parsed, 

850 context=context, 

851 ) 

852 

853 

854def _decode_list_object(top_level_keys, nested_keys, parsed, context): 

855 if parsed.get('EncodingType') == 'url' and context.get( 

856 'encoding_type_auto_set' 

857 ): 

858 # URL decode top-level keys in the response if present. 

859 for key in top_level_keys: 

860 if key in parsed: 

861 parsed[key] = unquote_str(parsed[key]) 

862 # URL decode nested keys from the response if present. 

863 for top_key, child_key in nested_keys: 

864 if top_key in parsed: 

865 for member in parsed[top_key]: 

866 member[child_key] = unquote_str(member[child_key]) 

867 

868 

869def convert_body_to_file_like_object(params, **kwargs): 

870 if 'Body' in params: 

871 if isinstance(params['Body'], str): 

872 params['Body'] = BytesIO(ensure_bytes(params['Body'])) 

873 elif isinstance(params['Body'], bytes): 

874 params['Body'] = BytesIO(params['Body']) 

875 

876 

877def _add_parameter_aliases(handler_list): 

878 # Mapping of original parameter to parameter alias. 

879 # The key is <service>.<operation>.parameter 

880 # The first part of the key is used for event registration. 

881 # The last part is the original parameter name and the value is the 

882 # alias to expose in documentation. 

883 aliases = { 

884 'ec2.*.Filter': 'Filters', 

885 'logs.CreateExportTask.from': 'fromTime', 

886 'cloudsearchdomain.Search.return': 'returnFields', 

887 } 

888 

889 for original, new_name in aliases.items(): 

890 event_portion, original_name = original.rsplit('.', 1) 

891 parameter_alias = ParameterAlias(original_name, new_name) 

892 

893 # Add the handlers to the list of handlers. 

894 # One handler is to handle when users provide the alias. 

895 # The other handler is to update the documentation to show only 

896 # the alias. 

897 parameter_build_event_handler_tuple = ( 

898 'before-parameter-build.' + event_portion, 

899 parameter_alias.alias_parameter_in_call, 

900 REGISTER_FIRST, 

901 ) 

902 docs_event_handler_tuple = ( 

903 'docs.*.' + event_portion + '.complete-section', 

904 parameter_alias.alias_parameter_in_documentation, 

905 ) 

906 handler_list.append(parameter_build_event_handler_tuple) 

907 handler_list.append(docs_event_handler_tuple) 

908 

909 

910class ParameterAlias: 

911 def __init__(self, original_name, alias_name): 

912 self._original_name = original_name 

913 self._alias_name = alias_name 

914 

915 def alias_parameter_in_call(self, params, model, **kwargs): 

916 if model.input_shape: 

917 # Only consider accepting the alias if it is modeled in the 

918 # input shape. 

919 if self._original_name in model.input_shape.members: 

920 if self._alias_name in params: 

921 if self._original_name in params: 

922 raise AliasConflictParameterError( 

923 original=self._original_name, 

924 alias=self._alias_name,