Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/handlers.py: 26%

1# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"). You 

4# may not use this file except in compliance with the License. A copy of 

5# the License is located at 

6# 

7# http://aws.amazon.com/apache2.0/ 

8# 

9# or in the "license" file accompanying this file. This file is 

10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 

11# ANY KIND, either express or implied. See the License for the specific 

12# language governing permissions and limitations under the License. 

13 

14"""Builtin event handlers. 

15 

16This module contains builtin handlers for events emitted by botocore. 

17""" 

18 

19import base64 

20import copy 

21import logging 

22import os 

23import re 

24import uuid 

25import warnings 

26from io import BytesIO 

27 

28import botocore 

29import botocore.auth 

30from botocore import utils 

31from botocore.compat import ( 

32 ETree, 

33 OrderedDict, 

34 XMLParseError, 

35 ensure_bytes, 

36 get_md5, 

37 json, 

38 quote, 

39 unquote, 

40 unquote_str, 

41 urlsplit, 

42 urlunsplit, 

43) 

44from botocore.docs.utils import ( 

45 AppendParamDocumentation, 

46 AutoPopulatedParam, 

47 HideParamFromOperations, 

48) 

49from botocore.endpoint_provider import VALID_HOST_LABEL_RE 

50from botocore.exceptions import ( 

51 AliasConflictParameterError, 

52 ParamValidationError, 

53 UnsupportedTLSVersionWarning, 

54) 

55from botocore.regions import EndpointResolverBuiltins 

56from botocore.signers import ( 

57 add_generate_db_auth_token, 

58 add_generate_presigned_post, 

59 add_generate_presigned_url, 

60) 

61from botocore.utils import ( 

62 SAFE_CHARS, 

63 ArnParser, 

64 conditionally_calculate_checksum, 

65 conditionally_calculate_md5, 

66 percent_encode, 

67 switch_host_with_param, 

68) 

69 

70# Keep these imported. There's pre-existing code that uses them. 

71from botocore import retryhandler # noqa 

72from botocore import translate # noqa 

73from botocore.compat import MD5_AVAILABLE # noqa 

74from botocore.exceptions import MissingServiceIdError # noqa 

75from botocore.utils import hyphenize_service_id # noqa 

76from botocore.utils import is_global_accesspoint # noqa 

77from botocore.utils import SERVICE_NAME_ALIASES # noqa 

78 

79 

80logger = logging.getLogger(__name__) 

81 

82REGISTER_FIRST = object() 

83REGISTER_LAST = object() 

84# From the S3 docs: 

85# The rules for bucket names in the US Standard region allow bucket names 

86# to be as long as 255 characters, and bucket names can contain any 

87# combination of uppercase letters, lowercase letters, numbers, periods 

88# (.), hyphens (-), and underscores (_). 

89VALID_BUCKET = re.compile(r'^[a-zA-Z0-9.\-_]{1,255}$') 

90_ACCESSPOINT_ARN = ( 

91 r'^arn:(aws).*:(s3|s3-object-lambda):[a-z\-0-9]*:[0-9]{12}:accesspoint[/:]' 

92 r'[a-zA-Z0-9\-.]{1,63}$' 

93) 

94_OUTPOST_ARN = ( 

95 r'^arn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]' 

96 r'[a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63}$' 

97) 

98VALID_S3_ARN = re.compile('|'.join([_ACCESSPOINT_ARN, _OUTPOST_ARN])) 

99# signing names used for the services s3 and s3-control, for example in 

100# botocore/data/s3/2006-03-01/endpoints-rule-set-1.json 

101S3_SIGNING_NAMES = ('s3', 's3-outposts', 's3-object-lambda', 's3express') 

102VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$') 

103 

104 

105def handle_service_name_alias(service_name, **kwargs): 

106 return SERVICE_NAME_ALIASES.get(service_name, service_name) 

107 

108 

109def add_recursion_detection_header(params, **kwargs): 

110 has_lambda_name = 'AWS_LAMBDA_FUNCTION_NAME' in os.environ 

111 trace_id = os.environ.get('_X_AMZN_TRACE_ID') 

112 if has_lambda_name and trace_id: 

113 headers = params['headers'] 

114 if 'X-Amzn-Trace-Id' not in headers: 

115 headers['X-Amzn-Trace-Id'] = quote(trace_id, safe='-=;:+&[]{}"\',') 

116 

117 

118def escape_xml_payload(params, **kwargs): 

119 # Replace \r and \n with the escaped sequence over the whole XML document 

120 # to avoid linebreak normalization modifying customer input when the 

121 # document is parsed. Ideally, we would do this in ElementTree.tostring, 

122 # but it doesn't allow us to override entity escaping for text fields. For 

123 # this operation \r and \n can only appear in the XML document if they were 

124 # passed as part of the customer input. 

125 body = params['body'] 

126 if b'\r' in body: 

127 body = body.replace(b'\r', b'
') 

128 if b'\n' in body: 

129 body = body.replace(b'\n', b'
') 

130 

131 params['body'] = body 

132 

133 

134def check_for_200_error(response, **kwargs): 

135 # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html 

136 # There are two opportunities for a copy request to return an error. One 

137 # can occur when Amazon S3 receives the copy request and the other can 

138 # occur while Amazon S3 is copying the files. If the error occurs before 

139 # the copy operation starts, you receive a standard Amazon S3 error. If the 

140 # error occurs during the copy operation, the error response is embedded in 

141 # the 200 OK response. This means that a 200 OK response can contain either 

142 # a success or an error. Make sure to design your application to parse the 

143 # contents of the response and handle it appropriately. 

144 # 

145 # So this handler checks for this case. Even though the server sends a 

146 # 200 response, conceptually this should be handled exactly like a 

147 # 500 response (with respect to raising exceptions, retries, etc.) 

148 # We're connected *before* all the other retry logic handlers, so as long 

149 # as we switch the error code to 500, we'll retry the error as expected. 

150 if response is None: 

151 # A None response can happen if an exception is raised while 

152 # trying to retrieve the response. See Endpoint._get_response(). 

153 return 

154 http_response, parsed = response 

155 if _looks_like_special_case_error(http_response): 

156 logger.debug( 

157 "Error found for response with 200 status code, " 

158 "errors: %s, changing status code to " 

159 "500.", 

160 parsed, 

161 ) 

162 http_response.status_code = 500 

163 

164 

165def _looks_like_special_case_error(http_response): 

166 if http_response.status_code == 200: 

167 try: 

168 parser = ETree.XMLParser( 

169 target=ETree.TreeBuilder(), encoding='utf-8' 

170 ) 

171 parser.feed(http_response.content) 

172 root = parser.close() 

173 except XMLParseError: 

174 # In cases of network disruptions, we may end up with a partial 

175 # streamed response from S3. We need to treat these cases as 

176 # 500 Service Errors and try again. 

177 return True 

178 if root.tag == 'Error': 

179 return True 

180 return False 

181 

182 

183def set_operation_specific_signer(context, signing_name, **kwargs): 

184 """Choose the operation-specific signer. 

185 

186 Individual operations may have a different auth type than the service as a 

187 whole. This will most often manifest as operations that should not be 

188 authenticated at all, but can include other auth modes such as sigv4 

189 without body signing. 

190 """ 

191 auth_type = context.get('auth_type') 

192 

193 # Auth type will be None if the operation doesn't have a configured auth 

194 # type. 

195 if not auth_type: 

196 return 

197 

198 # Auth type will be the string value 'none' if the operation should not 

199 # be signed at all. 

200 if auth_type == 'none': 

201 return botocore.UNSIGNED 

202 

203 if auth_type == 'bearer': 

204 return 'bearer' 

205 

206 if auth_type.startswith('v4'): 

207 if auth_type == 'v4-s3express': 

208 return auth_type 

209 

210 if auth_type == 'v4a': 

211 # If sigv4a is chosen, we must add additional signing config for 

212 # global signature. 

213 signing = {'region': '*', 'signing_name': signing_name} 

214 if 'signing' in context: 

215 context['signing'].update(signing) 

216 else: 

217 context['signing'] = signing 

218 signature_version = 'v4a' 

219 else: 

220 signature_version = 'v4' 

221 

222 # If the operation needs an unsigned body, we set additional context 

223 # allowing the signer to be aware of this. 

224 if auth_type == 'v4-unsigned-body': 

225 context['payload_signing_enabled'] = False 

226 

227 # Signing names used by s3 and s3-control use customized signers "s3v4" 

228 # and "s3v4a". 

229 if signing_name in S3_SIGNING_NAMES: 

230 signature_version = f's3{signature_version}' 

231 

232 return signature_version 

233 

234 

235def decode_console_output(parsed, **kwargs): 

236 if 'Output' in parsed: 

237 try: 

238 # We're using 'replace' for errors because it is 

239 # possible that console output contains non string 

240 # chars we can't utf-8 decode. 

241 value = base64.b64decode( 

242 bytes(parsed['Output'], 'latin-1') 

243 ).decode('utf-8', 'replace') 

244 parsed['Output'] = value 

245 except (ValueError, TypeError, AttributeError): 

246 logger.debug('Error decoding base64', exc_info=True) 

247 

248 

249def generate_idempotent_uuid(params, model, **kwargs): 

250 for name in model.idempotent_members: 

251 if name not in params: 

252 params[name] = str(uuid.uuid4()) 

253 logger.debug( 

254 "injecting idempotency token (%s) into param '%s'." 

255 % (params[name], name) 

256 ) 

257 

258 

259def decode_quoted_jsondoc(value): 

260 try: 

261 value = json.loads(unquote(value)) 

262 except (ValueError, TypeError): 

263 logger.debug('Error loading quoted JSON', exc_info=True) 

264 return value 

265 

266 

267def json_decode_template_body(parsed, **kwargs): 

268 if 'TemplateBody' in parsed: 

269 try: 

270 value = json.loads( 

271 parsed['TemplateBody'], object_pairs_hook=OrderedDict 

272 ) 

273 parsed['TemplateBody'] = value 

274 except (ValueError, TypeError): 

275 logger.debug('error loading JSON', exc_info=True) 

276 

277 

278def validate_bucket_name(params, **kwargs): 

279 if 'Bucket' not in params: 

280 return 

281 bucket = params['Bucket'] 

282 if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket): 

283 error_msg = ( 

284 f'Invalid bucket name "{bucket}": Bucket name must match ' 

285 f'the regex "{VALID_BUCKET.pattern}" or be an ARN matching ' 

286 f'the regex "{VALID_S3_ARN.pattern}"' 

287 ) 

288 raise ParamValidationError(report=error_msg) 

289 

290 

291def sse_md5(params, **kwargs): 

292 """ 

293 S3 server-side encryption requires the encryption key to be sent to the 

294 server base64 encoded, as well as a base64-encoded MD5 hash of the 

295 encryption key. This handler does both if the MD5 has not been set by 

296 the caller. 

297 """ 

298 _sse_md5(params, 'SSECustomer') 

299 

300 

301def copy_source_sse_md5(params, **kwargs): 

302 """ 

303 S3 server-side encryption requires the encryption key to be sent to the 

304 server base64 encoded, as well as a base64-encoded MD5 hash of the 

305 encryption key. This handler does both if the MD5 has not been set by 

306 the caller specifically if the parameter is for the copy-source sse-c key. 

307 """ 

308 _sse_md5(params, 'CopySourceSSECustomer') 

309 

310 

311def _sse_md5(params, sse_member_prefix='SSECustomer'): 

312 if not _needs_s3_sse_customization(params, sse_member_prefix): 

313 return 

314 

315 sse_key_member = sse_member_prefix + 'Key' 

316 sse_md5_member = sse_member_prefix + 'KeyMD5' 

317 key_as_bytes = params[sse_key_member] 

318 if isinstance(key_as_bytes, str): 

319 key_as_bytes = key_as_bytes.encode('utf-8') 

320 key_md5_str = base64.b64encode(get_md5(key_as_bytes).digest()).decode( 

321 'utf-8' 

322 ) 

323 key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8') 

324 params[sse_key_member] = key_b64_encoded 

325 params[sse_md5_member] = key_md5_str 

326 

327 

328def _needs_s3_sse_customization(params, sse_member_prefix): 

329 return ( 

330 params.get(sse_member_prefix + 'Key') is not None 

331 and sse_member_prefix + 'KeyMD5' not in params 

332 ) 

333 

334 

335def disable_signing(**kwargs): 

336 """ 

337 This handler disables request signing by setting the signer 

338 name to a special sentinel value. 

339 """ 

340 return botocore.UNSIGNED 

341 

342 

343def add_expect_header(model, params, **kwargs): 

344 if model.http.get('method', '') not in ['PUT', 'POST']: 

345 return 

346 if 'body' in params: 

347 body = params['body'] 

348 if hasattr(body, 'read'): 

349 # Any file like object will use an expect 100-continue 

350 # header regardless of size. 

351 logger.debug("Adding expect 100 continue header to request.") 

352 params['headers']['Expect'] = '100-continue' 

353 

354 

355class DeprecatedServiceDocumenter: 

356 def __init__(self, replacement_service_name): 

357 self._replacement_service_name = replacement_service_name 

358 

359 def inject_deprecation_notice(self, section, event_name, **kwargs): 

360 section.style.start_important() 

361 section.write('This service client is deprecated. Please use ') 

362 section.style.ref( 

363 self._replacement_service_name, 

364 self._replacement_service_name, 

365 ) 

366 section.write(' instead.') 

367 section.style.end_important() 

368 

369 

370def document_copy_source_form(section, event_name, **kwargs): 

371 if 'request-example' in event_name: 

372 parent = section.get_section('structure-value') 

373 param_line = parent.get_section('CopySource') 

374 value_portion = param_line.get_section('member-value') 

375 value_portion.clear_text() 

376 value_portion.write( 

377 "'string' or {'Bucket': 'string', " 

378 "'Key': 'string', 'VersionId': 'string'}" 

379 ) 

380 elif 'request-params' in event_name: 

381 param_section = section.get_section('CopySource') 

382 type_section = param_section.get_section('param-type') 

383 type_section.clear_text() 

384 type_section.write(':type CopySource: str or dict') 

385 doc_section = param_section.get_section('param-documentation') 

386 doc_section.clear_text() 

387 doc_section.write( 

388 "The name of the source bucket, key name of the source object, " 

389 "and optional version ID of the source object. You can either " 

390 "provide this value as a string or a dictionary. The " 

391 "string form is {bucket}/{key} or " 

392 "{bucket}/{key}?versionId={versionId} if you want to copy a " 

393 "specific version. You can also provide this value as a " 

394 "dictionary. The dictionary format is recommended over " 

395 "the string format because it is more explicit. The dictionary " 

396 "format is: {'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}." 

397 " Note that the VersionId key is optional and may be omitted." 

398 " To specify an S3 access point, provide the access point" 

399 " ARN for the ``Bucket`` key in the copy source dictionary. If you" 

400 " want to provide the copy source for an S3 access point as a" 

401 " string instead of a dictionary, the ARN provided must be the" 

402 " full S3 access point object ARN" 

403 " (i.e. {accesspoint_arn}/object/{key})" 

404 ) 

405 

406 

407def handle_copy_source_param(params, **kwargs): 

408 """Convert CopySource param for CopyObject/UploadPartCopy. 

409 

410 This handler will deal with two cases: 

411 

412 * CopySource provided as a string. We'll make a best effort 

413 to URL encode the key name as required. This will require 

414 parsing the bucket and version id from the CopySource value 

415 and only encoding the key. 

416 * CopySource provided as a dict. In this case we're 

417 explicitly given the Bucket, Key, and VersionId so we're 

418 able to encode the key and ensure this value is serialized 

419 and correctly sent to S3. 

420 

421 """ 

422 source = params.get('CopySource') 

423 if source is None: 

424 # The call will eventually fail but we'll let the 

425 # param validator take care of this. It will 

426 # give a better error message. 

427 return 

428 if isinstance(source, str): 

429 params['CopySource'] = _quote_source_header(source) 

430 elif isinstance(source, dict): 

431 params['CopySource'] = _quote_source_header_from_dict(source) 

432 

433 

434def _quote_source_header_from_dict(source_dict): 

435 try: 

436 bucket = source_dict['Bucket'] 

437 key = source_dict['Key'] 

438 version_id = source_dict.get('VersionId') 

439 if VALID_S3_ARN.search(bucket): 

440 final = f'{bucket}/object/{key}' 

441 else: 

442 final = f'{bucket}/{key}' 

443 except KeyError as e: 

444 raise ParamValidationError( 

445 report=f'Missing required parameter: {str(e)}' 

446 ) 

447 final = percent_encode(final, safe=SAFE_CHARS + '/') 

448 if version_id is not None: 

449 final += '?versionId=%s' % version_id 

450 return final 

451 

452 

453def _quote_source_header(value): 

454 result = VERSION_ID_SUFFIX.search(value) 

455 if result is None: 

456 return percent_encode(value, safe=SAFE_CHARS + '/') 

457 else: 

458 first, version_id = value[: result.start()], value[result.start() :] 

459 return percent_encode(first, safe=SAFE_CHARS + '/') + version_id 

460 

461 

462def _get_cross_region_presigned_url( 

463 request_signer, request_dict, model, source_region, destination_region 

464): 

465 # The better way to do this is to actually get the 

466 # endpoint_resolver and get the endpoint_url given the 

467 # source region. In this specific case, we know that 

468 # we can safely replace the dest region with the source 

469 # region because of the supported EC2 regions, but in 

470 # general this is not a safe assumption to make. 

471 # I think eventually we should try to plumb through something 

472 # that allows us to resolve endpoints from regions. 

473 request_dict_copy = copy.deepcopy(request_dict) 

474 request_dict_copy['body']['DestinationRegion'] = destination_region 

475 request_dict_copy['url'] = request_dict['url'].replace( 

476 destination_region, source_region 

477 ) 

478 request_dict_copy['method'] = 'GET' 

479 request_dict_copy['headers'] = {} 

480 return request_signer.generate_presigned_url( 

481 request_dict_copy, region_name=source_region, operation_name=model.name 

482 ) 

483 

484 

485def _get_presigned_url_source_and_destination_regions(request_signer, params): 

486 # Gets the source and destination regions to be used 

487 destination_region = request_signer._region_name 

488 source_region = params.get('SourceRegion') 

489 return source_region, destination_region 

490 

491 

492def inject_presigned_url_ec2(params, request_signer, model, **kwargs): 

493 # The customer can still provide this, so we should pass if they do. 

494 if 'PresignedUrl' in params['body']: 

495 return 

496 src, dest = _get_presigned_url_source_and_destination_regions( 

497 request_signer, params['body'] 

498 ) 

499 url = _get_cross_region_presigned_url( 

500 request_signer, params, model, src, dest 

501 ) 

502 params['body']['PresignedUrl'] = url 

503 # EC2 Requires that the destination region be sent over the wire in 

504 # addition to the source region. 

505 params['body']['DestinationRegion'] = dest 

506 

507 

508def inject_presigned_url_rds(params, request_signer, model, **kwargs): 

509 # SourceRegion is not required for RDS operations, so it's possible that 

510 # it isn't set. In that case it's probably a local copy so we don't need 

511 # to do anything else. 

512 if 'SourceRegion' not in params['body']: 

513 return 

514 

515 src, dest = _get_presigned_url_source_and_destination_regions( 

516 request_signer, params['body'] 

517 ) 

518 

519 # Since SourceRegion isn't actually modeled for RDS, it needs to be 

520 # removed from the request params before we send the actual request. 

521 del params['body']['SourceRegion'] 

522 

523 if 'PreSignedUrl' in params['body']: 

524 return 

525 

526 url = _get_cross_region_presigned_url( 

527 request_signer, params, model, src, dest 

528 ) 

529 params['body']['PreSignedUrl'] = url 

530 

531 

532def json_decode_policies(parsed, model, **kwargs): 

533 # Any time an IAM operation returns a policy document 

534 # it is a string that is json that has been urlencoded, 

535 # i.e urlencode(json.dumps(policy_document)). 

536 # To give users something more useful, we will urldecode 

537 # this value and json.loads() the result so that they have 

538 # the policy document as a dictionary. 

539 output_shape = model.output_shape 

540 if output_shape is not None: 

541 _decode_policy_types(parsed, model.output_shape) 

542 

543 

544def _decode_policy_types(parsed, shape): 

545 # IAM consistently uses the policyDocumentType shape to indicate 

546 # strings that have policy documents. 

547 shape_name = 'policyDocumentType' 

548 if shape.type_name == 'structure': 

549 for member_name, member_shape in shape.members.items(): 

550 if ( 

551 member_shape.type_name == 'string' 

552 and member_shape.name == shape_name 

553 and member_name in parsed 

554 ): 

555 parsed[member_name] = decode_quoted_jsondoc( 

556 parsed[member_name] 

557 ) 

558 elif member_name in parsed: 

559 _decode_policy_types(parsed[member_name], member_shape) 

560 if shape.type_name == 'list': 

561 shape_member = shape.member 

562 for item in parsed: 

563 _decode_policy_types(item, shape_member) 

564 

565 

566def parse_get_bucket_location(parsed, http_response, **kwargs): 

567 # s3.GetBucketLocation cannot be modeled properly. To 

568 # account for this we just manually parse the XML document. 

569 # The "parsed" passed in only has the ResponseMetadata 

570 # filled out. This handler will fill in the LocationConstraint 

571 # value. 

572 if http_response.raw is None: 

573 return 

574 response_body = http_response.content 

575 parser = ETree.XMLParser(target=ETree.TreeBuilder(), encoding='utf-8') 

576 parser.feed(response_body) 

577 root = parser.close() 

578 region = root.text 

579 parsed['LocationConstraint'] = region 

580 

581 

582def base64_encode_user_data(params, **kwargs): 

583 if 'UserData' in params: 

584 if isinstance(params['UserData'], str): 

585 # Encode it to bytes if it is text. 

586 params['UserData'] = params['UserData'].encode('utf-8') 

587 params['UserData'] = base64.b64encode(params['UserData']).decode( 

588 'utf-8' 

589 ) 

590 

591 

592def document_base64_encoding(param): 

593 description = ( 

594 '**This value will be base64 encoded automatically. Do ' 

595 'not base64 encode this value prior to performing the ' 

596 'operation.**' 

597 ) 

598 append = AppendParamDocumentation(param, description) 

599 return append.append_documentation 

600 

601 

602def validate_ascii_metadata(params, **kwargs): 

603 """Verify S3 Metadata only contains ascii characters. 

604 

605 From: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 

606 

607 "Amazon S3 stores user-defined metadata in lowercase. Each name, value pair 

608 must conform to US-ASCII when using REST and UTF-8 when using SOAP or 

609 browser-based uploads via POST." 

610 

611 """ 

612 metadata = params.get('Metadata') 

613 if not metadata or not isinstance(metadata, dict): 

614 # We have to at least type check the metadata as a dict type 

615 # because this handler is called before param validation. 

616 # We'll go ahead and return because the param validator will 

617 # give a descriptive error message for us. 

618 # We might need a post-param validation event. 

619 return 

620 for key, value in metadata.items(): 

621 try: 

622 key.encode('ascii') 

623 value.encode('ascii') 

624 except UnicodeEncodeError: 

625 error_msg = ( 

626 'Non ascii characters found in S3 metadata ' 

627 'for key "%s", value: "%s". \nS3 metadata can only ' 

628 'contain ASCII characters. ' % (key, value) 

629 ) 

630 raise ParamValidationError(report=error_msg) 

631 

632 

633def fix_route53_ids(params, model, **kwargs): 

634 """ 

635 Check for and split apart Route53 resource IDs, setting 

636 only the last piece. This allows the output of one operation 

637 (e.g. ``'foo/1234'``) to be used as input in another 

638 operation (e.g. it expects just ``'1234'``). 

639 """ 

640 input_shape = model.input_shape 

641 if not input_shape or not hasattr(input_shape, 'members'): 

642 return 

643 

644 members = [ 

645 name 

646 for (name, shape) in input_shape.members.items() 

647 if shape.name in ['ResourceId', 'DelegationSetId', 'ChangeId'] 

648 ] 

649 

650 for name in members: 

651 if name in params: 

652 orig_value = params[name] 

653 params[name] = orig_value.split('/')[-1] 

654 logger.debug('%s %s -> %s', name, orig_value, params[name]) 

655 

656 

657def inject_account_id(params, **kwargs): 

658 if params.get('accountId') is None: 

659 # Glacier requires accountId, but allows you 

660 # to specify '-' for the current owners account. 

661 # We add this default value if the user does not 

662 # provide the accountId as a convenience. 

663 params['accountId'] = '-' 

664 

665 

666def add_glacier_version(model, params, **kwargs): 

667 request_dict = params 

668 request_dict['headers']['x-amz-glacier-version'] = model.metadata[ 

669 'apiVersion' 

670 ] 

671 

672 

673def add_accept_header(model, params, **kwargs): 

674 if params['headers'].get('Accept', None) is None: 

675 request_dict = params 

676 request_dict['headers']['Accept'] = 'application/json' 

677 

678 

679def add_glacier_checksums(params, **kwargs): 

680 """Add glacier checksums to the http request. 

681 

682 This will add two headers to the http request: 

683 

684 * x-amz-content-sha256 

685 * x-amz-sha256-tree-hash 

686 

687 These values will only be added if they are not present 

688 in the HTTP request. 

689 

690 """ 

691 request_dict = params 

692 headers = request_dict['headers'] 

693 body = request_dict['body'] 

694 if isinstance(body, bytes): 

695 # If the user provided a bytes type instead of a file 

696 # like object, we're temporarily create a BytesIO object 

697 # so we can use the util functions to calculate the 

698 # checksums which assume file like objects. Note that 

699 # we're not actually changing the body in the request_dict. 

700 body = BytesIO(body) 

701 starting_position = body.tell() 

702 if 'x-amz-content-sha256' not in headers: 

703 headers['x-amz-content-sha256'] = utils.calculate_sha256( 

704 body, as_hex=True 

705 ) 

706 body.seek(starting_position) 

707 if 'x-amz-sha256-tree-hash' not in headers: 

708 headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body) 

709 body.seek(starting_position) 

710 

711 

712def document_glacier_tree_hash_checksum(): 

713 doc = ''' 

714 This is a required field. 

715 

716 Ideally you will want to compute this value with checksums from 

717 previous uploaded parts, using the algorithm described in 

718 `Glacier documentation <http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html>`_. 

719 

720 But if you prefer, you can also use botocore.utils.calculate_tree_hash() 

721 to compute it from raw file by:: 

722 

723 checksum = calculate_tree_hash(open('your_file.txt', 'rb')) 

724 

725 ''' 

726 return AppendParamDocumentation('checksum', doc).append_documentation 

727 

728 

729def document_cloudformation_get_template_return_type( 

730 section, event_name, **kwargs 

731): 

732 if 'response-params' in event_name: 

733 template_body_section = section.get_section('TemplateBody') 

734 type_section = template_body_section.get_section('param-type') 

735 type_section.clear_text() 

736 type_section.write('(*dict*) --') 

737 elif 'response-example' in event_name: 

738 parent = section.get_section('structure-value') 

739 param_line = parent.get_section('TemplateBody') 

740 value_portion = param_line.get_section('member-value') 

741 value_portion.clear_text() 

742 value_portion.write('{}') 

743 

744 

745def switch_host_machinelearning(request, **kwargs): 

746 switch_host_with_param(request, 'PredictEndpoint') 

747 

748 

749def check_openssl_supports_tls_version_1_2(**kwargs): 

750 import ssl 

751 

752 try: 

753 openssl_version_tuple = ssl.OPENSSL_VERSION_INFO 

754 if openssl_version_tuple < (1, 0, 1): 

755 warnings.warn( 

756 'Currently installed openssl version: %s does not ' 

757 'support TLS 1.2, which is required for use of iot-data. ' 

758 'Please use python installed with openssl version 1.0.1 or ' 

759 'higher.' % (ssl.OPENSSL_VERSION), 

760 UnsupportedTLSVersionWarning, 

761 ) 

762 # We cannot check the openssl version on python2.6, so we should just 

763 # pass on this conveniency check. 

764 except AttributeError: 

765 pass 

766 

767 

768def change_get_to_post(request, **kwargs): 

769 # This is useful when we need to change a potentially large GET request 

770 # into a POST with x-www-form-urlencoded encoding. 

771 if request.method == 'GET' and '?' in request.url: 

772 request.headers['Content-Type'] = 'application/x-www-form-urlencoded' 

773 request.method = 'POST' 

774 request.url, request.data = request.url.split('?', 1) 

775 

776 

777def set_list_objects_encoding_type_url(params, context, **kwargs): 

778 if 'EncodingType' not in params: 

779 # We set this context so that we know it wasn't the customer that 

780 # requested the encoding. 

781 context['encoding_type_auto_set'] = True 

782 params['EncodingType'] = 'url' 

783 

784 

785def decode_list_object(parsed, context, **kwargs): 

786 # This is needed because we are passing url as the encoding type. Since the 

787 # paginator is based on the key, we need to handle it before it can be 

788 # round tripped. 

789 # 

790 # From the documentation: If you specify encoding-type request parameter, 

791 # Amazon S3 includes this element in the response, and returns encoded key 

792 # name values in the following response elements: 

793 # Delimiter, Marker, Prefix, NextMarker, Key. 

794 _decode_list_object( 

795 top_level_keys=['Delimiter', 'Marker', 'NextMarker'], 

796 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

797 parsed=parsed, 

798 context=context, 

799 ) 

800 

801 

802def decode_list_object_v2(parsed, context, **kwargs): 

803 # From the documentation: If you specify encoding-type request parameter, 

804 # Amazon S3 includes this element in the response, and returns encoded key 

805 # name values in the following response elements: 

806 # Delimiter, Prefix, ContinuationToken, Key, and StartAfter. 

807 _decode_list_object( 

808 top_level_keys=['Delimiter', 'Prefix', 'StartAfter'], 

809 nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], 

810 parsed=parsed, 

811 context=context, 

812 ) 

813 

814 

815def decode_list_object_versions(parsed, context, **kwargs): 

816 # From the documentation: If you specify encoding-type request parameter, 

817 # Amazon S3 includes this element in the response, and returns encoded key 

818 # name values in the following response elements: 

819 # KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter. 

820 _decode_list_object( 

821 top_level_keys=[ 

822 'KeyMarker', 

823 'NextKeyMarker', 

824 'Prefix', 

825 'Delimiter', 

826 ], 

827 nested_keys=[ 

828 ('Versions', 'Key'), 

829 ('DeleteMarkers', 'Key'), 

830 ('CommonPrefixes', 'Prefix'), 

831 ], 

832 parsed=parsed, 

833 context=context, 

834 ) 

835 

836 

837def _decode_list_object(top_level_keys, nested_keys, parsed, context): 

838 if parsed.get('EncodingType') == 'url' and context.get( 

839 'encoding_type_auto_set' 

840 ): 

841 # URL decode top-level keys in the response if present. 

842 for key in top_level_keys: 

843 if key in parsed: 

844 parsed[key] = unquote_str(parsed[key]) 

845 # URL decode nested keys from the response if present. 

846 for top_key, child_key in nested_keys: 

847 if top_key in parsed: 

848 for member in parsed[top_key]: 

849 member[child_key] = unquote_str(member[child_key]) 

850 

851 

852def convert_body_to_file_like_object(params, **kwargs): 

853 if 'Body' in params: 

854 if isinstance(params['Body'], str): 

855 params['Body'] = BytesIO(ensure_bytes(params['Body'])) 

856 elif isinstance(params['Body'], bytes): 

857 params['Body'] = BytesIO(params['Body']) 

858 

859 

860def _add_parameter_aliases(handler_list): 

861 # Mapping of original parameter to parameter alias. 

862 # The key is <service>.<operation>.parameter 

863 # The first part of the key is used for event registration. 

864 # The last part is the original parameter name and the value is the 

865 # alias to expose in documentation. 

866 aliases = { 

867 'ec2.*.Filter': 'Filters', 

868 'logs.CreateExportTask.from': 'fromTime', 

869 'cloudsearchdomain.Search.return': 'returnFields', 

870 } 

871 

872 for original, new_name in aliases.items(): 

873 event_portion, original_name = original.rsplit('.', 1) 

874 parameter_alias = ParameterAlias(original_name, new_name) 

875 

876 # Add the handlers to the list of handlers. 

877 # One handler is to handle when users provide the alias. 

878 # The other handler is to update the documentation to show only 

879 # the alias. 

880 parameter_build_event_handler_tuple = ( 

881 'before-parameter-build.' + event_portion, 

882 parameter_alias.alias_parameter_in_call, 

883 REGISTER_FIRST, 

884 ) 

885 docs_event_handler_tuple = ( 

886 'docs.*.' + event_portion + '.complete-section', 

887 parameter_alias.alias_parameter_in_documentation, 

888 ) 

889 handler_list.append(parameter_build_event_handler_tuple) 

890 handler_list.append(docs_event_handler_tuple) 

891 

892 

893class ParameterAlias: 

894 def __init__(self, original_name, alias_name): 

895 self._original_name = original_name 

896 self._alias_name = alias_name 

897 

898 def alias_parameter_in_call(self, params, model, **kwargs): 

899 if model.input_shape: 

900 # Only consider accepting the alias if it is modeled in the 

901 # input shape. 

902 if self._original_name in model.input_shape.members: 

903 if self._alias_name in params: 

904 if self._original_name in params: 

905 raise AliasConflictParameterError( 

906 original=self._original_name, 

907 alias=self._alias_name, 

908 operation=model.name, 

909 ) 

910 # Remove the alias parameter value and use the old name 

911 # instead. 

912 params[self._original_name] = params.pop(self._alias_name) 

913 

914 def alias_parameter_in_documentation(self, event_name, section, **kwargs): 

915 if event_name.startswith('docs.request-params'): 

916 if self._original_name not in section.available_sections: 

917 return 

918 # Replace the name for parameter type 

919 param_section = section.get_section(self._original_name) 

920 param_type_section = param_section.get_section('param-type') 

921 self._replace_content(param_type_section) 

922