Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/awsrequest.py: 26%
277 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-26 06:03 +0000
1# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
2# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"). You
5# may not use this file except in compliance with the License. A copy of
6# the License is located at
7#
8# http://aws.amazon.com/apache2.0/
9#
10# or in the "license" file accompanying this file. This file is
11# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12# ANY KIND, either express or implied. See the License for the specific
13# language governing permissions and limitations under the License.
14import functools
15import logging
16from collections.abc import Mapping
18import urllib3.util
19from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection
20from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
22import botocore.utils
23from botocore.compat import (
24 HTTPHeaders,
25 HTTPResponse,
26 MutableMapping,
27 urlencode,
28 urlparse,
29 urlsplit,
30 urlunsplit,
31)
32from botocore.exceptions import UnseekableStreamError
34logger = logging.getLogger(__name__)
37class AWSHTTPResponse(HTTPResponse):
38 # The *args, **kwargs is used because the args are slightly
39 # different in py2.6 than in py2.7/py3.
40 def __init__(self, *args, **kwargs):
41 self._status_tuple = kwargs.pop('status_tuple')
42 HTTPResponse.__init__(self, *args, **kwargs)
44 def _read_status(self):
45 if self._status_tuple is not None:
46 status_tuple = self._status_tuple
47 self._status_tuple = None
48 return status_tuple
49 else:
50 return HTTPResponse._read_status(self)
53class AWSConnection:
54 """Mixin for HTTPConnection that supports Expect 100-continue.
56 This when mixed with a subclass of httplib.HTTPConnection (though
57 technically we subclass from urllib3, which subclasses
58 httplib.HTTPConnection) and we only override this class to support Expect
59 100-continue, which we need for S3. As far as I can tell, this is
60 general purpose enough to not be specific to S3, but I'm being
61 tentative and keeping it in botocore because I've only tested
62 this against AWS services.
64 """
66 def __init__(self, *args, **kwargs):
67 super().__init__(*args, **kwargs)
68 self._original_response_cls = self.response_class
69 # We'd ideally hook into httplib's states, but they're all
70 # __mangled_vars so we use our own state var. This variable is set
71 # when we receive an early response from the server. If this value is
72 # set to True, any calls to send() are noops. This value is reset to
73 # false every time _send_request is called. This is to workaround the
74 # fact that py2.6 (and only py2.6) has a separate send() call for the
75 # body in _send_request, as opposed to endheaders(), which is where the
76 # body is sent in all versions > 2.6.
77 self._response_received = False
78 self._expect_header_set = False
80 def close(self):
81 super().close()
82 # Reset all of our instance state we were tracking.
83 self._response_received = False
84 self._expect_header_set = False
85 self.response_class = self._original_response_cls
87 def _send_request(self, method, url, body, headers, *args, **kwargs):
88 self._response_received = False
89 if headers.get('Expect', b'') == b'100-continue':
90 self._expect_header_set = True
91 else:
92 self._expect_header_set = False
93 self.response_class = self._original_response_cls
94 rval = super()._send_request(
95 method, url, body, headers, *args, **kwargs
96 )
97 self._expect_header_set = False
98 return rval
100 def _convert_to_bytes(self, mixed_buffer):
101 # Take a list of mixed str/bytes and convert it
102 # all into a single bytestring.
103 # Any str will be encoded as utf-8.
104 bytes_buffer = []
105 for chunk in mixed_buffer:
106 if isinstance(chunk, str):
107 bytes_buffer.append(chunk.encode('utf-8'))
108 else:
109 bytes_buffer.append(chunk)
110 msg = b"\r\n".join(bytes_buffer)
111 return msg
113 def _send_output(self, message_body=None, *args, **kwargs):
114 self._buffer.extend((b"", b""))
115 msg = self._convert_to_bytes(self._buffer)
116 del self._buffer[:]
117 # If msg and message_body are sent in a single send() call,
118 # it will avoid performance problems caused by the interaction
119 # between delayed ack and the Nagle algorithm.
120 if isinstance(message_body, bytes):
121 msg += message_body
122 message_body = None
123 self.send(msg)
124 if self._expect_header_set:
125 # This is our custom behavior. If the Expect header was
126 # set, it will trigger this custom behavior.
127 logger.debug("Waiting for 100 Continue response.")
128 # Wait for 1 second for the server to send a response.
129 if urllib3.util.wait_for_read(self.sock, 1):
130 self._handle_expect_response(message_body)
131 return
132 else:
133 # From the RFC:
134 # Because of the presence of older implementations, the
135 # protocol allows ambiguous situations in which a client may
136 # send "Expect: 100-continue" without receiving either a 417
137 # (Expectation Failed) status or a 100 (Continue) status.
138 # Therefore, when a client sends this header field to an origin
139 # server (possibly via a proxy) from which it has never seen a
140 # 100 (Continue) status, the client SHOULD NOT wait for an
141 # indefinite period before sending the request body.
142 logger.debug(
143 "No response seen from server, continuing to "
144 "send the response body."
145 )
146 if message_body is not None:
147 # message_body was not a string (i.e. it is a file), and
148 # we must run the risk of Nagle.
149 self.send(message_body)
151 def _consume_headers(self, fp):
152 # Most servers (including S3) will just return
153 # the CLRF after the 100 continue response. However,
154 # some servers (I've specifically seen this for squid when
155 # used as a straight HTTP proxy) will also inject a
156 # Connection: keep-alive header. To account for this
157 # we'll read until we read '\r\n', and ignore any headers
158 # that come immediately after the 100 continue response.
159 current = None
160 while current != b'\r\n':
161 current = fp.readline()
163 def _handle_expect_response(self, message_body):
164 # This is called when we sent the request headers containing
165 # an Expect: 100-continue header and received a response.
166 # We now need to figure out what to do.
167 fp = self.sock.makefile('rb', 0)
168 try:
169 maybe_status_line = fp.readline()
170 parts = maybe_status_line.split(None, 2)
171 if self._is_100_continue_status(maybe_status_line):
172 self._consume_headers(fp)
173 logger.debug(
174 "100 Continue response seen, now sending request body."
175 )
176 self._send_message_body(message_body)
177 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):
178 # From the RFC:
179 # Requirements for HTTP/1.1 origin servers:
180 #
181 # - Upon receiving a request which includes an Expect
182 # request-header field with the "100-continue"
183 # expectation, an origin server MUST either respond with
184 # 100 (Continue) status and continue to read from the
185 # input stream, or respond with a final status code.
186 #
187 # So if we don't get a 100 Continue response, then
188 # whatever the server has sent back is the final response
189 # and don't send the message_body.
190 logger.debug(
191 "Received a non 100 Continue response "
192 "from the server, NOT sending request body."
193 )
194 status_tuple = (
195 parts[0].decode('ascii'),
196 int(parts[1]),
197 parts[2].decode('ascii'),
198 )
199 response_class = functools.partial(
200 AWSHTTPResponse, status_tuple=status_tuple
201 )
202 self.response_class = response_class
203 self._response_received = True
204 finally:
205 fp.close()
207 def _send_message_body(self, message_body):
208 if message_body is not None:
209 self.send(message_body)
211 def send(self, str):
212 if self._response_received:
213 logger.debug(
214 "send() called, but reseponse already received. "
215 "Not sending data."
216 )
217 return
218 return super().send(str)
220 def _is_100_continue_status(self, maybe_status_line):
221 parts = maybe_status_line.split(None, 2)
222 # Check for HTTP/<version> 100 Continue\r\n
223 return (
224 len(parts) >= 3
225 and parts[0].startswith(b'HTTP/')
226 and parts[1] == b'100'
227 )
230class AWSHTTPConnection(AWSConnection, HTTPConnection):
231 """An HTTPConnection that supports 100 Continue behavior."""
234class AWSHTTPSConnection(AWSConnection, VerifiedHTTPSConnection):
235 """An HTTPSConnection that supports 100 Continue behavior."""
238class AWSHTTPConnectionPool(HTTPConnectionPool):
239 ConnectionCls = AWSHTTPConnection
242class AWSHTTPSConnectionPool(HTTPSConnectionPool):
243 ConnectionCls = AWSHTTPSConnection
246def prepare_request_dict(
247 request_dict, endpoint_url, context=None, user_agent=None
248):
249 """
250 This method prepares a request dict to be created into an
251 AWSRequestObject. This prepares the request dict by adding the
252 url and the user agent to the request dict.
254 :type request_dict: dict
255 :param request_dict: The request dict (created from the
256 ``serialize`` module).
258 :type user_agent: string
259 :param user_agent: The user agent to use for this request.
261 :type endpoint_url: string
262 :param endpoint_url: The full endpoint url, which contains at least
263 the scheme, the hostname, and optionally any path components.
264 """
265 r = request_dict
266 if user_agent is not None:
267 headers = r['headers']
268 headers['User-Agent'] = user_agent
269 host_prefix = r.get('host_prefix')
270 url = _urljoin(endpoint_url, r['url_path'], host_prefix)
271 if r['query_string']:
272 # NOTE: This is to avoid circular import with utils. This is being
273 # done to avoid moving classes to different modules as to not cause
274 # breaking chainges.
275 percent_encode_sequence = botocore.utils.percent_encode_sequence
276 encoded_query_string = percent_encode_sequence(r['query_string'])
277 if '?' not in url:
278 url += '?%s' % encoded_query_string
279 else:
280 url += '&%s' % encoded_query_string
281 r['url'] = url
282 r['context'] = context
283 if context is None:
284 r['context'] = {}
287def create_request_object(request_dict):
288 """
289 This method takes a request dict and creates an AWSRequest object
290 from it.
292 :type request_dict: dict
293 :param request_dict: The request dict (created from the
294 ``prepare_request_dict`` method).
296 :rtype: ``botocore.awsrequest.AWSRequest``
297 :return: An AWSRequest object based on the request_dict.
299 """
300 r = request_dict
301 request_object = AWSRequest(
302 method=r['method'],
303 url=r['url'],
304 data=r['body'],
305 headers=r['headers'],
306 auth_path=r.get('auth_path'),
307 )
308 request_object.context = r['context']
309 return request_object
312def _urljoin(endpoint_url, url_path, host_prefix):
313 p = urlsplit(endpoint_url)
314 # <part> - <index>
315 # scheme - p[0]
316 # netloc - p[1]
317 # path - p[2]
318 # query - p[3]
319 # fragment - p[4]
320 if not url_path or url_path == '/':
321 # If there's no path component, ensure the URL ends with
322 # a '/' for backwards compatibility.
323 if not p[2]:
324 new_path = '/'
325 else:
326 new_path = p[2]
327 elif p[2].endswith('/') and url_path.startswith('/'):
328 new_path = p[2][:-1] + url_path
329 else:
330 new_path = p[2] + url_path
332 new_netloc = p[1]
333 if host_prefix is not None:
334 new_netloc = host_prefix + new_netloc
336 reconstructed = urlunsplit((p[0], new_netloc, new_path, p[3], p[4]))
337 return reconstructed
340class AWSRequestPreparer:
341 """
342 This class performs preparation on AWSRequest objects similar to that of
343 the PreparedRequest class does in the requests library. However, the logic
344 has been boiled down to meet the specific use cases in botocore. Of note
345 there are the following differences:
346 This class does not heavily prepare the URL. Requests performed many
347 validations and corrections to ensure the URL is properly formatted.
348 Botocore either performs these validations elsewhere or otherwise
349 consistently provides well formatted URLs.
351 This class does not heavily prepare the body. Body preperation is
352 simple and supports only the cases that we document: bytes and
353 file-like objects to determine the content-length. This will also
354 additionally prepare a body that is a dict to be url encoded params
355 string as some signers rely on this. Finally, this class does not
356 support multipart file uploads.
358 This class does not prepare the method, auth or cookies.
359 """
361 def prepare(self, original):
362 method = original.method
363 url = self._prepare_url(original)
364 body = self._prepare_body(original)
365 headers = self._prepare_headers(original, body)
366 stream_output = original.stream_output
368 return AWSPreparedRequest(method, url, headers, body, stream_output)
370 def _prepare_url(self, original):
371 url = original.url
372 if original.params:
373 url_parts = urlparse(url)
374 delim = '&' if url_parts.query else '?'
375 if isinstance(original.params, Mapping):
376 params_to_encode = list(original.params.items())
377 else:
378 params_to_encode = original.params
379 params = urlencode(params_to_encode, doseq=True)
380 url = delim.join((url, params))
381 return url
383 def _prepare_headers(self, original, prepared_body=None):
384 headers = HeadersDict(original.headers.items())
386 # If the transfer encoding or content length is already set, use that
387 if 'Transfer-Encoding' in headers or 'Content-Length' in headers:
388 return headers
390 # Ensure we set the content length when it is expected
391 if original.method not in ('GET', 'HEAD', 'OPTIONS'):
392 length = self._determine_content_length(prepared_body)
393 if length is not None:
394 headers['Content-Length'] = str(length)
395 else:
396 # Failed to determine content length, using chunked
397 # NOTE: This shouldn't ever happen in practice
398 body_type = type(prepared_body)
399 logger.debug('Failed to determine length of %s', body_type)
400 headers['Transfer-Encoding'] = 'chunked'
402 return headers
404 def _to_utf8(self, item):
405 key, value = item
406 if isinstance(key, str):
407 key = key.encode('utf-8')
408 if isinstance(value, str):
409 value = value.encode('utf-8')
410 return key, value
412 def _prepare_body(self, original):
413 """Prepares the given HTTP body data."""
414 body = original.data
415 if body == b'':
416 body = None
418 if isinstance(body, dict):
419 params = [self._to_utf8(item) for item in body.items()]
420 body = urlencode(params, doseq=True)
422 return body
424 def _determine_content_length(self, body):
425 return botocore.utils.determine_content_length(body)
428class AWSRequest:
429 """Represents the elements of an HTTP request.
431 This class was originally inspired by requests.models.Request, but has been
432 boiled down to meet the specific use cases in botocore. That being said this
433 class (even in requests) is effectively a named-tuple.
434 """
436 _REQUEST_PREPARER_CLS = AWSRequestPreparer
438 def __init__(
439 self,
440 method=None,
441 url=None,
442 headers=None,
443 data=None,
444 params=None,
445 auth_path=None,
446 stream_output=False,
447 ):
449 self._request_preparer = self._REQUEST_PREPARER_CLS()
451 # Default empty dicts for dict params.
452 params = {} if params is None else params
454 self.method = method
455 self.url = url
456 self.headers = HTTPHeaders()
457 self.data = data
458 self.params = params
459 self.auth_path = auth_path
460 self.stream_output = stream_output
462 if headers is not None:
463 for key, value in headers.items():
464 self.headers[key] = value
466 # This is a dictionary to hold information that is used when
467 # processing the request. What is inside of ``context`` is open-ended.
468 # For example, it may have a timestamp key that is used for holding
469 # what the timestamp is when signing the request. Note that none
470 # of the information that is inside of ``context`` is directly
471 # sent over the wire; the information is only used to assist in
472 # creating what is sent over the wire.
473 self.context = {}
475 def prepare(self):
476 """Constructs a :class:`AWSPreparedRequest <AWSPreparedRequest>`."""
477 return self._request_preparer.prepare(self)
479 @property
480 def body(self):
481 body = self.prepare().body
482 if isinstance(body, str):
483 body = body.encode('utf-8')
484 return body
487class AWSPreparedRequest:
488 """A data class representing a finalized request to be sent over the wire.
490 Requests at this stage should be treated as final, and the properties of
491 the request should not be modified.
493 :ivar method: The HTTP Method
494 :ivar url: The full url
495 :ivar headers: The HTTP headers to send.
496 :ivar body: The HTTP body.
497 :ivar stream_output: If the response for this request should be streamed.
498 """
500 def __init__(self, method, url, headers, body, stream_output):
501 self.method = method
502 self.url = url
503 self.headers = headers
504 self.body = body
505 self.stream_output = stream_output
507 def __repr__(self):
508 fmt = (
509 '<AWSPreparedRequest stream_output=%s, method=%s, url=%s, '
510 'headers=%s>'
511 )
512 return fmt % (self.stream_output, self.method, self.url, self.headers)
514 def reset_stream(self):
515 """Resets the streaming body to it's initial position.
517 If the request contains a streaming body (a streamable file-like object)
518 seek to the object's initial position to ensure the entire contents of
519 the object is sent. This is a no-op for static bytes-like body types.
520 """
521 # Trying to reset a stream when there is a no stream will
522 # just immediately return. It's not an error, it will produce
523 # the same result as if we had actually reset the stream (we'll send
524 # the entire body contents again if we need to).
525 # Same case if the body is a string/bytes/bytearray type.
527 non_seekable_types = (bytes, str, bytearray)
528 if self.body is None or isinstance(self.body, non_seekable_types):
529 return
530 try:
531 logger.debug("Rewinding stream: %s", self.body)
532 self.body.seek(0)
533 except Exception as e:
534 logger.debug("Unable to rewind stream: %s", e)
535 raise UnseekableStreamError(stream_object=self.body)
538class AWSResponse:
539 """A data class representing an HTTP response.
541 This class was originally inspired by requests.models.Response, but has
542 been boiled down to meet the specific use cases in botocore. This has
543 effectively been reduced to a named tuple.
545 :ivar url: The full url.
546 :ivar status_code: The status code of the HTTP response.
547 :ivar headers: The HTTP headers received.
548 :ivar body: The HTTP response body.
549 """
551 def __init__(self, url, status_code, headers, raw):
552 self.url = url
553 self.status_code = status_code
554 self.headers = HeadersDict(headers)
555 self.raw = raw
557 self._content = None
559 @property
560 def content(self):
561 """Content of the response as bytes."""
563 if self._content is None:
564 # Read the contents.
565 # NOTE: requests would attempt to call stream and fall back
566 # to a custom generator that would call read in a loop, but
567 # we don't rely on this behavior
568 self._content = bytes().join(self.raw.stream()) or bytes()
570 return self._content
572 @property
573 def text(self):
574 """Content of the response as a proper text type.
576 Uses the encoding type provided in the reponse headers to decode the
577 response content into a proper text type. If the encoding is not
578 present in the headers, UTF-8 is used as a default.
579 """
580 encoding = botocore.utils.get_encoding_from_headers(self.headers)
581 if encoding:
582 return self.content.decode(encoding)
583 else:
584 return self.content.decode('utf-8')
587class _HeaderKey:
588 def __init__(self, key):
589 self._key = key
590 self._lower = key.lower()
592 def __hash__(self):
593 return hash(self._lower)
595 def __eq__(self, other):
596 return isinstance(other, _HeaderKey) and self._lower == other._lower
598 def __str__(self):
599 return self._key
601 def __repr__(self):
602 return repr(self._key)
605class HeadersDict(MutableMapping):
606 """A case-insenseitive dictionary to represent HTTP headers."""
608 def __init__(self, *args, **kwargs):
609 self._dict = {}
610 self.update(*args, **kwargs)
612 def __setitem__(self, key, value):
613 self._dict[_HeaderKey(key)] = value
615 def __getitem__(self, key):
616 return self._dict[_HeaderKey(key)]
618 def __delitem__(self, key):
619 del self._dict[_HeaderKey(key)]
621 def __iter__(self):
622 return (str(key) for key in self._dict)
624 def __len__(self):
625 return len(self._dict)
627 def __repr__(self):
628 return repr(self._dict)
630 def copy(self):
631 return HeadersDict(self.items())