Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/botocore/awsrequest.py: 25%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
2# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"). You
5# may not use this file except in compliance with the License. A copy of
6# the License is located at
7#
8# http://aws.amazon.com/apache2.0/
9#
10# or in the "license" file accompanying this file. This file is
11# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12# ANY KIND, either express or implied. See the License for the specific
13# language governing permissions and limitations under the License.
14import functools
15import logging
16from collections.abc import Mapping
18import urllib3.util
19from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection
20from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
22import botocore.utils
23from botocore.compat import (
24 HTTPHeaders,
25 HTTPResponse,
26 MutableMapping,
27 urlencode,
28 urlparse,
29 urlsplit,
30 urlunsplit,
31)
32from botocore.exceptions import UnseekableStreamError
34logger = logging.getLogger(__name__)
37class AWSHTTPResponse(HTTPResponse):
38 # The *args, **kwargs is used because the args are slightly
39 # different in py2.6 than in py2.7/py3.
40 def __init__(self, *args, **kwargs):
41 self._status_tuple = kwargs.pop('status_tuple')
42 HTTPResponse.__init__(self, *args, **kwargs)
44 def _read_status(self):
45 if self._status_tuple is not None:
46 status_tuple = self._status_tuple
47 self._status_tuple = None
48 return status_tuple
49 else:
50 return HTTPResponse._read_status(self)
53class AWSConnection:
54 """Mixin for HTTPConnection that supports Expect 100-continue.
56 This when mixed with a subclass of httplib.HTTPConnection (though
57 technically we subclass from urllib3, which subclasses
58 httplib.HTTPConnection) and we only override this class to support Expect
59 100-continue, which we need for S3. As far as I can tell, this is
60 general purpose enough to not be specific to S3, but I'm being
61 tentative and keeping it in botocore because I've only tested
62 this against AWS services.
64 """
66 def __init__(self, *args, **kwargs):
67 super().__init__(*args, **kwargs)
68 self._original_response_cls = self.response_class
69 # This variable is set when we receive an early response from the
70 # server. If this value is set to True, any calls to send() are noops.
71 # This value is reset to false every time _send_request is called.
72 # This is to workaround changes in urllib3 2.0 which uses separate
73 # send() calls in request() instead of delegating to endheaders(),
74 # which is where the body is sent in CPython's HTTPConnection.
75 self._response_received = False
76 self._expect_header_set = False
77 self._send_called = False
79 def close(self):
80 super().close()
81 # Reset all of our instance state we were tracking.
82 self._response_received = False
83 self._expect_header_set = False
84 self._send_called = False
85 self.response_class = self._original_response_cls
87 def request(self, method, url, body=None, headers=None, *args, **kwargs):
88 if headers is None:
89 headers = {}
90 self._response_received = False
91 self.response_class = self._original_response_cls
92 if headers.get('Expect', b'') == b'100-continue':
93 self._expect_header_set = True
94 else:
95 self._expect_header_set = False
96 rval = super().request(method, url, body, headers, *args, **kwargs)
97 self._expect_header_set = False
98 return rval
100 def _convert_to_bytes(self, mixed_buffer):
101 # Take a list of mixed str/bytes and convert it
102 # all into a single bytestring.
103 # Any str will be encoded as utf-8.
104 bytes_buffer = []
105 for chunk in mixed_buffer:
106 if isinstance(chunk, str):
107 bytes_buffer.append(chunk.encode('utf-8'))
108 else:
109 bytes_buffer.append(chunk)
110 msg = b"\r\n".join(bytes_buffer)
111 return msg
113 def _send_output(self, message_body=None, *args, **kwargs):
114 self._buffer.extend((b"", b""))
115 msg = self._convert_to_bytes(self._buffer)
116 del self._buffer[:]
117 # If msg and message_body are sent in a single send() call,
118 # it will avoid performance problems caused by the interaction
119 # between delayed ack and the Nagle algorithm.
120 if isinstance(message_body, bytes):
121 msg += message_body
122 message_body = None
123 self.send(msg)
124 if self._expect_header_set:
125 # This is our custom behavior. If the Expect header was
126 # set, it will trigger this custom behavior.
127 logger.debug("Waiting for 100 Continue response.")
128 # Wait for 1 second for the server to send a response.
129 if urllib3.util.wait_for_read(self.sock, 1):
130 self._handle_expect_response(message_body)
131 return
132 else:
133 # From the RFC:
134 # Because of the presence of older implementations, the
135 # protocol allows ambiguous situations in which a client may
136 # send "Expect: 100-continue" without receiving either a 417
137 # (Expectation Failed) status or a 100 (Continue) status.
138 # Therefore, when a client sends this header field to an origin
139 # server (possibly via a proxy) from which it has never seen a
140 # 100 (Continue) status, the client SHOULD NOT wait for an
141 # indefinite period before sending the request body.
142 logger.debug(
143 "No response seen from server, continuing to "
144 "send the response body."
145 )
146 if message_body is not None:
147 # message_body was not a string (i.e. it is a file), and
148 # we must run the risk of Nagle.
149 self.send(message_body)
151 def _consume_headers(self, fp):
152 # Most servers (including S3) will just return
153 # the CLRF after the 100 continue response. However,
154 # some servers (I've specifically seen this for squid when
155 # used as a straight HTTP proxy) will also inject a
156 # Connection: keep-alive header. To account for this
157 # we'll read until we read '\r\n', and ignore any headers
158 # that come immediately after the 100 continue response.
159 current = None
160 while current != b'\r\n':
161 current = fp.readline()
163 def _handle_expect_response(self, message_body):
164 # This is called when we sent the request headers containing
165 # an Expect: 100-continue header and received a response.
166 # We now need to figure out what to do.
167 fp = self.sock.makefile('rb', 0)
168 try:
169 maybe_status_line = fp.readline()
170 parts = maybe_status_line.split(None, 2)
171 if self._is_100_continue_status(maybe_status_line):
172 self._consume_headers(fp)
173 logger.debug(
174 "100 Continue response seen, now sending request body."
175 )
176 self._send_message_body(message_body)
177 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):
178 # From the RFC:
179 # Requirements for HTTP/1.1 origin servers:
180 #
181 # - Upon receiving a request which includes an Expect
182 # request-header field with the "100-continue"
183 # expectation, an origin server MUST either respond with
184 # 100 (Continue) status and continue to read from the
185 # input stream, or respond with a final status code.
186 #
187 # So if we don't get a 100 Continue response, then
188 # whatever the server has sent back is the final response
189 # and don't send the message_body.
190 logger.debug(
191 "Received a non 100 Continue response "
192 "from the server, NOT sending request body."
193 )
194 status_tuple = (
195 parts[0].decode('ascii'),
196 int(parts[1]),
197 parts[2].decode('ascii'),
198 )
199 response_class = functools.partial(
200 AWSHTTPResponse, status_tuple=status_tuple
201 )
202 self.response_class = response_class
203 self._response_received = True
204 finally:
205 fp.close()
207 def _send_message_body(self, message_body):
208 if message_body is not None:
209 self.send(message_body)
211 def send(self, str):
212 if self._response_received:
213 if not self._send_called:
214 # urllib3 2.0 chunks and calls send potentially
215 # thousands of times inside `request` unlike the
216 # standard library. Only log this once for sanity.
217 logger.debug(
218 "send() called, but response already received. "
219 "Not sending data."
220 )
221 self._send_called = True
222 return
223 return super().send(str)
225 def _is_100_continue_status(self, maybe_status_line):
226 parts = maybe_status_line.split(None, 2)
227 # Check for HTTP/<version> 100 Continue\r\n or HTTP/<version> 100\r\n
228 return (
229 len(parts) >= 2
230 and parts[0].startswith(b'HTTP/')
231 and parts[1] == b'100'
232 )
235class AWSHTTPConnection(AWSConnection, HTTPConnection):
236 """An HTTPConnection that supports 100 Continue behavior."""
239class AWSHTTPSConnection(AWSConnection, VerifiedHTTPSConnection):
240 """An HTTPSConnection that supports 100 Continue behavior."""
243class AWSHTTPConnectionPool(HTTPConnectionPool):
244 ConnectionCls = AWSHTTPConnection
247class AWSHTTPSConnectionPool(HTTPSConnectionPool):
248 ConnectionCls = AWSHTTPSConnection
251def prepare_request_dict(
252 request_dict, endpoint_url, context=None, user_agent=None
253):
254 """
255 This method prepares a request dict to be created into an
256 AWSRequestObject. This prepares the request dict by adding the
257 url and the user agent to the request dict.
259 :type request_dict: dict
260 :param request_dict: The request dict (created from the
261 ``serialize`` module).
263 :type user_agent: string
264 :param user_agent: The user agent to use for this request.
266 :type endpoint_url: string
267 :param endpoint_url: The full endpoint url, which contains at least
268 the scheme, the hostname, and optionally any path components.
269 """
270 r = request_dict
271 if user_agent is not None:
272 headers = r['headers']
273 headers['User-Agent'] = user_agent
274 host_prefix = r.get('host_prefix')
275 url = _urljoin(endpoint_url, r['url_path'], host_prefix)
276 if r['query_string']:
277 # NOTE: This is to avoid circular import with utils. This is being
278 # done to avoid moving classes to different modules as to not cause
279 # breaking chainges.
280 percent_encode_sequence = botocore.utils.percent_encode_sequence
281 encoded_query_string = percent_encode_sequence(r['query_string'])
282 if '?' not in url:
283 url += f'?{encoded_query_string}'
284 else:
285 url += f'&{encoded_query_string}'
286 r['url'] = url
287 r['context'] = context
288 if context is None:
289 r['context'] = {}
292def create_request_object(request_dict):
293 """
294 This method takes a request dict and creates an AWSRequest object
295 from it.
297 :type request_dict: dict
298 :param request_dict: The request dict (created from the
299 ``prepare_request_dict`` method).
301 :rtype: ``botocore.awsrequest.AWSRequest``
302 :return: An AWSRequest object based on the request_dict.
304 """
305 r = request_dict
306 request_object = AWSRequest(
307 method=r['method'],
308 url=r['url'],
309 data=r['body'],
310 headers=r['headers'],
311 auth_path=r.get('auth_path'),
312 )
313 request_object.context = r['context']
314 return request_object
317def _urljoin(endpoint_url, url_path, host_prefix):
318 p = urlsplit(endpoint_url)
319 # <part> - <index>
320 # scheme - p[0]
321 # netloc - p[1]
322 # path - p[2]
323 # query - p[3]
324 # fragment - p[4]
325 if not url_path or url_path == '/':
326 # If there's no path component, ensure the URL ends with
327 # a '/' for backwards compatibility.
328 if not p[2]:
329 new_path = '/'
330 else:
331 new_path = p[2]
332 elif p[2].endswith('/') and url_path.startswith('/'):
333 new_path = p[2][:-1] + url_path
334 else:
335 new_path = p[2] + url_path
337 new_netloc = p[1]
338 if host_prefix is not None:
339 new_netloc = host_prefix + new_netloc
341 reconstructed = urlunsplit((p[0], new_netloc, new_path, p[3], p[4]))
342 return reconstructed
345class AWSRequestPreparer:
346 """
347 This class performs preparation on AWSRequest objects similar to that of
348 the PreparedRequest class does in the requests library. However, the logic
349 has been boiled down to meet the specific use cases in botocore. Of note
350 there are the following differences:
351 This class does not heavily prepare the URL. Requests performed many
352 validations and corrections to ensure the URL is properly formatted.
353 Botocore either performs these validations elsewhere or otherwise
354 consistently provides well formatted URLs.
356 This class does not heavily prepare the body. Body preperation is
357 simple and supports only the cases that we document: bytes and
358 file-like objects to determine the content-length. This will also
359 additionally prepare a body that is a dict to be url encoded params
360 string as some signers rely on this. Finally, this class does not
361 support multipart file uploads.
363 This class does not prepare the method, auth or cookies.
364 """
366 def prepare(self, original):
367 method = original.method
368 url = self._prepare_url(original)
369 body = self._prepare_body(original)
370 headers = self._prepare_headers(original, body)
371 stream_output = original.stream_output
372 context = original.context
374 return AWSPreparedRequest(
375 method, url, headers, body, stream_output, context
376 )
378 def _prepare_url(self, original):
379 url = original.url
380 if original.params:
381 url_parts = urlparse(url)
382 delim = '&' if url_parts.query else '?'
383 if isinstance(original.params, Mapping):
384 params_to_encode = list(original.params.items())
385 else:
386 params_to_encode = original.params
387 params = urlencode(params_to_encode, doseq=True)
388 url = delim.join((url, params))
389 return url
391 def _prepare_headers(self, original, prepared_body=None):
392 headers = HeadersDict(original.headers.items())
394 # If the transfer encoding or content length is already set, use that
395 if 'Transfer-Encoding' in headers or 'Content-Length' in headers:
396 return headers
398 # Ensure we set the content length when it is expected
399 if original.method not in ('GET', 'HEAD', 'OPTIONS'):
400 length = self._determine_content_length(prepared_body)
401 if length is not None:
402 headers['Content-Length'] = str(length)
403 else:
404 # Failed to determine content length, using chunked
405 # NOTE: This shouldn't ever happen in practice
406 body_type = type(prepared_body)
407 logger.debug('Failed to determine length of %s', body_type)
408 headers['Transfer-Encoding'] = 'chunked'
410 return headers
412 def _to_utf8(self, item):
413 key, value = item
414 if isinstance(key, str):
415 key = key.encode('utf-8')
416 if isinstance(value, str):
417 value = value.encode('utf-8')
418 return key, value
420 def _prepare_body(self, original):
421 """Prepares the given HTTP body data."""
422 body = original.data
423 if body == b'':
424 body = None
426 if isinstance(body, dict):
427 params = [self._to_utf8(item) for item in body.items()]
428 body = urlencode(params, doseq=True)
430 return body
432 def _determine_content_length(self, body):
433 return botocore.utils.determine_content_length(body)
436class AWSRequest:
437 """Represents the elements of an HTTP request.
439 This class was originally inspired by requests.models.Request, but has been
440 boiled down to meet the specific use cases in botocore. That being said this
441 class (even in requests) is effectively a named-tuple.
442 """
444 _REQUEST_PREPARER_CLS = AWSRequestPreparer
446 def __init__(
447 self,
448 method=None,
449 url=None,
450 headers=None,
451 data=None,
452 params=None,
453 auth_path=None,
454 stream_output=False,
455 ):
456 self._request_preparer = self._REQUEST_PREPARER_CLS()
458 # Default empty dicts for dict params.
459 params = {} if params is None else params
461 self.method = method
462 self.url = url
463 self.headers = HTTPHeaders()
464 self.data = data
465 self.params = params
466 self.auth_path = auth_path
467 self.stream_output = stream_output
469 if headers is not None:
470 for key, value in headers.items():
471 self.headers[key] = value
473 # This is a dictionary to hold information that is used when
474 # processing the request. What is inside of ``context`` is open-ended.
475 # For example, it may have a timestamp key that is used for holding
476 # what the timestamp is when signing the request. Note that none
477 # of the information that is inside of ``context`` is directly
478 # sent over the wire; the information is only used to assist in
479 # creating what is sent over the wire.
480 self.context = {}
482 def prepare(self):
483 """Constructs a :class:`AWSPreparedRequest <AWSPreparedRequest>`."""
484 return self._request_preparer.prepare(self)
486 @property
487 def body(self):
488 body = self.prepare().body
489 if isinstance(body, str):
490 body = body.encode('utf-8')
491 return body
494class AWSPreparedRequest:
495 """A data class representing a finalized request to be sent over the wire.
497 Requests at this stage should be treated as final, and the properties of
498 the request should not be modified.
500 :ivar method: The HTTP Method
501 :ivar url: The full url
502 :ivar headers: The HTTP headers to send.
503 :ivar body: The HTTP body.
504 :ivar stream_output: If the response for this request should be streamed.
505 :ivar context: The request context from the originating ``AWSRequest``.
506 """
508 def __init__(
509 self, method, url, headers, body, stream_output, context=None
510 ):
511 self.method = method
512 self.url = url
513 self.headers = headers
514 self.body = body
515 self.stream_output = stream_output
516 self.context = context
518 def __repr__(self):
519 fmt = (
520 '<AWSPreparedRequest stream_output=%s, method=%s, url=%s, '
521 'headers=%s>'
522 )
523 return fmt % (self.stream_output, self.method, self.url, self.headers)
525 def reset_stream(self):
526 """Resets the streaming body to it's initial position.
528 If the request contains a streaming body (a streamable file-like object)
529 seek to the object's initial position to ensure the entire contents of
530 the object is sent. This is a no-op for static bytes-like body types.
531 """
532 # Trying to reset a stream when there is a no stream will
533 # just immediately return. It's not an error, it will produce
534 # the same result as if we had actually reset the stream (we'll send
535 # the entire body contents again if we need to).
536 # Same case if the body is a string/bytes/bytearray type.
538 non_seekable_types = (bytes, str, bytearray)
539 if self.body is None or isinstance(self.body, non_seekable_types):
540 return
541 try:
542 logger.debug("Rewinding stream: %s", self.body)
543 self.body.seek(0)
544 except Exception as e:
545 logger.debug("Unable to rewind stream: %s", e)
546 raise UnseekableStreamError(stream_object=self.body)
549class AWSResponse:
550 """A data class representing an HTTP response.
552 This class was originally inspired by requests.models.Response, but has
553 been boiled down to meet the specific use cases in botocore. This has
554 effectively been reduced to a named tuple.
556 :ivar url: The full url.
557 :ivar status_code: The status code of the HTTP response.
558 :ivar headers: The HTTP headers received.
559 :ivar body: The HTTP response body.
560 """
562 def __init__(self, url, status_code, headers, raw):
563 self.url = url
564 self.status_code = status_code
565 self.headers = HeadersDict(headers)
566 self.raw = raw
568 self._content = None
570 @property
571 def content(self):
572 """Content of the response as bytes."""
574 if self._content is None:
575 # Read the contents.
576 # NOTE: requests would attempt to call stream and fall back
577 # to a custom generator that would call read in a loop, but
578 # we don't rely on this behavior
579 self._content = b''.join(self.raw.stream()) or b''
581 return self._content
583 @property
584 def text(self):
585 """Content of the response as a proper text type.
587 Uses the encoding type provided in the reponse headers to decode the
588 response content into a proper text type. If the encoding is not
589 present in the headers, UTF-8 is used as a default.
590 """
591 encoding = botocore.utils.get_encoding_from_headers(self.headers)
592 if encoding:
593 return self.content.decode(encoding)
594 else:
595 return self.content.decode('utf-8')
598class _HeaderKey:
599 def __init__(self, key):
600 self._key = key
601 self._lower = key.lower()
603 def __hash__(self):
604 return hash(self._lower)
606 def __eq__(self, other):
607 return isinstance(other, _HeaderKey) and self._lower == other._lower
609 def __str__(self):
610 return self._key
612 def __repr__(self):
613 return repr(self._key)
616class HeadersDict(MutableMapping):
617 """A case-insenseitive dictionary to represent HTTP headers."""
619 def __init__(self, *args, **kwargs):
620 self._dict = {}
621 self.update(*args, **kwargs)
623 def __setitem__(self, key, value):
624 self._dict[_HeaderKey(key)] = value
626 def __getitem__(self, key):
627 return self._dict[_HeaderKey(key)]
629 def __delitem__(self, key):
630 del self._dict[_HeaderKey(key)]
632 def __iter__(self):
633 return (str(key) for key in self._dict)
635 def __len__(self):
636 return len(self._dict)
638 def __repr__(self):
639 return repr(self._dict)
641 def copy(self):
642 return HeadersDict(self.items())