Coverage for /pythoncovmergedfiles/medio/medio/src/aiohttp/aiohttp/client.py: 26%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""HTTP Client for asyncio."""
3import asyncio
4import base64
5import dataclasses
6import hashlib
7import json
8import os
9import sys
10import traceback
11import warnings
12from contextlib import suppress
13from types import TracebackType
14from typing import (
15 TYPE_CHECKING,
16 Any,
17 Awaitable,
18 Callable,
19 Collection,
20 Coroutine,
21 Final,
22 FrozenSet,
23 Generator,
24 Generic,
25 Iterable,
26 List,
27 Mapping,
28 Optional,
29 Sequence,
30 Set,
31 Tuple,
32 Type,
33 TypedDict,
34 TypeVar,
35 Union,
36 final,
37)
39from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr
40from yarl import URL
42from . import hdrs, http, payload
43from ._websocket.reader import WebSocketDataQueue
44from .abc import AbstractCookieJar
45from .client_exceptions import (
46 ClientConnectionError,
47 ClientConnectionResetError,
48 ClientConnectorCertificateError,
49 ClientConnectorDNSError,
50 ClientConnectorError,
51 ClientConnectorSSLError,
52 ClientError,
53 ClientHttpProxyError,
54 ClientOSError,
55 ClientPayloadError,
56 ClientProxyConnectionError,
57 ClientResponseError,
58 ClientSSLError,
59 ConnectionTimeoutError,
60 ContentTypeError,
61 InvalidURL,
62 InvalidUrlClientError,
63 InvalidUrlRedirectClientError,
64 NonHttpUrlClientError,
65 NonHttpUrlRedirectClientError,
66 RedirectClientError,
67 ServerConnectionError,
68 ServerDisconnectedError,
69 ServerFingerprintMismatch,
70 ServerTimeoutError,
71 SocketTimeoutError,
72 TooManyRedirects,
73 WSMessageTypeError,
74 WSServerHandshakeError,
75)
76from .client_middlewares import ClientMiddlewareType, build_client_middlewares
77from .client_reqrep import (
78 SSL_ALLOWED_TYPES,
79 ClientRequest,
80 ClientResponse,
81 Fingerprint,
82 RequestInfo,
83)
84from .client_ws import (
85 DEFAULT_WS_CLIENT_TIMEOUT,
86 ClientWebSocketResponse,
87 ClientWSTimeout,
88)
89from .connector import (
90 HTTP_AND_EMPTY_SCHEMA_SET,
91 BaseConnector,
92 NamedPipeConnector,
93 TCPConnector,
94 UnixConnector,
95)
96from .cookiejar import CookieJar
97from .helpers import (
98 _SENTINEL,
99 EMPTY_BODY_METHODS,
100 BasicAuth,
101 TimeoutHandle,
102 frozen_dataclass_decorator,
103 get_env_proxy_for_url,
104 sentinel,
105 strip_auth_from_url,
106)
107from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter
108from .http_websocket import WSHandshakeError, ws_ext_gen, ws_ext_parse
109from .tracing import Trace, TraceConfig
110from .typedefs import JSONEncoder, LooseCookies, LooseHeaders, Query, StrOrURL
112__all__ = (
113 # client_exceptions
114 "ClientConnectionError",
115 "ClientConnectionResetError",
116 "ClientConnectorCertificateError",
117 "ClientConnectorDNSError",
118 "ClientConnectorError",
119 "ClientConnectorSSLError",
120 "ClientError",
121 "ClientHttpProxyError",
122 "ClientOSError",
123 "ClientPayloadError",
124 "ClientProxyConnectionError",
125 "ClientResponseError",
126 "ClientSSLError",
127 "ConnectionTimeoutError",
128 "ContentTypeError",
129 "InvalidURL",
130 "InvalidUrlClientError",
131 "RedirectClientError",
132 "NonHttpUrlClientError",
133 "InvalidUrlRedirectClientError",
134 "NonHttpUrlRedirectClientError",
135 "ServerConnectionError",
136 "ServerDisconnectedError",
137 "ServerFingerprintMismatch",
138 "ServerTimeoutError",
139 "SocketTimeoutError",
140 "TooManyRedirects",
141 "WSServerHandshakeError",
142 # client_reqrep
143 "ClientRequest",
144 "ClientResponse",
145 "Fingerprint",
146 "RequestInfo",
147 # connector
148 "BaseConnector",
149 "TCPConnector",
150 "UnixConnector",
151 "NamedPipeConnector",
152 # client_ws
153 "ClientWebSocketResponse",
154 # client
155 "ClientSession",
156 "ClientTimeout",
157 "ClientWSTimeout",
158 "request",
159 "WSMessageTypeError",
160)
163if TYPE_CHECKING:
164 from ssl import SSLContext
165else:
166 SSLContext = None
168if sys.version_info >= (3, 11) and TYPE_CHECKING:
169 from typing import Unpack
172class _RequestOptions(TypedDict, total=False):
173 params: Query
174 data: Any
175 json: Any
176 cookies: Union[LooseCookies, None]
177 headers: Union[LooseHeaders, None]
178 skip_auto_headers: Union[Iterable[str], None]
179 auth: Union[BasicAuth, None]
180 allow_redirects: bool
181 max_redirects: int
182 compress: Union[str, bool]
183 chunked: Union[bool, None]
184 expect100: bool
185 raise_for_status: Union[None, bool, Callable[[ClientResponse], Awaitable[None]]]
186 read_until_eof: bool
187 proxy: Union[StrOrURL, None]
188 proxy_auth: Union[BasicAuth, None]
189 timeout: "Union[ClientTimeout, _SENTINEL, None]"
190 ssl: Union[SSLContext, bool, Fingerprint]
191 server_hostname: Union[str, None]
192 proxy_headers: Union[LooseHeaders, None]
193 trace_request_ctx: Union[Mapping[str, Any], None]
194 read_bufsize: Union[int, None]
195 auto_decompress: Union[bool, None]
196 max_line_size: Union[int, None]
197 max_field_size: Union[int, None]
198 middlewares: Optional[Sequence[ClientMiddlewareType]]
201@frozen_dataclass_decorator
202class ClientTimeout:
203 total: Optional[float] = None
204 connect: Optional[float] = None
205 sock_read: Optional[float] = None
206 sock_connect: Optional[float] = None
207 ceil_threshold: float = 5
209 # pool_queue_timeout: Optional[float] = None
210 # dns_resolution_timeout: Optional[float] = None
211 # socket_connect_timeout: Optional[float] = None
212 # connection_acquiring_timeout: Optional[float] = None
213 # new_connection_timeout: Optional[float] = None
214 # http_header_timeout: Optional[float] = None
215 # response_body_timeout: Optional[float] = None
217 # to create a timeout specific for a single request, either
218 # - create a completely new one to overwrite the default
219 # - or use https://docs.python.org/3/library/dataclasses.html#dataclasses.replace
220 # to overwrite the defaults
223# 5 Minute default read timeout
224DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60, sock_connect=30)
226# https://www.rfc-editor.org/rfc/rfc9110#section-9.2.2
227IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE"})
229_RetType = TypeVar("_RetType", ClientResponse, ClientWebSocketResponse)
230_CharsetResolver = Callable[[ClientResponse, bytes], str]
233@final
234class ClientSession:
235 """First-class interface for making HTTP requests."""
237 __slots__ = (
238 "_base_url",
239 "_base_url_origin",
240 "_source_traceback",
241 "_connector",
242 "_loop",
243 "_cookie_jar",
244 "_connector_owner",
245 "_default_auth",
246 "_version",
247 "_json_serialize",
248 "_requote_redirect_url",
249 "_timeout",
250 "_raise_for_status",
251 "_auto_decompress",
252 "_trust_env",
253 "_default_headers",
254 "_skip_auto_headers",
255 "_request_class",
256 "_response_class",
257 "_ws_response_class",
258 "_trace_configs",
259 "_read_bufsize",
260 "_max_line_size",
261 "_max_field_size",
262 "_resolve_charset",
263 "_default_proxy",
264 "_default_proxy_auth",
265 "_retry_connection",
266 "_middlewares",
267 )
269 def __init__(
270 self,
271 base_url: Optional[StrOrURL] = None,
272 *,
273 connector: Optional[BaseConnector] = None,
274 cookies: Optional[LooseCookies] = None,
275 headers: Optional[LooseHeaders] = None,
276 proxy: Optional[StrOrURL] = None,
277 proxy_auth: Optional[BasicAuth] = None,
278 skip_auto_headers: Optional[Iterable[str]] = None,
279 auth: Optional[BasicAuth] = None,
280 json_serialize: JSONEncoder = json.dumps,
281 request_class: Type[ClientRequest] = ClientRequest,
282 response_class: Type[ClientResponse] = ClientResponse,
283 ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse,
284 version: HttpVersion = http.HttpVersion11,
285 cookie_jar: Optional[AbstractCookieJar] = None,
286 connector_owner: bool = True,
287 raise_for_status: Union[
288 bool, Callable[[ClientResponse], Awaitable[None]]
289 ] = False,
290 timeout: Union[_SENTINEL, ClientTimeout, None] = sentinel,
291 auto_decompress: bool = True,
292 trust_env: bool = False,
293 requote_redirect_url: bool = True,
294 trace_configs: Optional[List[TraceConfig[object]]] = None,
295 read_bufsize: int = 2**16,
296 max_line_size: int = 8190,
297 max_field_size: int = 8190,
298 fallback_charset_resolver: _CharsetResolver = lambda r, b: "utf-8",
299 middlewares: Sequence[ClientMiddlewareType] = (),
300 ) -> None:
301 # We initialise _connector to None immediately, as it's referenced in __del__()
302 # and could cause issues if an exception occurs during initialisation.
303 self._connector: Optional[BaseConnector] = None
304 if base_url is None or isinstance(base_url, URL):
305 self._base_url: Optional[URL] = base_url
306 self._base_url_origin = None if base_url is None else base_url.origin()
307 else:
308 self._base_url = URL(base_url)
309 self._base_url_origin = self._base_url.origin()
310 assert self._base_url.absolute, "Only absolute URLs are supported"
311 if self._base_url is not None and not self._base_url.path.endswith("/"):
312 raise ValueError("base_url must have a trailing '/'")
314 loop = asyncio.get_running_loop()
316 if timeout is sentinel or timeout is None:
317 timeout = DEFAULT_TIMEOUT
318 if not isinstance(timeout, ClientTimeout):
319 raise ValueError(
320 f"timeout parameter cannot be of {type(timeout)} type, "
321 "please use 'timeout=ClientTimeout(...)'",
322 )
323 self._timeout = timeout
325 if connector is None:
326 connector = TCPConnector()
327 # Initialize these three attrs before raising any exception,
328 # they are used in __del__
329 self._connector = connector
330 self._loop = loop
331 if loop.get_debug():
332 self._source_traceback: Optional[traceback.StackSummary] = (
333 traceback.extract_stack(sys._getframe(1))
334 )
335 else:
336 self._source_traceback = None
338 if connector._loop is not loop:
339 raise RuntimeError("Session and connector have to use same event loop")
341 if cookie_jar is None:
342 cookie_jar = CookieJar()
343 self._cookie_jar = cookie_jar
345 if cookies:
346 self._cookie_jar.update_cookies(cookies)
348 self._connector_owner = connector_owner
349 self._default_auth = auth
350 self._version = version
351 self._json_serialize = json_serialize
352 self._raise_for_status = raise_for_status
353 self._auto_decompress = auto_decompress
354 self._trust_env = trust_env
355 self._requote_redirect_url = requote_redirect_url
356 self._read_bufsize = read_bufsize
357 self._max_line_size = max_line_size
358 self._max_field_size = max_field_size
360 # Convert to list of tuples
361 if headers:
362 real_headers: CIMultiDict[str] = CIMultiDict(headers)
363 else:
364 real_headers = CIMultiDict()
365 self._default_headers: CIMultiDict[str] = real_headers
366 if skip_auto_headers is not None:
367 self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers)
368 else:
369 self._skip_auto_headers = frozenset()
371 self._request_class = request_class
372 self._response_class = response_class
373 self._ws_response_class = ws_response_class
375 self._trace_configs = trace_configs or []
376 for trace_config in self._trace_configs:
377 trace_config.freeze()
379 self._resolve_charset = fallback_charset_resolver
381 self._default_proxy = proxy
382 self._default_proxy_auth = proxy_auth
383 self._retry_connection: bool = True
384 self._middlewares = middlewares
386 def __init_subclass__(cls: Type["ClientSession"]) -> None:
387 raise TypeError(
388 "Inheritance class {} from ClientSession "
389 "is forbidden".format(cls.__name__)
390 )
392 def __del__(self, _warnings: Any = warnings) -> None:
393 if not self.closed:
394 _warnings.warn(
395 f"Unclosed client session {self!r}",
396 ResourceWarning,
397 source=self,
398 )
399 context = {"client_session": self, "message": "Unclosed client session"}
400 if self._source_traceback is not None:
401 context["source_traceback"] = self._source_traceback
402 self._loop.call_exception_handler(context)
404 if sys.version_info >= (3, 11) and TYPE_CHECKING:
406 def request(
407 self,
408 method: str,
409 url: StrOrURL,
410 **kwargs: Unpack[_RequestOptions],
411 ) -> "_RequestContextManager": ...
413 else:
415 def request(
416 self, method: str, url: StrOrURL, **kwargs: Any
417 ) -> "_RequestContextManager":
418 """Perform HTTP request."""
419 return _RequestContextManager(self._request(method, url, **kwargs))
421 def _build_url(self, str_or_url: StrOrURL) -> URL:
422 url = URL(str_or_url)
423 if self._base_url and not url.absolute:
424 return self._base_url.join(url)
425 return url
427 async def _request(
428 self,
429 method: str,
430 str_or_url: StrOrURL,
431 *,
432 params: Query = None,
433 data: Any = None,
434 json: Any = None,
435 cookies: Optional[LooseCookies] = None,
436 headers: Optional[LooseHeaders] = None,
437 skip_auto_headers: Optional[Iterable[str]] = None,
438 auth: Optional[BasicAuth] = None,
439 allow_redirects: bool = True,
440 max_redirects: int = 10,
441 compress: Union[str, bool] = False,
442 chunked: Optional[bool] = None,
443 expect100: bool = False,
444 raise_for_status: Union[
445 None, bool, Callable[[ClientResponse], Awaitable[None]]
446 ] = None,
447 read_until_eof: bool = True,
448 proxy: Optional[StrOrURL] = None,
449 proxy_auth: Optional[BasicAuth] = None,
450 timeout: Union[ClientTimeout, _SENTINEL, None] = sentinel,
451 ssl: Union[SSLContext, bool, Fingerprint] = True,
452 server_hostname: Optional[str] = None,
453 proxy_headers: Optional[LooseHeaders] = None,
454 trace_request_ctx: Optional[Mapping[str, Any]] = None,
455 read_bufsize: Optional[int] = None,
456 auto_decompress: Optional[bool] = None,
457 max_line_size: Optional[int] = None,
458 max_field_size: Optional[int] = None,
459 middlewares: Optional[Sequence[ClientMiddlewareType]] = None,
460 ) -> ClientResponse:
461 # NOTE: timeout clamps existing connect and read timeouts. We cannot
462 # set the default to None because we need to detect if the user wants
463 # to use the existing timeouts by setting timeout to None.
465 if self.closed:
466 raise RuntimeError("Session is closed")
468 if not isinstance(ssl, SSL_ALLOWED_TYPES):
469 raise TypeError(
470 "ssl should be SSLContext, Fingerprint, or bool, "
471 "got {!r} instead.".format(ssl)
472 )
474 if data is not None and json is not None:
475 raise ValueError(
476 "data and json parameters can not be used at the same time"
477 )
478 elif json is not None:
479 data = payload.JsonPayload(json, dumps=self._json_serialize)
481 redirects = 0
482 history: List[ClientResponse] = []
483 version = self._version
484 params = params or {}
486 # Merge with default headers and transform to CIMultiDict
487 headers = self._prepare_headers(headers)
489 try:
490 url = self._build_url(str_or_url)
491 except ValueError as e:
492 raise InvalidUrlClientError(str_or_url) from e
494 assert self._connector is not None
495 if url.scheme not in self._connector.allowed_protocol_schema_set:
496 raise NonHttpUrlClientError(url)
498 skip_headers: Optional[Iterable[istr]]
499 if skip_auto_headers is not None:
500 skip_headers = {
501 istr(i) for i in skip_auto_headers
502 } | self._skip_auto_headers
503 elif self._skip_auto_headers:
504 skip_headers = self._skip_auto_headers
505 else:
506 skip_headers = None
508 if proxy is None:
509 proxy = self._default_proxy
510 if proxy_auth is None:
511 proxy_auth = self._default_proxy_auth
513 if proxy is None:
514 proxy_headers = None
515 else:
516 proxy_headers = self._prepare_headers(proxy_headers)
517 try:
518 proxy = URL(proxy)
519 except ValueError as e:
520 raise InvalidURL(proxy) from e
522 if timeout is sentinel or timeout is None:
523 real_timeout: ClientTimeout = self._timeout
524 else:
525 real_timeout = timeout
526 # timeout is cumulative for all request operations
527 # (request, redirects, responses, data consuming)
528 tm = TimeoutHandle(
529 self._loop, real_timeout.total, ceil_threshold=real_timeout.ceil_threshold
530 )
531 handle = tm.start()
533 if read_bufsize is None:
534 read_bufsize = self._read_bufsize
536 if auto_decompress is None:
537 auto_decompress = self._auto_decompress
539 if max_line_size is None:
540 max_line_size = self._max_line_size
542 if max_field_size is None:
543 max_field_size = self._max_field_size
545 traces = [
546 Trace(
547 self,
548 trace_config,
549 trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx),
550 )
551 for trace_config in self._trace_configs
552 ]
554 for trace in traces:
555 await trace.send_request_start(method, url.update_query(params), headers)
557 timer = tm.timer()
558 try:
559 with timer:
560 # https://www.rfc-editor.org/rfc/rfc9112.html#name-retrying-requests
561 retry_persistent_connection = (
562 self._retry_connection and method in IDEMPOTENT_METHODS
563 )
564 while True:
565 url, auth_from_url = strip_auth_from_url(url)
566 if not url.raw_host:
567 # NOTE: Bail early, otherwise, causes `InvalidURL` through
568 # NOTE: `self._request_class()` below.
569 err_exc_cls = (
570 InvalidUrlRedirectClientError
571 if redirects
572 else InvalidUrlClientError
573 )
574 raise err_exc_cls(url)
575 # If `auth` was passed for an already authenticated URL,
576 # disallow only if this is the initial URL; this is to avoid issues
577 # with sketchy redirects that are not the caller's responsibility
578 if not history and (auth and auth_from_url):
579 raise ValueError(
580 "Cannot combine AUTH argument with "
581 "credentials encoded in URL"
582 )
584 # Override the auth with the one from the URL only if we
585 # have no auth, or if we got an auth from a redirect URL
586 if auth is None or (history and auth_from_url is not None):
587 auth = auth_from_url
589 if (
590 auth is None
591 and self._default_auth
592 and (
593 not self._base_url or self._base_url_origin == url.origin()
594 )
595 ):
596 auth = self._default_auth
597 # It would be confusing if we support explicit
598 # Authorization header with auth argument
599 if auth is not None and hdrs.AUTHORIZATION in headers:
600 raise ValueError(
601 "Cannot combine AUTHORIZATION header "
602 "with AUTH argument or credentials "
603 "encoded in URL"
604 )
606 all_cookies = self._cookie_jar.filter_cookies(url)
608 if cookies is not None:
609 tmp_cookie_jar = CookieJar(
610 quote_cookie=self._cookie_jar.quote_cookie
611 )
612 tmp_cookie_jar.update_cookies(cookies)
613 req_cookies = tmp_cookie_jar.filter_cookies(url)
614 if req_cookies:
615 all_cookies.load(req_cookies)
617 proxy_: Optional[URL] = None
618 if proxy is not None:
619 proxy_ = URL(proxy)
620 elif self._trust_env:
621 with suppress(LookupError):
622 proxy_, proxy_auth = await asyncio.to_thread(
623 get_env_proxy_for_url, url
624 )
626 req = self._request_class(
627 method,
628 url,
629 params=params,
630 headers=headers,
631 skip_auto_headers=skip_headers,
632 data=data,
633 cookies=all_cookies,
634 auth=auth,
635 version=version,
636 compress=compress,
637 chunked=chunked,
638 expect100=expect100,
639 loop=self._loop,
640 response_class=self._response_class,
641 proxy=proxy_,
642 proxy_auth=proxy_auth,
643 timer=timer,
644 session=self,
645 ssl=ssl,
646 server_hostname=server_hostname,
647 proxy_headers=proxy_headers,
648 traces=traces,
649 trust_env=self.trust_env,
650 )
652 async def _connect_and_send_request(
653 req: ClientRequest,
654 ) -> ClientResponse:
655 # connection timeout
656 assert self._connector is not None
657 try:
658 conn = await self._connector.connect(
659 req, traces=traces, timeout=real_timeout
660 )
661 except asyncio.TimeoutError as exc:
662 raise ConnectionTimeoutError(
663 f"Connection timeout to host {req.url}"
664 ) from exc
666 assert conn.protocol is not None
667 conn.protocol.set_response_params(
668 timer=timer,
669 skip_payload=req.method in EMPTY_BODY_METHODS,
670 read_until_eof=read_until_eof,
671 auto_decompress=auto_decompress,
672 read_timeout=real_timeout.sock_read,
673 read_bufsize=read_bufsize,
674 timeout_ceil_threshold=self._connector._timeout_ceil_threshold,
675 max_line_size=max_line_size,
676 max_field_size=max_field_size,
677 )
678 try:
679 resp = await req.send(conn)
680 try:
681 await resp.start(conn)
682 except BaseException:
683 resp.close()
684 raise
685 except BaseException:
686 conn.close()
687 raise
688 return resp
690 # Apply middleware (if any) - per-request middleware overrides session middleware
691 effective_middlewares = (
692 self._middlewares if middlewares is None else middlewares
693 )
695 if effective_middlewares:
696 handler = build_client_middlewares(
697 _connect_and_send_request, effective_middlewares
698 )
699 else:
700 handler = _connect_and_send_request
702 try:
703 resp = await handler(req)
704 # Client connector errors should not be retried
705 except (
706 ConnectionTimeoutError,
707 ClientConnectorError,
708 ClientConnectorCertificateError,
709 ClientConnectorSSLError,
710 ):
711 raise
712 except (ClientOSError, ServerDisconnectedError):
713 if retry_persistent_connection:
714 retry_persistent_connection = False
715 continue
716 raise
717 except ClientError:
718 raise
719 except OSError as exc:
720 if exc.errno is None and isinstance(exc, asyncio.TimeoutError):
721 raise
722 raise ClientOSError(*exc.args) from exc
724 if cookies := resp._cookies:
725 self._cookie_jar.update_cookies(cookies, resp.url)
727 # redirects
728 if resp.status in (301, 302, 303, 307, 308) and allow_redirects:
729 for trace in traces:
730 await trace.send_request_redirect(
731 method, url.update_query(params), headers, resp
732 )
734 redirects += 1
735 history.append(resp)
736 if max_redirects and redirects >= max_redirects:
737 resp.close()
738 raise TooManyRedirects(
739 history[0].request_info, tuple(history)
740 )
742 # For 301 and 302, mimic IE, now changed in RFC
743 # https://github.com/kennethreitz/requests/pull/269
744 if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or (
745 resp.status in (301, 302) and resp.method == hdrs.METH_POST
746 ):
747 method = hdrs.METH_GET
748 data = None
749 if headers.get(hdrs.CONTENT_LENGTH):
750 headers.pop(hdrs.CONTENT_LENGTH)
752 r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get(
753 hdrs.URI
754 )
755 if r_url is None:
756 # see github.com/aio-libs/aiohttp/issues/2022
757 break
758 else:
759 # reading from correct redirection
760 # response is forbidden
761 resp.release()
763 try:
764 parsed_redirect_url = URL(
765 r_url, encoded=not self._requote_redirect_url
766 )
767 except ValueError as e:
768 raise InvalidUrlRedirectClientError(
769 r_url,
770 "Server attempted redirecting to a location that does not look like a URL",
771 ) from e
773 scheme = parsed_redirect_url.scheme
774 if scheme not in HTTP_AND_EMPTY_SCHEMA_SET:
775 resp.close()
776 raise NonHttpUrlRedirectClientError(r_url)
777 elif not scheme:
778 parsed_redirect_url = url.join(parsed_redirect_url)
780 is_same_host_https_redirect = (
781 url.host == parsed_redirect_url.host
782 and parsed_redirect_url.scheme == "https"
783 and url.scheme == "http"
784 )
786 try:
787 redirect_origin = parsed_redirect_url.origin()
788 except ValueError as origin_val_err:
789 raise InvalidUrlRedirectClientError(
790 parsed_redirect_url,
791 "Invalid redirect URL origin",
792 ) from origin_val_err
794 if (
795 not is_same_host_https_redirect
796 and url.origin() != redirect_origin
797 ):
798 auth = None
799 headers.pop(hdrs.AUTHORIZATION, None)
801 url = parsed_redirect_url
802 params = {}
803 resp.release()
804 continue
806 break
808 # check response status
809 if raise_for_status is None:
810 raise_for_status = self._raise_for_status
812 if raise_for_status is None:
813 pass
814 elif callable(raise_for_status):
815 await raise_for_status(resp)
816 elif raise_for_status:
817 resp.raise_for_status()
819 # register connection
820 if handle is not None:
821 if resp.connection is not None:
822 resp.connection.add_callback(handle.cancel)
823 else:
824 handle.cancel()
826 resp._history = tuple(history)
828 for trace in traces:
829 await trace.send_request_end(
830 method, url.update_query(params), headers, resp
831 )
832 return resp
834 except BaseException as e:
835 # cleanup timer
836 tm.close()
837 if handle:
838 handle.cancel()
839 handle = None
841 for trace in traces:
842 await trace.send_request_exception(
843 method, url.update_query(params), headers, e
844 )
845 raise
847 def ws_connect(
848 self,
849 url: StrOrURL,
850 *,
851 method: str = hdrs.METH_GET,
852 protocols: Collection[str] = (),
853 timeout: Union[ClientWSTimeout, _SENTINEL] = sentinel,
854 receive_timeout: Optional[float] = None,
855 autoclose: bool = True,
856 autoping: bool = True,
857 heartbeat: Optional[float] = None,
858 auth: Optional[BasicAuth] = None,
859 origin: Optional[str] = None,
860 params: Query = None,
861 headers: Optional[LooseHeaders] = None,
862 proxy: Optional[StrOrURL] = None,
863 proxy_auth: Optional[BasicAuth] = None,
864 ssl: Union[SSLContext, bool, Fingerprint] = True,
865 server_hostname: Optional[str] = None,
866 proxy_headers: Optional[LooseHeaders] = None,
867 compress: int = 0,
868 max_msg_size: int = 4 * 1024 * 1024,
869 ) -> "_WSRequestContextManager":
870 """Initiate websocket connection."""
871 return _WSRequestContextManager(
872 self._ws_connect(
873 url,
874 method=method,
875 protocols=protocols,
876 timeout=timeout,
877 receive_timeout=receive_timeout,
878 autoclose=autoclose,
879 autoping=autoping,
880 heartbeat=heartbeat,
881 auth=auth,
882 origin=origin,
883 params=params,
884 headers=headers,
885 proxy=proxy,
886 proxy_auth=proxy_auth,
887 ssl=ssl,
888 server_hostname=server_hostname,
889 proxy_headers=proxy_headers,
890 compress=compress,
891 max_msg_size=max_msg_size,
892 )
893 )
895 async def _ws_connect(
896 self,
897 url: StrOrURL,
898 *,
899 method: str = hdrs.METH_GET,
900 protocols: Collection[str] = (),
901 timeout: Union[ClientWSTimeout, _SENTINEL] = sentinel,
902 receive_timeout: Optional[float] = None,
903 autoclose: bool = True,
904 autoping: bool = True,
905 heartbeat: Optional[float] = None,
906 auth: Optional[BasicAuth] = None,
907 origin: Optional[str] = None,
908 params: Query = None,
909 headers: Optional[LooseHeaders] = None,
910 proxy: Optional[StrOrURL] = None,
911 proxy_auth: Optional[BasicAuth] = None,
912 ssl: Union[SSLContext, bool, Fingerprint] = True,
913 server_hostname: Optional[str] = None,
914 proxy_headers: Optional[LooseHeaders] = None,
915 compress: int = 0,
916 max_msg_size: int = 4 * 1024 * 1024,
917 ) -> ClientWebSocketResponse:
918 if timeout is not sentinel:
919 if isinstance(timeout, ClientWSTimeout):
920 ws_timeout = timeout
921 else:
922 warnings.warn( # type: ignore[unreachable]
923 "parameter 'timeout' of type 'float' "
924 "is deprecated, please use "
925 "'timeout=ClientWSTimeout(ws_close=...)'",
926 DeprecationWarning,
927 stacklevel=2,
928 )
929 ws_timeout = ClientWSTimeout(ws_close=timeout)
930 else:
931 ws_timeout = DEFAULT_WS_CLIENT_TIMEOUT
932 if receive_timeout is not None:
933 warnings.warn(
934 "float parameter 'receive_timeout' "
935 "is deprecated, please use parameter "
936 "'timeout=ClientWSTimeout(ws_receive=...)'",
937 DeprecationWarning,
938 stacklevel=2,
939 )
940 ws_timeout = dataclasses.replace(ws_timeout, ws_receive=receive_timeout)
942 if headers is None:
943 real_headers: CIMultiDict[str] = CIMultiDict()
944 else:
945 real_headers = CIMultiDict(headers)
947 default_headers = {
948 hdrs.UPGRADE: "websocket",
949 hdrs.CONNECTION: "Upgrade",
950 hdrs.SEC_WEBSOCKET_VERSION: "13",
951 }
953 for key, value in default_headers.items():
954 real_headers.setdefault(key, value)
956 sec_key = base64.b64encode(os.urandom(16))
957 real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode()
959 if protocols:
960 real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ",".join(protocols)
961 if origin is not None:
962 real_headers[hdrs.ORIGIN] = origin
963 if compress:
964 extstr = ws_ext_gen(compress=compress)
965 real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr
967 if not isinstance(ssl, SSL_ALLOWED_TYPES):
968 raise TypeError(
969 "ssl should be SSLContext, Fingerprint, or bool, "
970 "got {!r} instead.".format(ssl)
971 )
973 # send request
974 resp = await self.request(
975 method,
976 url,
977 params=params,
978 headers=real_headers,
979 read_until_eof=False,
980 auth=auth,
981 proxy=proxy,
982 proxy_auth=proxy_auth,
983 ssl=ssl,
984 server_hostname=server_hostname,
985 proxy_headers=proxy_headers,
986 )
988 try:
989 # check handshake
990 if resp.status != 101:
991 raise WSServerHandshakeError(
992 resp.request_info,
993 resp.history,
994 message="Invalid response status",
995 status=resp.status,
996 headers=resp.headers,
997 )
999 if resp.headers.get(hdrs.UPGRADE, "").lower() != "websocket":
1000 raise WSServerHandshakeError(
1001 resp.request_info,
1002 resp.history,
1003 message="Invalid upgrade header",
1004 status=resp.status,
1005 headers=resp.headers,
1006 )
1008 if resp.headers.get(hdrs.CONNECTION, "").lower() != "upgrade":
1009 raise WSServerHandshakeError(
1010 resp.request_info,
1011 resp.history,
1012 message="Invalid connection header",
1013 status=resp.status,
1014 headers=resp.headers,
1015 )
1017 # key calculation
1018 r_key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, "")
1019 match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest()).decode()
1020 if r_key != match:
1021 raise WSServerHandshakeError(
1022 resp.request_info,
1023 resp.history,
1024 message="Invalid challenge response",
1025 status=resp.status,
1026 headers=resp.headers,
1027 )
1029 # websocket protocol
1030 protocol = None
1031 if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers:
1032 resp_protocols = [
1033 proto.strip()
1034 for proto in resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",")
1035 ]
1037 for proto in resp_protocols:
1038 if proto in protocols:
1039 protocol = proto
1040 break
1042 # websocket compress
1043 notakeover = False
1044 if compress:
1045 compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
1046 if compress_hdrs:
1047 try:
1048 compress, notakeover = ws_ext_parse(compress_hdrs)
1049 except WSHandshakeError as exc:
1050 raise WSServerHandshakeError(
1051 resp.request_info,
1052 resp.history,
1053 message=exc.args[0],
1054 status=resp.status,
1055 headers=resp.headers,
1056 ) from exc
1057 else:
1058 compress = 0
1059 notakeover = False
1061 conn = resp.connection
1062 assert conn is not None
1063 conn_proto = conn.protocol
1064 assert conn_proto is not None
1066 # For WS connection the read_timeout must be either ws_timeout.ws_receive or greater
1067 # None == no timeout, i.e. infinite timeout, so None is the max timeout possible
1068 if ws_timeout.ws_receive is None:
1069 # Reset regardless
1070 conn_proto.read_timeout = None
1071 elif conn_proto.read_timeout is not None:
1072 conn_proto.read_timeout = max(
1073 ws_timeout.ws_receive, conn_proto.read_timeout
1074 )
1076 transport = conn.transport
1077 assert transport is not None
1078 reader = WebSocketDataQueue(conn_proto, 2**16, loop=self._loop)
1079 conn_proto.set_parser(WebSocketReader(reader, max_msg_size), reader)
1080 writer = WebSocketWriter(
1081 conn_proto,
1082 transport,
1083 use_mask=True,
1084 compress=compress,
1085 notakeover=notakeover,
1086 )
1087 except BaseException:
1088 resp.close()
1089 raise
1090 else:
1091 return self._ws_response_class(
1092 reader,
1093 writer,
1094 protocol,
1095 resp,
1096 ws_timeout,
1097 autoclose,
1098 autoping,
1099 self._loop,
1100 heartbeat=heartbeat,
1101 compress=compress,
1102 client_notakeover=notakeover,
1103 )
1105 def _prepare_headers(self, headers: Optional[LooseHeaders]) -> "CIMultiDict[str]":
1106 """Add default headers and transform it to CIMultiDict"""
1107 # Convert headers to MultiDict
1108 result = CIMultiDict(self._default_headers)
1109 if headers:
1110 if not isinstance(headers, (MultiDictProxy, MultiDict)):
1111 headers = CIMultiDict(headers)
1112 added_names: Set[str] = set()
1113 for key, value in headers.items():
1114 if key in added_names:
1115 result.add(key, value)
1116 else:
1117 result[key] = value
1118 added_names.add(key)
1119 return result
1121 if sys.version_info >= (3, 11) and TYPE_CHECKING:
1123 def get(
1124 self,
1125 url: StrOrURL,
1126 **kwargs: Unpack[_RequestOptions],
1127 ) -> "_RequestContextManager": ...
1129 def options(
1130 self,
1131 url: StrOrURL,
1132 **kwargs: Unpack[_RequestOptions],
1133 ) -> "_RequestContextManager": ...
1135 def head(
1136 self,
1137 url: StrOrURL,
1138 **kwargs: Unpack[_RequestOptions],
1139 ) -> "_RequestContextManager": ...
1141 def post(
1142 self,
1143 url: StrOrURL,
1144 **kwargs: Unpack[_RequestOptions],
1145 ) -> "_RequestContextManager": ...
1147 def put(
1148 self,
1149 url: StrOrURL,
1150 **kwargs: Unpack[_RequestOptions],
1151 ) -> "_RequestContextManager": ...
1153 def patch(
1154 self,
1155 url: StrOrURL,
1156 **kwargs: Unpack[_RequestOptions],
1157 ) -> "_RequestContextManager": ...
1159 def delete(
1160 self,
1161 url: StrOrURL,
1162 **kwargs: Unpack[_RequestOptions],
1163 ) -> "_RequestContextManager": ...
1165 else:
1167 def get(
1168 self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any
1169 ) -> "_RequestContextManager":
1170 """Perform HTTP GET request."""
1171 return _RequestContextManager(
1172 self._request(
1173 hdrs.METH_GET, url, allow_redirects=allow_redirects, **kwargs
1174 )
1175 )
1177 def options(
1178 self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any
1179 ) -> "_RequestContextManager":
1180 """Perform HTTP OPTIONS request."""
1181 return _RequestContextManager(
1182 self._request(
1183 hdrs.METH_OPTIONS, url, allow_redirects=allow_redirects, **kwargs
1184 )
1185 )
1187 def head(
1188 self, url: StrOrURL, *, allow_redirects: bool = False, **kwargs: Any
1189 ) -> "_RequestContextManager":
1190 """Perform HTTP HEAD request."""
1191 return _RequestContextManager(
1192 self._request(
1193 hdrs.METH_HEAD, url, allow_redirects=allow_redirects, **kwargs
1194 )
1195 )
1197 def post(
1198 self, url: StrOrURL, *, data: Any = None, **kwargs: Any
1199 ) -> "_RequestContextManager":
1200 """Perform HTTP POST request."""
1201 return _RequestContextManager(
1202 self._request(hdrs.METH_POST, url, data=data, **kwargs)
1203 )
1205 def put(
1206 self, url: StrOrURL, *, data: Any = None, **kwargs: Any
1207 ) -> "_RequestContextManager":
1208 """Perform HTTP PUT request."""
1209 return _RequestContextManager(
1210 self._request(hdrs.METH_PUT, url, data=data, **kwargs)
1211 )
1213 def patch(
1214 self, url: StrOrURL, *, data: Any = None, **kwargs: Any
1215 ) -> "_RequestContextManager":
1216 """Perform HTTP PATCH request."""
1217 return _RequestContextManager(
1218 self._request(hdrs.METH_PATCH, url, data=data, **kwargs)
1219 )
1221 def delete(self, url: StrOrURL, **kwargs: Any) -> "_RequestContextManager":
1222 """Perform HTTP DELETE request."""
1223 return _RequestContextManager(
1224 self._request(hdrs.METH_DELETE, url, **kwargs)
1225 )
1227 async def close(self) -> None:
1228 """Close underlying connector.
1230 Release all acquired resources.
1231 """
1232 if not self.closed:
1233 if self._connector is not None and self._connector_owner:
1234 await self._connector.close()
1235 self._connector = None
1237 @property
1238 def closed(self) -> bool:
1239 """Is client session closed.
1241 A readonly property.
1242 """
1243 return self._connector is None or self._connector.closed
1245 @property
1246 def connector(self) -> Optional[BaseConnector]:
1247 """Connector instance used for the session."""
1248 return self._connector
1250 @property
1251 def cookie_jar(self) -> AbstractCookieJar:
1252 """The session cookies."""
1253 return self._cookie_jar
1255 @property
1256 def version(self) -> Tuple[int, int]:
1257 """The session HTTP protocol version."""
1258 return self._version
1260 @property
1261 def requote_redirect_url(self) -> bool:
1262 """Do URL requoting on redirection handling."""
1263 return self._requote_redirect_url
1265 @property
1266 def timeout(self) -> ClientTimeout:
1267 """Timeout for the session."""
1268 return self._timeout
1270 @property
1271 def headers(self) -> "CIMultiDict[str]":
1272 """The default headers of the client session."""
1273 return self._default_headers
1275 @property
1276 def skip_auto_headers(self) -> FrozenSet[istr]:
1277 """Headers for which autogeneration should be skipped"""
1278 return self._skip_auto_headers
1280 @property
1281 def auth(self) -> Optional[BasicAuth]: # type: ignore[misc]
1282 """An object that represents HTTP Basic Authorization"""
1283 return self._default_auth
1285 @property
1286 def json_serialize(self) -> JSONEncoder:
1287 """Json serializer callable"""
1288 return self._json_serialize
1290 @property
1291 def connector_owner(self) -> bool:
1292 """Should connector be closed on session closing"""
1293 return self._connector_owner
1295 @property
1296 def raise_for_status(
1297 self,
1298 ) -> Union[bool, Callable[[ClientResponse], Awaitable[None]]]:
1299 """Should `ClientResponse.raise_for_status()` be called for each response."""
1300 return self._raise_for_status
1302 @property
1303 def auto_decompress(self) -> bool:
1304 """Should the body response be automatically decompressed."""
1305 return self._auto_decompress
1307 @property
1308 def trust_env(self) -> bool:
1309 """
1310 Should proxies information from environment or netrc be trusted.
1312 Information is from HTTP_PROXY / HTTPS_PROXY environment variables
1313 or ~/.netrc file if present.
1314 """
1315 return self._trust_env
1317 @property
1318 def trace_configs(self) -> List[TraceConfig[Any]]: # type: ignore[misc]
1319 """A list of TraceConfig instances used for client tracing"""
1320 return self._trace_configs
1322 def detach(self) -> None:
1323 """Detach connector from session without closing the former.
1325 Session is switched to closed state anyway.
1326 """
1327 self._connector = None
1329 async def __aenter__(self) -> "ClientSession":
1330 return self
1332 async def __aexit__(
1333 self,
1334 exc_type: Optional[Type[BaseException]],
1335 exc_val: Optional[BaseException],
1336 exc_tb: Optional[TracebackType],
1337 ) -> None:
1338 await self.close()
1341class _BaseRequestContextManager(Coroutine[Any, Any, _RetType], Generic[_RetType]):
1342 __slots__ = ("_coro", "_resp")
1344 def __init__(self, coro: Coroutine["asyncio.Future[Any]", None, _RetType]) -> None:
1345 self._coro: Coroutine["asyncio.Future[Any]", None, _RetType] = coro
1347 def send(self, arg: None) -> "asyncio.Future[Any]":
1348 return self._coro.send(arg)
1350 def throw(self, *args: Any, **kwargs: Any) -> "asyncio.Future[Any]":
1351 return self._coro.throw(*args, **kwargs)
1353 def close(self) -> None:
1354 return self._coro.close()
1356 def __await__(self) -> Generator[Any, None, _RetType]:
1357 ret = self._coro.__await__()
1358 return ret
1360 def __iter__(self) -> Generator[Any, None, _RetType]:
1361 return self.__await__()
1363 async def __aenter__(self) -> _RetType:
1364 self._resp: _RetType = await self._coro
1365 return await self._resp.__aenter__()
1367 async def __aexit__(
1368 self,
1369 exc_type: Optional[Type[BaseException]],
1370 exc: Optional[BaseException],
1371 tb: Optional[TracebackType],
1372 ) -> None:
1373 await self._resp.__aexit__(exc_type, exc, tb)
1376_RequestContextManager = _BaseRequestContextManager[ClientResponse]
1377_WSRequestContextManager = _BaseRequestContextManager[ClientWebSocketResponse]
1380class _SessionRequestContextManager:
1381 __slots__ = ("_coro", "_resp", "_session")
1383 def __init__(
1384 self,
1385 coro: Coroutine["asyncio.Future[Any]", None, ClientResponse],
1386 session: ClientSession,
1387 ) -> None:
1388 self._coro = coro
1389 self._resp: Optional[ClientResponse] = None
1390 self._session = session
1392 async def __aenter__(self) -> ClientResponse:
1393 try:
1394 self._resp = await self._coro
1395 except BaseException:
1396 await self._session.close()
1397 raise
1398 else:
1399 return self._resp
1401 async def __aexit__(
1402 self,
1403 exc_type: Optional[Type[BaseException]],
1404 exc: Optional[BaseException],
1405 tb: Optional[TracebackType],
1406 ) -> None:
1407 assert self._resp is not None
1408 self._resp.close()
1409 await self._session.close()
1412if sys.version_info >= (3, 11) and TYPE_CHECKING:
1414 def request(
1415 method: str,
1416 url: StrOrURL,
1417 *,
1418 version: HttpVersion = http.HttpVersion11,
1419 connector: Optional[BaseConnector] = None,
1420 **kwargs: Unpack[_RequestOptions],
1421 ) -> _SessionRequestContextManager: ...
1423else:
1425 def request(
1426 method: str,
1427 url: StrOrURL,
1428 *,
1429 version: HttpVersion = http.HttpVersion11,
1430 connector: Optional[BaseConnector] = None,
1431 **kwargs: Any,
1432 ) -> _SessionRequestContextManager:
1433 """Constructs and sends a request.
1435 Returns response object.
1436 method - HTTP method
1437 url - request url
1438 params - (optional) Dictionary or bytes to be sent in the query
1439 string of the new request
1440 data - (optional) Dictionary, bytes, or file-like object to
1441 send in the body of the request
1442 json - (optional) Any json compatible python object
1443 headers - (optional) Dictionary of HTTP Headers to send with
1444 the request
1445 cookies - (optional) Dict object to send with the request
1446 auth - (optional) BasicAuth named tuple represent HTTP Basic Auth
1447 auth - aiohttp.helpers.BasicAuth
1448 allow_redirects - (optional) If set to False, do not follow
1449 redirects
1450 version - Request HTTP version.
1451 compress - Set to True if request has to be compressed
1452 with deflate encoding.
1453 chunked - Set to chunk size for chunked transfer encoding.
1454 expect100 - Expect 100-continue response from server.
1455 connector - BaseConnector sub-class instance to support
1456 connection pooling.
1457 read_until_eof - Read response until eof if response
1458 does not have Content-Length header.
1459 loop - Optional event loop.
1460 timeout - Optional ClientTimeout settings structure, 5min
1461 total timeout by default.
1462 Usage::
1463 >>> import aiohttp
1464 >>> async with aiohttp.request('GET', 'http://python.org/') as resp:
1465 ... print(resp)
1466 ... data = await resp.read()
1467 <ClientResponse(https://www.python.org/) [200 OK]>
1468 """
1469 connector_owner = False
1470 if connector is None:
1471 connector_owner = True
1472 connector = TCPConnector(force_close=True)
1474 session = ClientSession(
1475 cookies=kwargs.pop("cookies", None),
1476 version=version,
1477 timeout=kwargs.pop("timeout", sentinel),
1478 connector=connector,
1479 connector_owner=connector_owner,
1480 )
1482 return _SessionRequestContextManager(
1483 session._request(method, url, **kwargs),
1484 session,
1485 )