Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/aiohttp/http_parser.py: 18%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import abc
2import asyncio
3import re
4import string
5import sys
6from contextlib import suppress
7from enum import IntEnum
8from re import Pattern
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 ClassVar,
13 Final,
14 Generic,
15 Literal,
16 NamedTuple,
17 TypeVar,
18)
20from multidict import CIMultiDict, CIMultiDictProxy, istr
21from yarl import URL
23from . import hdrs
24from .base_protocol import BaseProtocol
25from .compression_utils import (
26 HAS_BROTLI,
27 HAS_ZSTD,
28 BrotliDecompressor,
29 ZLibDecompressor,
30 ZSTDDecompressor,
31)
32from .helpers import (
33 _EXC_SENTINEL,
34 DEBUG,
35 DEFAULT_CHUNK_SIZE,
36 EMPTY_BODY_METHODS,
37 EMPTY_BODY_STATUS_CODES,
38 NO_EXTENSIONS,
39 BaseTimerContext,
40 set_exception,
41)
42from .http_exceptions import (
43 BadHttpMessage,
44 BadHttpMethod,
45 BadStatusLine,
46 ContentEncodingError,
47 ContentLengthError,
48 InvalidHeader,
49 InvalidURLError,
50 LineTooLong,
51 TransferEncodingError,
52)
53from .http_writer import HttpVersion, HttpVersion10, HttpVersion11
54from .streams import EMPTY_PAYLOAD, StreamReader
55from .typedefs import RawHeaders
57if TYPE_CHECKING:
58 from .client_proto import ResponseHandler
60__all__ = (
61 "HeadersParser",
62 "HttpParser",
63 "HttpRequestParser",
64 "HttpResponseParser",
65 "RawRequestMessage",
66 "RawResponseMessage",
67)
69_SEP = Literal[b"\r\n", b"\n"]
71ASCIISET: Final[set[str]] = set(string.printable)
73# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview
74# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens
75#
76# method = token
77# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
78# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
79# token = 1*tchar
80_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~")
81TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+")
82VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII)
83DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII)
84HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+")
85# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5
86_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(
87 r"[\x00-\x08\x0a-\x1f\x7f]"
88)
89_TARGET_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(r"[\x00-\x1f\x7f]")
92def _has_authority(target: str) -> bool:
93 """Tell if an absolute-form request target has "//" and a non-empty authority.
95 RFC 9110 section 4.2 requires a host for http and https. yarl's default
96 WHATWG mode reads a host from "http:host/p" or "http:///host/p", so the
97 target is checked before it is parsed.
98 """
99 rest = target.partition(":")[2]
100 return rest[:2] == "//" and rest[2:3] not in ("", "/", "?", "#")
103# RFC 9110 singleton headers — duplicates are rejected in strict mode.
104# In lax mode (response parser default), the check is skipped entirely
105# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
106# duplicate headers like Content-Type or Server.
107# Lowercased for case-insensitive matching against wire names.
108SINGLETON_HEADERS: Final[frozenset[str]] = frozenset(
109 {
110 "content-length",
111 "content-location",
112 "content-range",
113 "content-type",
114 "etag",
115 "host",
116 "max-forwards",
117 "server",
118 "transfer-encoding",
119 "user-agent",
120 }
121)
124class RawRequestMessage(NamedTuple):
125 method: str
126 path: str
127 version: HttpVersion
128 headers: "CIMultiDictProxy[str]"
129 raw_headers: RawHeaders
130 should_close: bool
131 compression: str | None
132 upgrade: bool
133 chunked: bool
134 url: URL
137class RawResponseMessage(NamedTuple):
138 version: HttpVersion
139 code: int
140 reason: str
141 headers: CIMultiDictProxy[str]
142 raw_headers: RawHeaders
143 should_close: bool
144 compression: str | None
145 upgrade: bool
146 chunked: bool
149_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage)
152class PayloadState(IntEnum):
153 PAYLOAD_COMPLETE = 0
154 PAYLOAD_NEEDS_INPUT = 1
155 PAYLOAD_HAS_PENDING_INPUT = 2
158class ParseState(IntEnum):
160 PARSE_NONE = 0
161 PARSE_LENGTH = 1
162 PARSE_CHUNKED = 2
163 PARSE_UNTIL_EOF = 3
166class ChunkState(IntEnum):
167 PARSE_CHUNKED_SIZE = 0
168 PARSE_CHUNKED_CHUNK = 1
169 PARSE_CHUNKED_CHUNK_EOF = 2
170 PARSE_MAYBE_TRAILERS = 3
171 PARSE_TRAILERS = 4
174class HeadersParser:
175 def __init__(
176 self,
177 max_line_size: int = 8190,
178 max_headers: int = 32768,
179 max_field_size: int = 8190,
180 lax: bool = False,
181 ) -> None:
182 self.max_line_size = max_line_size
183 self.max_headers = max_headers
184 self.max_field_size = max_field_size
185 self._lax = lax
187 def parse_headers(
188 self, lines: list[bytes]
189 ) -> tuple["CIMultiDictProxy[str]", RawHeaders]:
190 headers: CIMultiDict[str] = CIMultiDict()
191 # note: "raw" does not mean inclusion of OWS before/after the field value
192 raw_headers = []
194 lines_idx = 0
195 line = lines[lines_idx]
196 line_count = len(lines)
198 while line:
199 # Parse initial header name : value pair.
200 try:
201 bname, bvalue = line.split(b":", 1)
202 except ValueError:
203 raise InvalidHeader(line) from None
205 if len(bname) == 0:
206 raise InvalidHeader(bname)
208 # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2
209 if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"}
210 raise InvalidHeader(line)
212 bvalue = bvalue.lstrip(b" \t")
213 name = bname.decode("utf-8", "surrogateescape")
214 if not TOKENRE.fullmatch(name):
215 raise InvalidHeader(bname)
217 # next line
218 lines_idx += 1
219 line = lines[lines_idx]
221 # consume continuation lines
222 continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t')
224 # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding
225 if continuation:
226 header_length = len(bvalue)
227 bvalue_lst = [bvalue]
228 while continuation:
229 header_length += len(line)
230 if header_length > self.max_field_size:
231 header_line = bname + b": " + b"".join(bvalue_lst)
232 raise LineTooLong(
233 header_line[:100] + b"...", self.max_field_size
234 )
235 bvalue_lst.append(line)
237 # next line
238 lines_idx += 1
239 if lines_idx < line_count:
240 line = lines[lines_idx]
241 if line:
242 continuation = line[0] in (32, 9) # (' ', '\t')
243 else:
244 line = b""
245 break
246 bvalue = b"".join(bvalue_lst)
248 bvalue = bvalue.strip(b" \t")
249 value = bvalue.decode("utf-8", "surrogateescape")
251 # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5
252 if self._lax:
253 if "\n" in value or "\r" in value or "\x00" in value:
254 raise InvalidHeader(bvalue)
255 elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value):
256 raise InvalidHeader(bvalue)
258 if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS:
259 raise BadHttpMessage(f"Duplicate '{name}' header found.")
260 headers.add(name, value)
261 raw_headers.append((bname, bvalue))
263 return (CIMultiDictProxy(headers), tuple(raw_headers))
266def _is_supported_upgrade(headers: CIMultiDictProxy[str]) -> bool:
267 """Check if the upgrade header is supported."""
268 u = headers.get(hdrs.UPGRADE, "")
269 # .lower() can transform non-ascii characters.
270 return u.isascii() and u.lower() in {"tcp", "websocket"}
273class HttpParser(abc.ABC, Generic[_MsgT]):
274 lax: ClassVar[bool] = False
276 def __init__(
277 self,
278 protocol: BaseProtocol | None = None,
279 loop: asyncio.AbstractEventLoop | None = None,
280 limit: int = 2**16,
281 max_line_size: int = 8190,
282 max_headers: int = 128,
283 max_field_size: int = 8190,
284 timer: BaseTimerContext | None = None,
285 code: int | None = None,
286 method: str | None = None,
287 payload_exception: type[BaseException] | None = None,
288 response_with_body: bool = True,
289 read_until_eof: bool = False,
290 auto_decompress: bool = True,
291 max_msg_queue_size: int = 0,
292 ) -> None:
293 self.protocol = protocol
294 self.loop = loop
295 self.max_line_size = max_line_size
296 self.max_headers = max_headers
297 self.max_field_size = max_field_size
298 self.max_headers = max_headers
299 self.timer = timer
300 self.code = code
301 self.method = method
302 self.payload_exception = payload_exception
303 self.response_with_body = response_with_body
304 self.read_until_eof = read_until_eof
306 self._lines: list[bytes] = []
307 self._tail = b""
308 self._upgraded = False
309 self._pending_upgrade = False
310 self._payload = None
311 self._payload_parser: HttpPayloadParser | None = None
312 self._payload_has_more_data = False
313 self._auto_decompress = auto_decompress
314 self._limit = limit
315 self._headers_parser = HeadersParser(
316 max_line_size, max_headers, max_field_size, self.lax
317 )
318 # Stop emitting messages once this many are queued unconsumed (0 = off).
319 self._max_msg_queue_size = max_msg_queue_size
320 self._msg_in_flight = 0
322 @abc.abstractmethod
323 def parse_message(self, lines: list[bytes]) -> _MsgT: ...
325 @abc.abstractmethod
326 def _is_chunked_te(self, te: str) -> bool: ...
328 def pause_reading(self) -> None:
329 assert self._payload_parser is not None
330 self._payload_parser.pause_reading()
332 def message_consumed(self) -> None:
333 """Protocol drained a queued message; free a slot for parsing."""
334 if self._msg_in_flight > 0:
335 self._msg_in_flight -= 1
337 def feed_eof(self) -> _MsgT | None:
338 if self._payload_parser is not None:
339 self._payload_parser.feed_eof()
340 if self._payload_parser.done:
341 self._payload_parser = None
342 else:
343 # try to extract partial message
344 if self._tail:
345 self._lines.append(self._tail)
347 if self._lines:
348 if self._lines[-1] != "\r\n":
349 self._lines.append(b"")
350 with suppress(Exception):
351 return self.parse_message(self._lines)
352 return None
354 def feed_data(
355 self,
356 data: bytes,
357 SEP: _SEP = b"\r\n",
358 EMPTY: bytes = b"",
359 CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH,
360 METH_CONNECT: str = hdrs.METH_CONNECT,
361 SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1,
362 ) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]:
364 messages = []
366 if self._tail:
367 data, self._tail = self._tail + data, b""
369 data_len = len(data)
370 start_pos = 0
371 loop = self.loop
372 max_line_length = self.max_line_size
374 should_close = False
375 while start_pos < data_len or self._payload_has_more_data:
376 # read HTTP message (request/response line + headers), \r\n\r\n
377 # and split by lines
378 if self._payload_parser is None and not self._upgraded:
379 if (
380 self._max_msg_queue_size
381 and self._msg_in_flight >= self._max_msg_queue_size
382 ):
383 # Queue full: buffer the rest and stop. Safe pause point;
384 # any preceding body is consumed before the next request
385 # line. Resumes via feed_data(b"") when the queue drains.
386 self._tail = data[start_pos:]
387 # The remainder now lives in self._tail only. Don't return it.
388 data = EMPTY
389 break
390 pos = data.find(SEP, start_pos)
391 # consume \r\n
392 if pos == start_pos and not self._lines:
393 start_pos = pos + len(SEP)
394 continue
396 if pos >= start_pos:
397 if should_close:
398 raise BadHttpMessage("Data after `Connection: close`")
400 # line found
401 line = data[start_pos:pos]
402 if SEP == b"\n": # For lax response parsing
403 line = line.rstrip(b"\r")
404 if len(line) > max_line_length:
405 raise LineTooLong(line[:100] + b"...", max_line_length)
407 self._lines.append(line)
408 # After processing the status/request line, everything is a header.
409 max_line_length = self.max_field_size
411 if len(self._lines) > self.max_headers:
412 raise BadHttpMessage("Too many headers received")
414 start_pos = pos + len(SEP)
416 # \r\n\r\n found
417 if self._lines[-1] == EMPTY:
418 max_trailers = self.max_headers - len(self._lines)
419 try:
420 msg: _MsgT = self.parse_message(self._lines)
421 finally:
422 self._lines.clear()
424 def get_content_length() -> int | None:
425 # payload length
426 length_hdr = msg.headers.get(CONTENT_LENGTH)
427 if length_hdr is None:
428 return None
430 # Shouldn't allow +/- or other number formats.
431 # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2
432 # msg.headers is already stripped of leading/trailing wsp
433 if not DIGITS.fullmatch(length_hdr):
434 raise InvalidHeader(CONTENT_LENGTH)
436 return int(length_hdr)
438 length = get_content_length()
439 # do not support old websocket spec
440 if SEC_WEBSOCKET_KEY1 in msg.headers:
441 raise InvalidHeader(SEC_WEBSOCKET_KEY1)
443 upgraded = msg.upgrade and _is_supported_upgrade(msg.headers)
445 method = getattr(msg, "method", self.method)
446 # code is only present on responses
447 code = getattr(msg, "code", 0)
449 assert self.protocol is not None
450 # calculate payload
451 # https://www.rfc-editor.org/info/rfc9112/#name-message-body-length
452 # https://www.rfc-editor.org/info/rfc9110/#section-9.3.1-6
453 # EMPTY_BODY_METHODS should only apply to responses.
454 # self.method is None on request parser.
455 empty_body = code in EMPTY_BODY_STATUS_CODES or bool(
456 self.method and self.method in EMPTY_BODY_METHODS
457 )
458 if not empty_body and (
459 (length is not None and length > 0) or msg.chunked
460 ):
461 payload = StreamReader(
462 self.protocol,
463 timer=self.timer,
464 loop=loop,
465 limit=self._limit,
466 )
467 payload_parser = HttpPayloadParser(
468 payload,
469 length=length,
470 chunked=msg.chunked,
471 method=method,
472 compression=msg.compression,
473 code=self.code,
474 response_with_body=self.response_with_body,
475 auto_decompress=self._auto_decompress,
476 lax=self.lax,
477 headers_parser=self._headers_parser,
478 max_line_size=self.max_line_size,
479 max_field_size=self.max_field_size,
480 max_trailers=max_trailers,
481 limit=self._limit,
482 )
483 if not payload_parser.done:
484 self._payload_parser = payload_parser
485 # https://www.rfc-editor.org/info/rfc9110/#section-7.8-15
486 # Defer any requested upgrade until the
487 # complete request has been read.
488 self._pending_upgrade = upgraded
489 elif method == METH_CONNECT:
490 assert isinstance(msg, RawRequestMessage)
491 payload = StreamReader(
492 self.protocol,
493 timer=self.timer,
494 loop=loop,
495 limit=self._limit,
496 )
497 self._upgraded = True
498 self._payload_parser = HttpPayloadParser(
499 payload,
500 method=msg.method,
501 compression=msg.compression,
502 auto_decompress=self._auto_decompress,
503 lax=self.lax,
504 headers_parser=self._headers_parser,
505 max_line_size=self.max_line_size,
506 max_field_size=self.max_field_size,
507 max_trailers=max_trailers,
508 limit=self._limit,
509 )
510 elif not empty_body and length is None and self.read_until_eof:
511 payload = StreamReader(
512 self.protocol,
513 timer=self.timer,
514 loop=loop,
515 limit=self._limit,
516 )
517 payload_parser = HttpPayloadParser(
518 payload,
519 length=length,
520 chunked=msg.chunked,
521 method=method,
522 compression=msg.compression,
523 code=self.code,
524 response_with_body=self.response_with_body,
525 auto_decompress=self._auto_decompress,
526 lax=self.lax,
527 headers_parser=self._headers_parser,
528 max_line_size=self.max_line_size,
529 max_field_size=self.max_field_size,
530 max_trailers=max_trailers,
531 limit=self._limit,
532 )
533 if not payload_parser.done:
534 self._payload_parser = payload_parser
535 elif upgraded:
536 # No body to read, so the connection switches to
537 # the upgraded protocol immediately.
538 self._upgraded = True
539 payload = EMPTY_PAYLOAD
540 else:
541 payload = EMPTY_PAYLOAD
543 messages.append((msg, payload))
544 if self._max_msg_queue_size:
545 self._msg_in_flight += 1
546 should_close = msg.should_close
547 else:
548 self._tail = data[start_pos:]
549 # A bare LF here means CRLF was required:
550 # reject instead of buffering, else a following request's
551 # bytes get appended to this line and leak in the error.
552 if b"\n" in self._tail:
553 raise BadHttpMessage("Bad line ending, expected CRLF")
554 if len(self._tail) > self.max_line_size:
555 raise LineTooLong(self._tail[:100] + b"...", self.max_line_size)
556 data = EMPTY
557 break
559 # no parser, just store
560 elif self._payload_parser is None and self._upgraded:
561 assert not self._lines
562 break
564 # feed payload
565 else:
566 assert not self._lines
567 assert self._payload_parser is not None
568 try:
569 payload_state, data = self._payload_parser.feed_data(
570 data[start_pos:], SEP
571 )
572 except Exception as underlying_exc:
573 reraised_exc: BaseException = underlying_exc
574 if self.payload_exception is not None:
575 reraised_exc = self.payload_exception(str(underlying_exc))
577 set_exception(
578 self._payload_parser.payload,
579 reraised_exc,
580 underlying_exc,
581 )
583 payload_state = PayloadState.PAYLOAD_COMPLETE
584 data = b""
585 if isinstance(
586 underlying_exc, (InvalidHeader, TransferEncodingError)
587 ):
588 raise
590 self._payload_has_more_data = (
591 payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT
592 )
594 if payload_state is not PayloadState.PAYLOAD_COMPLETE:
595 # We've either consumed all available data, or we're pausing
596 # until the reader buffer is freed up.
597 break
599 start_pos = 0
600 data_len = len(data)
601 self._payload_parser = None
602 if self._pending_upgrade:
603 # Body fully read: the deferred upgrade takes effect and
604 # the rest of the connection is the upgraded protocol.
605 self._upgraded = True
606 self._pending_upgrade = False
608 if data and start_pos < data_len:
609 data = data[start_pos:]
610 else:
611 data = EMPTY
613 return messages, self._upgraded, data
615 def parse_headers(
616 self, lines: list[bytes]
617 ) -> tuple[
618 "CIMultiDictProxy[str]", RawHeaders, bool | None, str | None, bool, bool
619 ]:
620 """Parses RFC 5322 headers from a stream.
622 Line continuations are supported. Returns list of header name
623 and value pairs. Header name is in upper case.
624 """
625 headers, raw_headers = self._headers_parser.parse_headers(lines)
626 close_conn = None
627 encoding = None
628 upgrade = False
629 chunked = False
631 # keep-alive and protocol switching
632 # RFC 9110 section 7.6.1 defines Connection as a comma-separated list.
633 conn_values = headers.getall(hdrs.CONNECTION, ())
634 if conn_values:
635 conn_tokens = {
636 token.lower()
637 for conn_value in conn_values
638 for token in (part.strip(" \t") for part in conn_value.split(","))
639 if token and token.isascii()
640 }
642 if "close" in conn_tokens:
643 close_conn = True
644 elif "keep-alive" in conn_tokens:
645 close_conn = False
647 # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols
648 if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE):
649 upgrade = True
651 # encoding
652 enc = headers.get(hdrs.CONTENT_ENCODING, "")
653 if enc.isascii(): # Must be checked before .lower()
654 enc = enc.lower()
655 if enc in {"gzip", "deflate", "br", "zstd"}:
656 encoding = enc
658 # chunking
659 te = headers.get(hdrs.TRANSFER_ENCODING)
660 if te is not None:
661 if self._is_chunked_te(te):
662 chunked = True
664 if hdrs.CONTENT_LENGTH in headers:
665 raise BadHttpMessage(
666 "Transfer-Encoding can't be present with Content-Length",
667 )
669 return (headers, raw_headers, close_conn, encoding, upgrade, chunked)
671 def set_upgraded(self, val: bool) -> None:
672 """Set connection upgraded (to websocket) mode.
674 :param bool val: new state.
675 """
676 self._upgraded = val
679class HttpRequestParser(HttpParser[RawRequestMessage]):
680 """Read request status line.
682 Exception .http_exceptions.BadStatusLine
683 could be raised in case of any errors in status line.
684 Returns RawRequestMessage.
685 """
687 def parse_message(self, lines: list[bytes]) -> RawRequestMessage:
688 # request line
689 line = lines[0].decode("utf-8", "surrogateescape")
690 try:
691 method, path, version = line.split(" ", maxsplit=2)
692 except ValueError:
693 raise BadHttpMethod(line) from None
695 # method
696 if not TOKENRE.fullmatch(method):
697 raise BadHttpMethod(method)
698 method = method.upper()
700 # https://www.rfc-editor.org/rfc/rfc9112#section-3.2-4
701 if _TARGET_FORBIDDEN_CTL_RE.search(path):
702 raise InvalidURLError(
703 path.encode(errors="surrogateescape").decode("latin1")
704 )
706 # version
707 match = VERSRE.fullmatch(version)
708 if match is None:
709 raise BadStatusLine(line)
710 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
712 if method == "CONNECT":
713 # authority-form,
714 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
715 try:
716 url = URL.build(authority=path, encoded=True)
717 host = url.raw_host
718 except ValueError:
719 # Duplicated so mypy understands that url must be defined below.
720 raise InvalidURLError(
721 path.encode(errors="surrogateescape").decode("latin1")
722 )
723 if host is None:
724 raise InvalidURLError(
725 path.encode(errors="surrogateescape").decode("latin1")
726 )
727 elif path.startswith("/"):
728 # origin-form,
729 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
730 path_part, _hash_separator, url_fragment = path.partition("#")
731 path_part, _question_mark_separator, qs_part = path_part.partition("?")
733 # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based
734 # NOTE: parser does, otherwise it results into the same
735 # NOTE: HTTP Request-Line input producing different
736 # NOTE: `yarl.URL()` objects
737 url = URL.build(
738 path=path_part,
739 query_string=qs_part,
740 fragment=url_fragment,
741 encoded=True,
742 )
743 elif path == "*" and method == "OPTIONS":
744 # asterisk-form,
745 url = URL(path, encoded=True)
746 else:
747 # absolute-form for proxy maybe,
748 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
749 if not _has_authority(path):
750 raise InvalidURLError(
751 path.encode(errors="surrogateescape").decode("latin1")
752 )
753 try:
754 url = URL(path, encoded=True)
755 host = url.raw_host
756 except ValueError:
757 # Duplicated so mypy understands that url must be defined below.
758 raise InvalidURLError(
759 path.encode(errors="surrogateescape").decode("latin1")
760 )
761 # https://www.rfc-editor.org/rfc/rfc9110#section-4.2.1-4
762 if host is None:
763 raise InvalidURLError(
764 path.encode(errors="surrogateescape").decode("latin1")
765 )
767 # read headers
768 (
769 headers,
770 raw_headers,
771 close,
772 compression,
773 upgrade,
774 chunked,
775 ) = self.parse_headers(lines[1:])
777 if version_o == HttpVersion11 and hdrs.HOST not in headers:
778 raise BadHttpMessage("Missing 'Host' header in request.")
780 if close is None: # then the headers weren't set in the request
781 if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close
782 close = True
783 else: # HTTP 1.1 must ask to close.
784 close = False
786 return RawRequestMessage(
787 method,
788 path,
789 version_o,
790 headers,
791 raw_headers,
792 close,
793 compression,
794 upgrade,
795 chunked,
796 url,
797 )
799 def _is_chunked_te(self, te: str) -> bool:
800 te = te.rsplit(",", maxsplit=1)[-1].strip(" \t")
801 # .lower() transforms some non-ascii chars, so must check first.
802 if te.isascii() and te.lower() == "chunked":
803 return True
804 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3
805 raise BadHttpMessage("Request has invalid `Transfer-Encoding`")
808class HttpResponseParser(HttpParser[RawResponseMessage]):
809 """Read response status line and headers.
811 BadStatusLine could be raised in case of any errors in status line.
812 Returns RawResponseMessage.
813 """
815 protocol: "ResponseHandler"
817 # Lax mode should only be enabled on response parser.
818 lax = not DEBUG
820 def feed_data(
821 self,
822 data: bytes,
823 SEP: _SEP | None = None,
824 *args: Any,
825 **kwargs: Any,
826 ) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]:
827 if SEP is None:
828 SEP = b"\r\n" if DEBUG else b"\n"
829 return super().feed_data(data, SEP, *args, **kwargs)
831 def parse_message(self, lines: list[bytes]) -> RawResponseMessage:
832 line = lines[0].decode("utf-8", "surrogateescape")
833 try:
834 version, status = line.split(maxsplit=1)
835 except ValueError:
836 raise BadStatusLine(line) from None
838 try:
839 status, reason = status.split(maxsplit=1)
840 except ValueError:
841 status = status.strip()
842 reason = ""
844 # version
845 match = VERSRE.fullmatch(version)
846 if match is None:
847 raise BadStatusLine(line)
848 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
850 # The status code is a three-digit ASCII number, no padding
851 if len(status) != 3 or not DIGITS.fullmatch(status):
852 raise BadStatusLine(line)
853 status_i = int(status)
855 # read headers
856 (
857 headers,
858 raw_headers,
859 close,
860 compression,
861 upgrade,
862 chunked,
863 ) = self.parse_headers(lines[1:])
865 if close is None:
866 if version_o <= HttpVersion10:
867 close = True
868 # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length
869 elif 100 <= status_i < 200 or status_i in {204, 304}:
870 close = False
871 elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers:
872 close = False
873 else:
874 # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8
875 close = True
877 return RawResponseMessage(
878 version_o,
879 status_i,
880 reason.strip(),
881 headers,
882 raw_headers,
883 close,
884 compression,
885 upgrade,
886 chunked,
887 )
889 def _is_chunked_te(self, te: str) -> bool:
890 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2
891 return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked"
894class HttpPayloadParser:
895 def __init__(
896 self,
897 payload: StreamReader,
898 length: int | None = None,
899 chunked: bool = False,
900 compression: str | None = None,
901 code: int | None = None,
902 method: str | None = None,
903 response_with_body: bool = True,
904 auto_decompress: bool = True,
905 lax: bool = False,
906 *,
907 headers_parser: HeadersParser,
908 max_line_size: int = 8190,
909 max_field_size: int = 8190,
910 max_trailers: int = 128,
911 limit: int = DEFAULT_CHUNK_SIZE,
912 ) -> None:
913 self._length = 0
914 self._paused = False
915 self._type = ParseState.PARSE_UNTIL_EOF
916 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
917 self._chunk_size = 0
918 self._chunk_tail = b""
919 self._auto_decompress = auto_decompress
920 self._lax = lax
921 self._headers_parser = headers_parser
922 self._max_line_size = max_line_size
923 self._max_field_size = max_field_size
924 self._max_trailers = max_trailers
925 self._more_data_available = False
926 self._trailer_lines: list[bytes] = []
927 self.done = False
928 self._eof_pending = False
930 # payload decompression wrapper
931 if response_with_body and compression and self._auto_decompress:
932 real_payload: StreamReader | DeflateBuffer = DeflateBuffer(
933 payload, compression, max_decompress_size=limit
934 )
935 else:
936 real_payload = payload
938 # payload parser
939 if not response_with_body:
940 # don't parse payload if it's not expected to be received
941 self._type = ParseState.PARSE_NONE
942 real_payload.feed_eof()
943 self.done = True
944 elif chunked:
945 self._type = ParseState.PARSE_CHUNKED
946 elif length is not None:
947 self._type = ParseState.PARSE_LENGTH
948 self._length = length
949 self._length_expected = length
950 if self._length == 0:
951 real_payload.feed_eof()
952 self.done = True
954 self.payload = real_payload
956 def pause_reading(self) -> None:
957 self._paused = True
959 def feed_eof(self) -> None:
960 if self._type == ParseState.PARSE_UNTIL_EOF:
961 self._eof_pending = True
962 while self._more_data_available:
963 if self._paused:
964 self._paused = False
965 return # Will resume via feed_data(b"") later
966 self._more_data_available = self.payload.feed_data(b"", 0)
967 self.payload.feed_eof()
968 self.done = True
969 self._eof_pending = False
970 elif self._type == ParseState.PARSE_LENGTH:
971 if self._length:
972 received = self._length_expected - self._length
973 raise ContentLengthError(
974 f"Not enough data to satisfy content length header "
975 f"(received {received} of {self._length_expected} bytes)."
976 )
977 # Body has already been received, but parser paused.
978 while self._more_data_available:
979 if self._paused:
980 self._paused = False
981 return # Will resume via feed_data(b"") later
982 self._more_data_available = self.payload.feed_data(b"", 0)
983 self.payload.feed_eof()
984 self.done = True
985 elif self._type == ParseState.PARSE_CHUNKED:
986 raise TransferEncodingError(
987 "Not enough data to satisfy transfer length header."
988 )
990 def feed_data(
991 self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";"
992 ) -> tuple[PayloadState, bytes]:
993 """Receive a chunk of data to process.
995 Return:
996 PayloadState - The current state of payload processing.
997 This function may be called with empty bytes after returning
998 PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause.
999 bytes - If payload is complete, this is the unconsumed bytes intended for the
1000 next message/payload, b"" otherwise.
1001 """
1002 # Read specified amount of bytes
1003 if self._type == ParseState.PARSE_LENGTH:
1004 if self._chunk_tail:
1005 chunk = self._chunk_tail + chunk
1006 self._chunk_tail = b""
1008 required = self._length
1009 self._length = max(required - len(chunk), 0)
1010 self._more_data_available = self.payload.feed_data(
1011 chunk[:required], required
1012 )
1013 while self._more_data_available:
1014 if self._paused:
1015 self._paused = False
1016 self._chunk_tail = chunk[required:]
1017 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1018 self._more_data_available = self.payload.feed_data(b"", 0)
1020 if self._length == 0:
1021 self.payload.feed_eof()
1022 return PayloadState.PAYLOAD_COMPLETE, chunk[required:]
1023 # Chunked transfer encoding parser
1024 elif self._type == ParseState.PARSE_CHUNKED:
1025 if self._chunk_tail:
1026 # We should check the length is sane when not processing payload body.
1027 if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK:
1028 max_line_length = self._max_line_size
1029 if self._chunk == ChunkState.PARSE_TRAILERS:
1030 max_line_length = self._max_field_size
1031 if len(self._chunk_tail) > max_line_length:
1032 raise LineTooLong(
1033 self._chunk_tail[:100] + b"...", max_line_length
1034 )
1036 chunk = self._chunk_tail + chunk
1037 self._chunk_tail = b""
1039 while chunk or self._more_data_available:
1040 # read next chunk size
1041 if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
1042 pos = chunk.find(SEP)
1043 if pos >= 0:
1044 # Only chunk-size lines reach here; trailers enforce
1045 # _max_field_size separately in PARSE_TRAILERS below.
1046 if pos > self._max_line_size:
1047 raise LineTooLong(chunk[:100] + b"...", self._max_line_size)
1048 i = chunk.find(CHUNK_EXT, 0, pos)
1049 if i >= 0:
1050 size_b = chunk[:i] # strip chunk-extensions
1051 # Verify no LF in the chunk-extension
1052 if b"\n" in (ext := chunk[i:pos]):
1053 exc = TransferEncodingError(
1054 f"Unexpected LF in chunk-extension: {ext!r}"
1055 )
1056 set_exception(self.payload, exc)
1057 raise exc
1058 else:
1059 size_b = chunk[:pos]
1061 if self._lax: # Allow whitespace in lax mode.
1062 size_b = size_b.strip()
1064 if not re.fullmatch(HEXDIGITS, size_b):
1065 exc = TransferEncodingError(
1066 chunk[:pos].decode("ascii", "surrogateescape")
1067 )
1068 set_exception(self.payload, exc)
1069 raise exc
1070 size = int(bytes(size_b), 16)
1072 chunk = chunk[pos + len(SEP) :]
1073 if size == 0: # eof marker
1074 self._chunk = ChunkState.PARSE_TRAILERS
1075 if self._lax and chunk.startswith(b"\r"):
1076 chunk = chunk[1:]
1077 else:
1078 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK
1079 self._chunk_size = size
1080 self.payload.begin_http_chunk_receiving()
1081 else:
1082 if b"\n" in chunk:
1083 exc = TransferEncodingError(
1084 "Bad chunk-size line ending, expected CRLF"
1085 )
1086 set_exception(self.payload, exc)
1087 raise exc
1088 self._chunk_tail = chunk
1089 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1091 # read chunk and feed buffer
1092 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK:
1093 if self._paused:
1094 self._paused = False
1095 self._chunk_tail = chunk
1096 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1098 required = self._chunk_size
1099 self._chunk_size = max(required - len(chunk), 0)
1100 self._more_data_available = self.payload.feed_data(
1101 chunk[:required], required
1102 )
1103 chunk = chunk[required:]
1105 if self._more_data_available:
1106 continue
1108 if self._chunk_size:
1109 self._paused = False
1110 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1111 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
1112 self.payload.end_http_chunk_receiving()
1114 # toss the CRLF at the end of the chunk
1115 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF:
1116 if self._lax and chunk.startswith(b"\r"):
1117 chunk = chunk[1:]
1118 if chunk[: len(SEP)] == SEP:
1119 chunk = chunk[len(SEP) :]
1120 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
1121 elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]:
1122 exc = TransferEncodingError(
1123 "Chunk size mismatch: expected CRLF after chunk data"
1124 )
1125 set_exception(self.payload, exc)
1126 raise exc
1127 else:
1128 self._chunk_tail = chunk
1129 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1131 if self._chunk == ChunkState.PARSE_TRAILERS:
1132 pos = chunk.find(SEP)
1133 if pos < 0: # No line found
1134 if b"\n" in chunk:
1135 exc = TransferEncodingError(
1136 "Bad trailer line ending, expected CRLF"
1137 )
1138 set_exception(self.payload, exc)
1139 raise exc
1140 self._chunk_tail = chunk
1141 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1143 line = chunk[:pos]
1144 chunk = chunk[pos + len(SEP) :]
1145 if SEP == b"\n": # For lax response parsing
1146 line = line.rstrip(b"\r")
1148 if len(line) > self._max_field_size:
1149 raise LineTooLong(line[:100] + b"...", self._max_field_size)
1151 self._trailer_lines.append(line)
1153 if len(self._trailer_lines) > self._max_trailers:
1154 raise BadHttpMessage("Too many trailers received")
1156 # \r\n\r\n found, end of stream
1157 if self._trailer_lines[-1] == b"":
1158 # Headers and trailers are defined the same way,
1159 # so we reuse the HeadersParser here.
1160 try:
1161 trailers, raw_trailers = self._headers_parser.parse_headers(
1162 self._trailer_lines
1163 )
1164 finally:
1165 self._trailer_lines.clear()
1166 self.payload.feed_eof()
1167 return PayloadState.PAYLOAD_COMPLETE, chunk
1169 # Read all bytes until eof
1170 elif self._type == ParseState.PARSE_UNTIL_EOF:
1171 self._more_data_available = self.payload.feed_data(chunk, len(chunk))
1172 while self._more_data_available:
1173 if self._paused:
1174 self._paused = False
1175 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1176 self._more_data_available = self.payload.feed_data(b"", 0)
1178 if self._eof_pending:
1179 self.payload.feed_eof()
1180 self.done = True
1181 self._eof_pending = False
1182 return PayloadState.PAYLOAD_COMPLETE, b""
1184 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1187class DeflateBuffer:
1188 """DeflateStream decompress stream and feed data into specified stream."""
1190 decompressor: Any
1192 def __init__(
1193 self,
1194 out: StreamReader,
1195 encoding: str | None,
1196 max_decompress_size: int = DEFAULT_CHUNK_SIZE,
1197 ) -> None:
1198 self.out = out
1199 self.size = 0
1200 out.total_compressed_bytes = self.size
1201 self.encoding = encoding
1202 self._started_decoding = False
1204 self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor
1205 if encoding == "br":
1206 if not HAS_BROTLI: # pragma: no cover
1207 raise ContentEncodingError(
1208 "Can not decode content-encoding: brotli (br). "
1209 "Please install `Brotli`"
1210 )
1211 self.decompressor = BrotliDecompressor()
1212 elif encoding == "zstd":
1213 if not HAS_ZSTD:
1214 raise ContentEncodingError(
1215 "Can not decode content-encoding: zstandard (zstd). "
1216 "Please install `backports.zstd`"
1217 )
1218 self.decompressor = ZSTDDecompressor()
1219 else:
1220 self.decompressor = ZLibDecompressor(encoding=encoding)
1222 self._max_decompress_size = max_decompress_size
1224 def set_exception(
1225 self,
1226 exc: BaseException,
1227 exc_cause: BaseException = _EXC_SENTINEL,
1228 ) -> None:
1229 set_exception(self.out, exc, exc_cause)
1231 def feed_data(self, chunk: bytes, size: int) -> bool:
1232 self.size += size
1233 self.out.total_compressed_bytes = self.size
1235 # Inspect the first real byte once to choose the decompressor. An empty
1236 # chunk (e.g. a chunk-size line arriving without body bytes) has no
1237 # header to sniff, so skip it and wait for the first data byte.
1238 if not self._started_decoding and chunk:
1239 # RFC1950
1240 # bits 0..3 = CM = 0b1000 = 8 = "deflate"
1241 # bits 4..7 = CINFO = 1..7 = windows size.
1242 if self.encoding == "deflate" and chunk[0] & 0xF != 8:
1243 # Change the decoder to decompress incorrectly compressed data
1244 # Actually we should issue a warning about non-RFC-compliant data.
1245 self.decompressor = ZLibDecompressor(
1246 encoding=self.encoding, suppress_deflate_header=True
1247 )
1248 self._started_decoding = True
1250 low_water = self.out._low_water
1251 max_length = (
1252 0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water)
1253 )
1254 try:
1255 chunk = self.decompressor.decompress_sync(chunk, max_length=max_length)
1256 except Exception:
1257 raise ContentEncodingError(
1258 "Can not decode content-encoding: %s" % self.encoding
1259 )
1261 if chunk:
1262 self.out.feed_data(chunk, len(chunk))
1263 return self.decompressor.data_available # type: ignore[no-any-return]
1265 def feed_eof(self) -> None:
1266 chunk = self.decompressor.flush()
1267 # This should never contain data as we defer the call until exhausting
1268 # the decompression. If .flush() is returning data, this may indicate a
1269 # zip bomb vulnerability as it will decompress all remaining data at once.
1270 assert not chunk
1272 if self.size > 0:
1273 if self.encoding == "deflate" and not self.decompressor.eof:
1274 raise ContentEncodingError("deflate")
1276 self.out.feed_eof()
1278 def begin_http_chunk_receiving(self) -> None:
1279 self.out.begin_http_chunk_receiving()
1281 def end_http_chunk_receiving(self) -> None:
1282 self.out.end_http_chunk_receiving()
1285HttpRequestParserPy = HttpRequestParser
1286HttpResponseParserPy = HttpResponseParser
1287RawRequestMessagePy = RawRequestMessage
1288RawResponseMessagePy = RawResponseMessage
1290try:
1291 if not NO_EXTENSIONS:
1292 from ._http_parser import ( # type: ignore[import-not-found,no-redef]
1293 HttpRequestParser,
1294 HttpResponseParser,
1295 RawRequestMessage,
1296 RawResponseMessage,
1297 )
1299 HttpRequestParserC = HttpRequestParser
1300 HttpResponseParserC = HttpResponseParser
1301 RawRequestMessageC = RawRequestMessage
1302 RawResponseMessageC = RawResponseMessage
1303except ImportError: # pragma: no cover
1304 pass