Coverage for /pythoncovmergedfiles/medio/medio/src/aiohttp/aiohttp/http_parser.py: 81%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import abc
2import asyncio
3import re
4import string
5import sys
6from contextlib import suppress
7from enum import IntEnum
8from re import Pattern
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 ClassVar,
13 Final,
14 Generic,
15 Literal,
16 NamedTuple,
17 TypeVar,
18)
20from multidict import CIMultiDict, istr
21from yarl import URL
23from . import hdrs
24from .base_protocol import BaseProtocol
25from .compression_utils import (
26 HAS_BROTLI,
27 HAS_ZSTD,
28 BrotliDecompressor,
29 ZLibDecompressor,
30 ZSTDDecompressor,
31)
32from .helpers import (
33 _EXC_SENTINEL,
34 DEBUG,
35 DEFAULT_CHUNK_SIZE,
36 EMPTY_BODY_METHODS,
37 EMPTY_BODY_STATUS_CODES,
38 NO_EXTENSIONS,
39 BaseTimerContext,
40 HeadersDictProxy,
41 set_exception,
42)
43from .http_exceptions import (
44 BadHttpMessage,
45 BadHttpMethod,
46 BadStatusLine,
47 ContentEncodingError,
48 ContentLengthError,
49 InvalidHeader,
50 InvalidURLError,
51 LineTooLong,
52 TransferEncodingError,
53)
54from .http_writer import HttpVersion, HttpVersion10, HttpVersion11
55from .streams import EMPTY_PAYLOAD, StreamReader
56from .typedefs import RawHeaders
58if TYPE_CHECKING:
59 from .client_proto import ResponseHandler
61__all__ = (
62 "HeadersParser",
63 "HttpParser",
64 "HttpRequestParser",
65 "HttpResponseParser",
66 "RawRequestMessage",
67 "RawResponseMessage",
68)
70_T = TypeVar("_T")
72_SEP = Literal[b"\r\n", b"\n"]
74ASCIISET: Final[set[str]] = set(string.printable)
76# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview
77# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens
78#
79# method = token
80# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
81# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
82# token = 1*tchar
83_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~")
84TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+")
85# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5
86_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(
87 r"[\x00-\x08\x0a-\x1f\x7f]"
88)
89VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII)
90DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII)
91HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+")
93# RFC 9110 singleton headers — duplicates are rejected in strict mode.
94# In lax mode (response parser default), the check is skipped entirely
95# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
96# duplicate headers like Content-Type or Server.
97# Lowercased for case-insensitive matching against wire names.
98SINGLETON_HEADERS: Final[frozenset[str]] = frozenset(
99 {
100 "content-length",
101 "content-location",
102 "content-range",
103 "content-type",
104 "etag",
105 "host",
106 "max-forwards",
107 "server",
108 "transfer-encoding",
109 "user-agent",
110 }
111)
114class RawRequestMessage(NamedTuple):
115 method: str
116 path: str
117 version: HttpVersion
118 headers: HeadersDictProxy
119 raw_headers: RawHeaders
120 should_close: bool
121 compression: str | None
122 upgrade: bool
123 chunked: bool
124 url: URL
127class RawResponseMessage(NamedTuple):
128 version: HttpVersion
129 code: int
130 reason: str
131 headers: HeadersDictProxy
132 raw_headers: RawHeaders
133 should_close: bool
134 compression: str | None
135 upgrade: bool
136 chunked: bool
139_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage)
142class PayloadState(IntEnum):
143 PAYLOAD_COMPLETE = 0
144 PAYLOAD_NEEDS_INPUT = 1
145 PAYLOAD_HAS_PENDING_INPUT = 2
148class ParseState(IntEnum):
149 PARSE_NONE = 0
150 PARSE_LENGTH = 1
151 PARSE_CHUNKED = 2
152 PARSE_UNTIL_EOF = 3
155class ChunkState(IntEnum):
156 PARSE_CHUNKED_SIZE = 0
157 PARSE_CHUNKED_CHUNK = 1
158 PARSE_CHUNKED_CHUNK_EOF = 2
159 PARSE_TRAILERS = 4
162class HeadersParser:
163 def __init__(self, max_field_size: int = 8190, lax: bool = False) -> None:
164 self.max_field_size = max_field_size
165 self._lax = lax
167 def parse_headers(self, lines: list[bytes]) -> tuple[HeadersDictProxy, RawHeaders]:
168 headers: CIMultiDict[str] = CIMultiDict()
169 # note: "raw" does not mean inclusion of OWS before/after the field value
170 raw_headers = []
172 lines_idx = 0
173 line = lines[lines_idx]
174 line_count = len(lines)
176 while line:
177 # Parse initial header name : value pair.
178 try:
179 bname, bvalue = line.split(b":", 1)
180 except ValueError:
181 raise InvalidHeader(line) from None
183 if len(bname) == 0:
184 raise InvalidHeader(bname)
186 # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2
187 if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"}
188 raise InvalidHeader(line)
190 bvalue = bvalue.lstrip(b" \t")
191 name = bname.decode("utf-8", "surrogateescape")
192 if not TOKENRE.fullmatch(name):
193 raise InvalidHeader(bname)
195 # next line
196 lines_idx += 1
197 line = lines[lines_idx]
199 # consume continuation lines
200 continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t')
202 # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding
203 if continuation:
204 header_length = len(bvalue)
205 bvalue_lst = [bvalue]
206 while continuation:
207 header_length += len(line)
208 if header_length > self.max_field_size:
209 header_line = bname + b": " + b"".join(bvalue_lst)
210 raise LineTooLong(
211 header_line[:100] + b"...", self.max_field_size
212 )
213 bvalue_lst.append(line)
215 # next line
216 lines_idx += 1
217 if lines_idx < line_count:
218 line = lines[lines_idx]
219 if line:
220 continuation = line[0] in (32, 9) # (' ', '\t')
221 else:
222 line = b""
223 break
224 bvalue = b"".join(bvalue_lst)
226 bvalue = bvalue.strip(b" \t")
227 value = bvalue.decode("utf-8", "surrogateescape")
229 # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5
230 if self._lax:
231 if "\n" in value or "\r" in value or "\x00" in value:
232 raise InvalidHeader(bvalue)
233 elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value):
234 raise InvalidHeader(bvalue)
236 if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS:
237 raise BadHttpMessage(f"Duplicate '{name}' header found.")
238 headers.add(name, value)
239 raw_headers.append((bname, bvalue))
241 return (HeadersDictProxy(headers), tuple(raw_headers))
244def _is_supported_upgrade(headers: HeadersDictProxy) -> bool:
245 """Check if the upgrade header is supported."""
246 u = headers.get(hdrs.UPGRADE, "")
247 # .lower() can transform non-ascii characters.
248 return u.isascii() and u.lower() in {"tcp", "websocket"}
251class HttpParser(abc.ABC, Generic[_MsgT]):
252 lax: ClassVar[bool] = False
254 def __init__(
255 self,
256 protocol: BaseProtocol,
257 loop: asyncio.AbstractEventLoop,
258 limit: int,
259 max_line_size: int = 8190,
260 max_headers: int = 128,
261 max_field_size: int = 8190,
262 timer: BaseTimerContext | None = None,
263 code: int | None = None,
264 method: str | None = None,
265 payload_exception: type[BaseException] | None = None,
266 response_with_body: bool = True,
267 read_until_eof: bool = False,
268 auto_decompress: bool = True,
269 max_msg_queue_size: int = 0,
270 ) -> None:
271 self.protocol = protocol
272 self.loop = loop
273 self.max_line_size = max_line_size
274 self.max_field_size = max_field_size
275 self.max_headers = max_headers
276 self.timer = timer
277 self.code = code
278 self.method = method
279 self.payload_exception = payload_exception
280 self.response_with_body = response_with_body
281 self.read_until_eof = read_until_eof
283 self._lines: list[bytes] = []
284 self._tail = b""
285 self._upgraded = False
286 self._pending_upgrade = False
287 self._payload = None
288 self._payload_parser: HttpPayloadParser | None = None
289 self._payload_has_more_data = False
290 self._auto_decompress = auto_decompress
291 self._limit = limit
292 self._headers_parser = HeadersParser(max_field_size, self.lax)
293 # Stop emitting messages once this many are queued unconsumed (0 = off).
294 self._max_msg_queue_size = max_msg_queue_size
295 self._msg_in_flight = 0
297 @abc.abstractmethod
298 def parse_message(self, lines: list[bytes]) -> _MsgT: ...
300 @abc.abstractmethod
301 def _is_chunked_te(self, te: str) -> bool: ...
303 def pause_reading(self) -> None:
304 assert self._payload_parser is not None
305 self._payload_parser.pause_reading()
307 def message_consumed(self) -> None:
308 """Protocol drained a queued message; free a slot for parsing."""
309 if self._msg_in_flight > 0:
310 self._msg_in_flight -= 1
312 def feed_eof(self) -> _MsgT | None:
313 if self._payload_parser is not None:
314 self._payload_parser.feed_eof()
315 if self._payload_parser.done:
316 self._payload_parser = None
317 else:
318 # try to extract partial message
319 if self._tail:
320 self._lines.append(self._tail)
322 if self._lines:
323 if self._lines[-1] != "\r\n":
324 self._lines.append(b"")
325 with suppress(Exception):
326 return self.parse_message(self._lines)
327 return None
329 def feed_data(
330 self,
331 data: bytes,
332 SEP: _SEP = b"\r\n",
333 EMPTY: bytes = b"",
334 CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH,
335 METH_CONNECT: str = hdrs.METH_CONNECT,
336 SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1,
337 ) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]:
338 messages = []
340 if self._tail:
341 data, self._tail = self._tail + data, b""
343 data_len = len(data)
344 start_pos = 0
345 loop = self.loop
346 max_line_length = self.max_line_size
348 should_close = False
349 while start_pos < data_len or self._payload_has_more_data:
350 # read HTTP message (request/response line + headers), \r\n\r\n
351 # and split by lines
352 if self._payload_parser is None and not self._upgraded:
353 if (
354 self._max_msg_queue_size
355 and self._msg_in_flight >= self._max_msg_queue_size
356 ):
357 # Queue full: buffer the rest and stop. Safe pause point;
358 # any preceding body is consumed before the next request
359 # line. Resumes via feed_data(b"") when the queue drains.
360 self._tail = data[start_pos:]
361 break
362 pos = data.find(SEP, start_pos)
363 # consume \r\n
364 if pos == start_pos and not self._lines:
365 start_pos = pos + len(SEP)
366 continue
368 if pos >= start_pos:
369 if should_close:
370 raise BadHttpMessage("Data after `Connection: close`")
372 # line found
373 line = data[start_pos:pos]
374 if SEP == b"\n": # For lax response parsing
375 line = line.rstrip(b"\r")
376 if len(line) > max_line_length:
377 raise LineTooLong(line[:100] + b"...", max_line_length)
379 self._lines.append(line)
380 # After processing the status/request line, everything is a header.
381 max_line_length = self.max_field_size
383 if len(self._lines) > self.max_headers:
384 raise BadHttpMessage("Too many headers received")
386 start_pos = pos + len(SEP)
388 # \r\n\r\n found
389 if self._lines[-1] == EMPTY:
390 max_trailers = self.max_headers - len(self._lines)
391 try:
392 msg: _MsgT = self.parse_message(self._lines)
393 finally:
394 self._lines.clear()
396 def get_content_length() -> int | None:
397 # payload length
398 length_hdr = msg.headers.get(CONTENT_LENGTH)
399 if length_hdr is None:
400 return None
402 # Shouldn't allow +/- or other number formats.
403 # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2
404 # msg.headers is already stripped of leading/trailing wsp
405 if not DIGITS.fullmatch(length_hdr):
406 raise InvalidHeader(CONTENT_LENGTH)
408 return int(length_hdr)
410 length = get_content_length()
411 # do not support old websocket spec
412 if SEC_WEBSOCKET_KEY1 in msg.headers:
413 raise InvalidHeader(SEC_WEBSOCKET_KEY1)
415 upgraded = msg.upgrade and _is_supported_upgrade(msg.headers)
417 method = getattr(msg, "method", self.method)
418 # code is only present on responses
419 code = getattr(msg, "code", 0)
421 assert self.protocol is not None
422 # calculate payload
423 empty_body = code in EMPTY_BODY_STATUS_CODES or bool(
424 method and method in EMPTY_BODY_METHODS
425 )
426 if not empty_body and (
427 (length is not None and length > 0) or msg.chunked
428 ):
429 payload = StreamReader(
430 self.protocol,
431 timer=self.timer,
432 loop=loop,
433 limit=self._limit,
434 )
435 payload_parser = HttpPayloadParser(
436 payload,
437 length=length,
438 chunked=msg.chunked,
439 method=method,
440 compression=msg.compression,
441 code=self.code,
442 response_with_body=self.response_with_body,
443 auto_decompress=self._auto_decompress,
444 lax=self.lax,
445 headers_parser=self._headers_parser,
446 max_line_size=self.max_line_size,
447 max_field_size=self.max_field_size,
448 max_trailers=max_trailers,
449 limit=self._limit,
450 )
451 if not payload_parser.done:
452 self._payload_parser = payload_parser
453 # https://www.rfc-editor.org/info/rfc9110/#section-7.8-15
454 # Defer any requested upgrade until the
455 # complete request has been read.
456 self._pending_upgrade = upgraded
457 elif method == METH_CONNECT:
458 assert isinstance(msg, RawRequestMessage)
459 payload = StreamReader(
460 self.protocol,
461 timer=self.timer,
462 loop=loop,
463 limit=self._limit,
464 )
465 self._upgraded = True
466 self._payload_parser = HttpPayloadParser(
467 payload,
468 method=msg.method,
469 compression=msg.compression,
470 auto_decompress=self._auto_decompress,
471 lax=self.lax,
472 headers_parser=self._headers_parser,
473 max_line_size=self.max_line_size,
474 max_field_size=self.max_field_size,
475 max_trailers=max_trailers,
476 limit=self._limit,
477 )
478 elif not empty_body and length is None and self.read_until_eof:
479 payload = StreamReader(
480 self.protocol,
481 timer=self.timer,
482 loop=loop,
483 limit=self._limit,
484 )
485 payload_parser = HttpPayloadParser(
486 payload,
487 length=length,
488 chunked=msg.chunked,
489 method=method,
490 compression=msg.compression,
491 code=self.code,
492 response_with_body=self.response_with_body,
493 auto_decompress=self._auto_decompress,
494 lax=self.lax,
495 headers_parser=self._headers_parser,
496 max_line_size=self.max_line_size,
497 max_field_size=self.max_field_size,
498 max_trailers=max_trailers,
499 limit=self._limit,
500 )
501 if not payload_parser.done:
502 self._payload_parser = payload_parser
503 elif upgraded:
504 # No body to read, so the connection switches to
505 # the upgraded protocol immediately.
506 self._upgraded = True
507 payload = EMPTY_PAYLOAD
508 else:
509 payload = EMPTY_PAYLOAD
511 messages.append((msg, payload))
512 if self._max_msg_queue_size:
513 self._msg_in_flight += 1
514 should_close = msg.should_close
515 else:
516 self._tail = data[start_pos:]
517 # A bare LF here means CRLF was required:
518 # reject instead of buffering, else a following request's
519 # bytes get appended to this line and leak in the error.
520 if b"\n" in self._tail:
521 raise BadHttpMessage("Bad line ending, expected CRLF")
522 if len(self._tail) > self.max_line_size:
523 raise LineTooLong(self._tail[:100] + b"...", self.max_line_size)
524 data = EMPTY
525 break
527 # no parser, just store
528 elif self._payload_parser is None and self._upgraded:
529 assert not self._lines
530 break
532 # feed payload
533 else:
534 assert not self._lines
535 assert self._payload_parser is not None
536 try:
537 payload_state, data = self._payload_parser.feed_data(
538 data[start_pos:], SEP
539 )
540 except Exception as underlying_exc:
541 reraised_exc: BaseException = underlying_exc
542 if self.payload_exception is not None:
543 reraised_exc = self.payload_exception(str(underlying_exc))
545 set_exception(
546 self._payload_parser.payload,
547 reraised_exc,
548 underlying_exc,
549 )
551 payload_state = PayloadState.PAYLOAD_COMPLETE
552 data = b""
553 if isinstance(
554 underlying_exc, (InvalidHeader, TransferEncodingError)
555 ):
556 raise
558 self._payload_has_more_data = (
559 payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT
560 )
562 if payload_state is not PayloadState.PAYLOAD_COMPLETE:
563 # We've either consumed all available data, or we're pausing
564 # until the reader buffer is freed up.
565 break
567 start_pos = 0
568 data_len = len(data)
569 self._payload_parser = None
570 if self._pending_upgrade:
571 # Body fully read: the deferred upgrade takes effect and
572 # the rest of the connection is the upgraded protocol.
573 self._upgraded = True
574 self._pending_upgrade = False
576 if data and start_pos < data_len:
577 data = data[start_pos:]
578 else:
579 data = EMPTY
581 return messages, self._upgraded, data
583 def parse_headers(
584 self, lines: list[bytes]
585 ) -> tuple[HeadersDictProxy, RawHeaders, bool | None, str | None, bool, bool]:
586 """Parses RFC 5322 headers from a stream.
588 Line continuations are supported. Returns list of header name
589 and value pairs. Header name is in upper case.
590 """
591 headers, raw_headers = self._headers_parser.parse_headers(lines)
592 close_conn = None
593 encoding = None
594 upgrade = False
595 chunked = False
597 # keep-alive and protocol switching
598 # RFC 9110 section 7.6.1 defines Connection as a comma-separated list.
599 # We use a simple comma split here rather than getall() for performance,
600 # as the target tokens (close, keep-alive, upgrade) are simple ASCII
601 # values that never contain commas.
602 conn_values = headers.get(hdrs.CONNECTION)
603 if conn_values:
604 conn_tokens = {
605 token.lower()
606 for token in (part.strip(" \t") for part in conn_values.split(","))
607 if token and token.isascii()
608 }
610 if "close" in conn_tokens:
611 close_conn = True
612 elif "keep-alive" in conn_tokens:
613 close_conn = False
615 # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols
616 if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE):
617 upgrade = True
619 # encoding
620 enc = headers.get(hdrs.CONTENT_ENCODING, "")
621 if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}:
622 encoding = enc
624 # chunking
625 te = headers.get(hdrs.TRANSFER_ENCODING)
626 if te is not None:
627 if self._is_chunked_te(te):
628 chunked = True
630 if hdrs.CONTENT_LENGTH in headers:
631 raise BadHttpMessage(
632 "Transfer-Encoding can't be present with Content-Length",
633 )
635 return (headers, raw_headers, close_conn, encoding, upgrade, chunked)
637 def set_upgraded(self, val: bool) -> None:
638 """Set connection upgraded (to websocket) mode.
640 :param bool val: new state.
641 """
642 self._upgraded = val
645class HttpRequestParser(HttpParser[RawRequestMessage]):
646 """Read request status line.
648 Exception .http_exceptions.BadStatusLine
649 could be raised in case of any errors in status line.
650 Returns RawRequestMessage.
651 """
653 def parse_message(self, lines: list[bytes]) -> RawRequestMessage:
654 # request line
655 line = lines[0].decode("utf-8", "surrogateescape")
656 try:
657 method, path, version = line.split(" ", maxsplit=2)
658 except ValueError:
659 raise BadHttpMethod(line) from None
661 # method
662 if not TOKENRE.fullmatch(method):
663 raise BadHttpMethod(method)
664 method = method.upper()
666 # version
667 match = VERSRE.fullmatch(version)
668 if match is None:
669 raise BadStatusLine(line)
670 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
672 if method == "CONNECT":
673 # authority-form,
674 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
675 url = URL.build(authority=path, encoded=True)
676 elif path.startswith("/"):
677 # origin-form,
678 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
679 path_part, _hash_separator, url_fragment = path.partition("#")
680 path_part, _question_mark_separator, qs_part = path_part.partition("?")
682 # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based
683 # NOTE: parser does, otherwise it results into the same
684 # NOTE: HTTP Request-Line input producing different
685 # NOTE: `yarl.URL()` objects
686 url = URL.build(
687 path=path_part,
688 query_string=qs_part,
689 fragment=url_fragment,
690 encoded=True,
691 )
692 elif path == "*" and method == "OPTIONS":
693 # asterisk-form,
694 url = URL(path, encoded=True)
695 else:
696 # absolute-form for proxy maybe,
697 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
698 url = URL(path, encoded=True)
699 if not url.absolute:
700 # authority-form is only allowed with CONNECT
701 # https://www.rfc-editor.org/info/rfc9112/#section-3.2.3-1
702 raise InvalidURLError(
703 path.encode(errors="surrogateescape").decode("latin1")
704 )
706 # read headers
707 (
708 headers,
709 raw_headers,
710 close,
711 compression,
712 upgrade,
713 chunked,
714 ) = self.parse_headers(lines[1:])
716 if version_o == HttpVersion11 and hdrs.HOST not in headers:
717 raise BadHttpMessage("Missing 'Host' header in request.")
719 if close is None: # then the headers weren't set in the request
720 if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close
721 close = True
722 else: # HTTP 1.1 must ask to close.
723 close = False
725 return RawRequestMessage(
726 method,
727 path,
728 version_o,
729 headers,
730 raw_headers,
731 close,
732 compression,
733 upgrade,
734 chunked,
735 url,
736 )
738 def _is_chunked_te(self, te: str) -> bool:
739 # https://www.rfc-editor.org/rfc/rfc9112#section-7.1-3
740 # "A sender MUST NOT apply the chunked transfer coding more
741 # than once to a message body"
742 parts = [p.strip(" \t") for p in te.split(",")]
743 chunked_count = sum(1 for p in parts if p.isascii() and p.lower() == "chunked")
744 if chunked_count > 1:
745 raise BadHttpMessage("Request has duplicate `chunked` Transfer-Encoding")
746 last = parts[-1]
747 # .lower() transforms some non-ascii chars, so must check first.
748 if last.isascii() and last.lower() == "chunked":
749 return True
750 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3
751 raise BadHttpMessage("Request has invalid `Transfer-Encoding`")
754class HttpResponseParser(HttpParser[RawResponseMessage]):
755 """Read response status line and headers.
757 BadStatusLine could be raised in case of any errors in status line.
758 Returns RawResponseMessage.
759 """
761 protocol: "ResponseHandler"
763 # Lax mode should only be enabled on response parser.
764 lax = not DEBUG
766 def feed_data(
767 self,
768 data: bytes,
769 SEP: _SEP | None = None,
770 *args: Any,
771 **kwargs: Any,
772 ) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]:
773 if SEP is None:
774 SEP = b"\r\n" if DEBUG else b"\n"
775 return super().feed_data(data, SEP, *args, **kwargs)
777 def parse_message(self, lines: list[bytes]) -> RawResponseMessage:
778 line = lines[0].decode("utf-8", "surrogateescape")
779 try:
780 version, status = line.split(maxsplit=1)
781 except ValueError:
782 raise BadStatusLine(line) from None
784 try:
785 status, reason = status.split(maxsplit=1)
786 except ValueError:
787 status = status.strip()
788 reason = ""
790 # version
791 match = VERSRE.fullmatch(version)
792 if match is None:
793 raise BadStatusLine(line)
794 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
796 # The status code is a three-digit ASCII number, no padding
797 if len(status) != 3 or not DIGITS.fullmatch(status):
798 raise BadStatusLine(line)
799 status_i = int(status)
801 # read headers
802 (
803 headers,
804 raw_headers,
805 close,
806 compression,
807 upgrade,
808 chunked,
809 ) = self.parse_headers(lines[1:])
811 if close is None:
812 if version_o <= HttpVersion10:
813 close = True
814 # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length
815 elif 100 <= status_i < 200 or status_i in {204, 304}:
816 close = False
817 elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers:
818 close = False
819 else:
820 # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8
821 close = True
823 return RawResponseMessage(
824 version_o,
825 status_i,
826 reason.strip(),
827 headers,
828 raw_headers,
829 close,
830 compression,
831 upgrade,
832 chunked,
833 )
835 def _is_chunked_te(self, te: str) -> bool:
836 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2
837 return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked"
840class HttpPayloadParser:
841 def __init__(
842 self,
843 payload: StreamReader,
844 length: int | None = None,
845 chunked: bool = False,
846 compression: str | None = None,
847 code: int | None = None,
848 method: str | None = None,
849 response_with_body: bool = True,
850 auto_decompress: bool = True,
851 lax: bool = False,
852 *,
853 headers_parser: HeadersParser,
854 max_line_size: int = 8190,
855 max_field_size: int = 8190,
856 max_trailers: int = 128,
857 limit: int = DEFAULT_CHUNK_SIZE,
858 ) -> None:
859 self._length = 0
860 self._paused = False
861 self._type = ParseState.PARSE_UNTIL_EOF
862 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
863 self._chunk_size = 0
864 self._chunk_tail = b""
865 self._auto_decompress = auto_decompress
866 self._lax = lax
867 self._headers_parser = headers_parser
868 self._max_line_size = max_line_size
869 self._max_field_size = max_field_size
870 self._max_trailers = max_trailers
871 self._more_data_available = False
872 self._trailer_lines: list[bytes] = []
873 self.done = False
874 self._eof_pending = False
876 # payload decompression wrapper
877 if response_with_body and compression and self._auto_decompress:
878 real_payload: StreamReader | DeflateBuffer = DeflateBuffer(
879 payload, compression, max_decompress_size=limit
880 )
881 else:
882 real_payload = payload
884 # payload parser
885 if not response_with_body:
886 # don't parse payload if it's not expected to be received
887 self._type = ParseState.PARSE_NONE
888 real_payload.feed_eof()
889 self.done = True
890 elif chunked:
891 self._type = ParseState.PARSE_CHUNKED
892 elif length is not None:
893 self._type = ParseState.PARSE_LENGTH
894 self._length = length
895 self._length_expected = length
896 if self._length == 0:
897 real_payload.feed_eof()
898 self.done = True
900 self.payload = real_payload
902 def pause_reading(self) -> None:
903 self._paused = True
905 def feed_eof(self) -> None:
906 if self._type == ParseState.PARSE_UNTIL_EOF:
907 self._eof_pending = True
908 while self._more_data_available:
909 if self._paused:
910 self._paused = False
911 return # Will resume via feed_data(b"") later
912 self._more_data_available = self.payload.feed_data(b"")
913 self.payload.feed_eof()
914 self.done = True
915 self._eof_pending = False
916 elif self._type == ParseState.PARSE_LENGTH:
917 received = self._length_expected - self._length
918 raise ContentLengthError(
919 f"Not enough data to satisfy content length header "
920 f"(received {received} of {self._length_expected} bytes)."
921 )
922 elif self._type == ParseState.PARSE_CHUNKED:
923 raise TransferEncodingError(
924 "Not enough data to satisfy transfer length header."
925 )
927 def feed_data(
928 self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";"
929 ) -> tuple[PayloadState, bytes]:
930 """Receive a chunk of data to process.
932 Return:
933 PayloadState - The current state of payload processing.
934 This function may be called with empty bytes after returning
935 PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause.
936 bytes - If payload is complete, this is the unconsumed bytes intended for the
937 next message/payload, b"" otherwise.
938 """
939 # Read specified amount of bytes
940 if self._type == ParseState.PARSE_LENGTH:
941 if self._chunk_tail:
942 chunk = self._chunk_tail + chunk
943 self._chunk_tail = b""
945 required = self._length
946 self._length = max(required - len(chunk), 0)
947 self._more_data_available = self.payload.feed_data(chunk[:required])
948 while self._more_data_available:
949 if self._paused:
950 self._paused = False
951 self._chunk_tail = chunk[required:]
952 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
953 self._more_data_available = self.payload.feed_data(b"")
955 if self._length == 0:
956 self.payload.feed_eof()
957 return PayloadState.PAYLOAD_COMPLETE, chunk[required:]
958 # Chunked transfer encoding parser
959 elif self._type == ParseState.PARSE_CHUNKED:
960 if self._chunk_tail:
961 # We should check the length is sane when not processing payload body.
962 if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK:
963 max_line_length = self._max_line_size
964 if self._chunk == ChunkState.PARSE_TRAILERS:
965 max_line_length = self._max_field_size
966 if len(self._chunk_tail) > max_line_length:
967 raise LineTooLong(
968 self._chunk_tail[:100] + b"...", max_line_length
969 )
971 chunk = self._chunk_tail + chunk
972 self._chunk_tail = b""
974 while chunk or self._more_data_available:
975 # read next chunk size
976 if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
977 pos = chunk.find(SEP)
978 if pos >= 0:
979 # Only chunk-size lines reach here; trailers enforce
980 # _max_field_size separately in PARSE_TRAILERS below.
981 if pos > self._max_line_size:
982 raise LineTooLong(chunk[:100] + b"...", self._max_line_size)
983 i = chunk.find(CHUNK_EXT, 0, pos)
984 if i >= 0:
985 size_b = chunk[:i] # strip chunk-extensions
986 # Verify no LF in the chunk-extension
987 if b"\n" in (ext := chunk[i:pos]):
988 exc = TransferEncodingError(
989 f"Unexpected LF in chunk-extension: {ext!r}"
990 )
991 set_exception(self.payload, exc)
992 raise exc
993 else:
994 size_b = chunk[:pos]
996 if self._lax: # Allow whitespace in lax mode.
997 size_b = size_b.strip()
999 if not re.fullmatch(HEXDIGITS, size_b):
1000 exc = TransferEncodingError(
1001 chunk[:pos].decode("ascii", "surrogateescape")
1002 )
1003 set_exception(self.payload, exc)
1004 raise exc
1005 size = int(bytes(size_b), 16)
1007 chunk = chunk[pos + len(SEP) :]
1008 if size == 0: # eof marker
1009 self._chunk = ChunkState.PARSE_TRAILERS
1010 if self._lax and chunk.startswith(b"\r"):
1011 chunk = chunk[1:]
1012 else:
1013 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK
1014 self._chunk_size = size
1015 self.payload.begin_http_chunk_receiving()
1016 else:
1017 if b"\n" in chunk:
1018 exc = TransferEncodingError(
1019 "Bad chunk-size line ending, expected CRLF"
1020 )
1021 set_exception(self.payload, exc)
1022 raise exc
1023 self._chunk_tail = chunk
1024 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1026 # read chunk and feed buffer
1027 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK:
1028 if self._paused:
1029 self._paused = False
1030 self._chunk_tail = chunk
1031 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1033 required = self._chunk_size
1034 self._chunk_size = max(required - len(chunk), 0)
1035 self._more_data_available = self.payload.feed_data(chunk[:required])
1036 chunk = chunk[required:]
1038 if self._more_data_available:
1039 continue
1041 if self._chunk_size:
1042 self._paused = False
1043 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1044 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
1045 self.payload.end_http_chunk_receiving()
1047 # toss the CRLF at the end of the chunk
1048 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF:
1049 if self._lax and chunk.startswith(b"\r"):
1050 chunk = chunk[1:]
1051 if chunk[: len(SEP)] == SEP:
1052 chunk = chunk[len(SEP) :]
1053 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
1054 elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]:
1055 exc = TransferEncodingError(
1056 "Chunk size mismatch: expected CRLF after chunk data"
1057 )
1058 set_exception(self.payload, exc)
1059 raise exc
1060 else:
1061 self._chunk_tail = chunk
1062 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1064 if self._chunk == ChunkState.PARSE_TRAILERS:
1065 pos = chunk.find(SEP)
1066 if pos < 0: # No line found
1067 if b"\n" in chunk:
1068 exc = TransferEncodingError(
1069 "Bad trailer line ending, expected CRLF"
1070 )
1071 set_exception(self.payload, exc)
1072 raise exc
1073 self._chunk_tail = chunk
1074 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1076 line = chunk[:pos]
1077 chunk = chunk[pos + len(SEP) :]
1078 if SEP == b"\n": # For lax response parsing
1079 line = line.rstrip(b"\r")
1081 if len(line) > self._max_field_size:
1082 raise LineTooLong(line[:100] + b"...", self._max_field_size)
1084 self._trailer_lines.append(line)
1086 if len(self._trailer_lines) > self._max_trailers:
1087 raise BadHttpMessage("Too many trailers received")
1089 # \r\n\r\n found, end of stream
1090 if self._trailer_lines[-1] == b"":
1091 # Headers and trailers are defined the same way,
1092 # so we reuse the HeadersParser here.
1093 try:
1094 trailers, raw_trailers = self._headers_parser.parse_headers(
1095 self._trailer_lines
1096 )
1097 finally:
1098 self._trailer_lines.clear()
1099 self.payload.feed_eof()
1100 return PayloadState.PAYLOAD_COMPLETE, chunk
1102 # Read all bytes until eof
1103 elif self._type == ParseState.PARSE_UNTIL_EOF:
1104 self._more_data_available = self.payload.feed_data(chunk)
1105 while self._more_data_available:
1106 if self._paused:
1107 self._paused = False
1108 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1109 self._more_data_available = self.payload.feed_data(b"")
1111 if self._eof_pending:
1112 self.payload.feed_eof()
1113 self.done = True
1114 self._eof_pending = False
1115 return PayloadState.PAYLOAD_COMPLETE, b""
1117 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1120class DeflateBuffer:
1121 """DeflateStream decompress stream and feed data into specified stream."""
1123 def __init__(
1124 self,
1125 out: StreamReader,
1126 encoding: str | None,
1127 max_decompress_size: int = DEFAULT_CHUNK_SIZE,
1128 ) -> None:
1129 self.out = out
1130 self.size = 0
1131 out.total_compressed_bytes = self.size
1132 self.encoding = encoding
1133 self._started_decoding = False
1135 self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor
1136 if encoding == "br":
1137 if not HAS_BROTLI:
1138 raise ContentEncodingError(
1139 "Can not decode content-encoding: brotli (br). "
1140 "Please install `Brotli`"
1141 )
1142 self.decompressor = BrotliDecompressor()
1143 elif encoding == "zstd":
1144 if not HAS_ZSTD:
1145 raise ContentEncodingError(
1146 "Can not decode content-encoding: zstandard (zstd). "
1147 "Please install `backports.zstd`"
1148 )
1149 self.decompressor = ZSTDDecompressor()
1150 else:
1151 self.decompressor = ZLibDecompressor(encoding=encoding)
1153 self._max_decompress_size = max_decompress_size
1155 def set_exception(
1156 self,
1157 exc: type[BaseException] | BaseException,
1158 exc_cause: BaseException = _EXC_SENTINEL,
1159 ) -> None:
1160 set_exception(self.out, exc, exc_cause)
1162 def feed_data(self, chunk: bytes) -> bool:
1163 """Return True if more data is available and this method should be called again with b""."""
1164 self.size += len(chunk)
1165 self.out.total_compressed_bytes = self.size
1167 # Inspect the first real byte once to choose the decompressor. An empty
1168 # chunk (e.g. a chunk-size line arriving without body bytes) has no
1169 # header to sniff, so skip it and wait for the first data byte.
1170 if not self._started_decoding and chunk:
1171 # RFC1950
1172 # bits 0..3 = CM = 0b1000 = 8 = "deflate"
1173 # bits 4..7 = CINFO = 1..7 = windows size.
1174 if self.encoding == "deflate" and chunk[0] & 0xF != 8:
1175 # Change the decoder to decompress incorrectly compressed data
1176 # Actually we should issue a warning about non-RFC-compliant data.
1177 self.decompressor = ZLibDecompressor(
1178 encoding=self.encoding, suppress_deflate_header=True
1179 )
1180 self._started_decoding = True
1182 low_water = self.out._low_water
1183 max_length = (
1184 0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water)
1185 )
1186 try:
1187 chunk = self.decompressor.decompress_sync(chunk, max_length=max_length)
1188 except Exception:
1189 raise ContentEncodingError(
1190 "Can not decode content-encoding: %s" % self.encoding
1191 )
1193 if chunk:
1194 self.out.feed_data(chunk)
1195 return self.decompressor.data_available
1197 def feed_eof(self) -> None:
1198 chunk = self.decompressor.flush()
1199 # This should never contain data as we defer the call until exhausting
1200 # the decompression. If .flush() is returning data, this may indicate a
1201 # zip bomb vulnerability as it will decompress all remaining data at once.
1202 assert not chunk
1204 if self.size > 0:
1205 # decompressor is not brotli unless encoding is "br"
1206 if self.encoding == "deflate" and not self.decompressor.eof: # type: ignore[union-attr]
1207 raise ContentEncodingError("deflate")
1209 self.out.feed_eof()
1211 def begin_http_chunk_receiving(self) -> None:
1212 self.out.begin_http_chunk_receiving()
1214 def end_http_chunk_receiving(self) -> None:
1215 self.out.end_http_chunk_receiving()
1218HttpRequestParserPy = HttpRequestParser
1219HttpResponseParserPy = HttpResponseParser
1220RawRequestMessagePy = RawRequestMessage
1221RawResponseMessagePy = RawResponseMessage
1223with suppress(ImportError):
1224 if not NO_EXTENSIONS:
1225 from ._http_parser import ( # type: ignore[import-not-found,no-redef]
1226 HttpRequestParser,
1227 HttpResponseParser,
1228 RawRequestMessage,
1229 RawResponseMessage,
1230 )
1232 HttpRequestParserC = HttpRequestParser
1233 HttpResponseParserC = HttpResponseParser
1234 RawRequestMessageC = RawRequestMessage
1235 RawResponseMessageC = RawResponseMessage