Coverage for /pythoncovmergedfiles/medio/medio/src/aiohttp/aiohttp/http_parser.py: 78%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import abc
2import asyncio
3import re
4import string
5import sys
6from contextlib import suppress
7from enum import IntEnum
8from re import Pattern
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 ClassVar,
13 Final,
14 Generic,
15 Literal,
16 NamedTuple,
17 TypeVar,
18)
20from multidict import CIMultiDict, istr
21from yarl import URL
23from . import hdrs
24from .base_protocol import BaseProtocol
25from .compression_utils import (
26 HAS_BROTLI,
27 HAS_ZSTD,
28 BrotliDecompressor,
29 ZLibDecompressor,
30 ZSTDDecompressor,
31)
32from .helpers import (
33 _EXC_SENTINEL,
34 DEBUG,
35 DEFAULT_CHUNK_SIZE,
36 EMPTY_BODY_METHODS,
37 EMPTY_BODY_STATUS_CODES,
38 NO_EXTENSIONS,
39 BaseTimerContext,
40 HeadersDictProxy,
41 set_exception,
42)
43from .http_exceptions import (
44 BadHttpMessage,
45 BadHttpMethod,
46 BadStatusLine,
47 ContentEncodingError,
48 ContentLengthError,
49 InvalidHeader,
50 InvalidURLError,
51 LineTooLong,
52 TransferEncodingError,
53)
54from .http_writer import HttpVersion, HttpVersion10, HttpVersion11
55from .streams import EMPTY_PAYLOAD, StreamReader
56from .typedefs import RawHeaders
58if TYPE_CHECKING:
59 from .client_proto import ResponseHandler
61__all__ = (
62 "HeadersParser",
63 "HttpParser",
64 "HttpRequestParser",
65 "HttpResponseParser",
66 "RawRequestMessage",
67 "RawResponseMessage",
68)
70_T = TypeVar("_T")
72_SEP = Literal[b"\r\n", b"\n"]
74ASCIISET: Final[set[str]] = set(string.printable)
76# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview
77# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens
78#
79# method = token
80# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
81# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
82# token = 1*tchar
83_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~")
84TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+")
85# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5
86_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(
87 r"[\x00-\x08\x0a-\x1f\x7f]"
88)
89_TARGET_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(r"[\x00-\x1f\x7f]")
90VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII)
91DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII)
92HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+")
95def _has_authority(target: str) -> bool:
96 """Tell if an absolute-form request target has "//" and a non-empty authority.
98 RFC 9110 section 4.2 requires a host for http and https. yarl's default
99 WHATWG mode reads a host from "http:host/p" or "http:///host/p", so the
100 target is checked before it is parsed.
101 """
102 rest = target.partition(":")[2]
103 return rest[:2] == "//" and rest[2:3] not in ("", "/", "?", "#")
106# RFC 9110 singleton headers — duplicates are rejected in strict mode.
107# In lax mode (response parser default), the check is skipped entirely
108# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
109# duplicate headers like Content-Type or Server.
110# Lowercased for case-insensitive matching against wire names.
111SINGLETON_HEADERS: Final[frozenset[str]] = frozenset(
112 {
113 "content-length",
114 "content-location",
115 "content-range",
116 "content-type",
117 "etag",
118 "host",
119 "max-forwards",
120 "server",
121 "transfer-encoding",
122 "user-agent",
123 }
124)
127class RawRequestMessage(NamedTuple):
128 method: str
129 path: str
130 version: HttpVersion
131 headers: HeadersDictProxy
132 raw_headers: RawHeaders
133 should_close: bool
134 compression: str | None
135 upgrade: bool
136 chunked: bool
137 url: URL
140class RawResponseMessage(NamedTuple):
141 version: HttpVersion
142 code: int
143 reason: str
144 headers: HeadersDictProxy
145 raw_headers: RawHeaders
146 should_close: bool
147 compression: str | None
148 upgrade: bool
149 chunked: bool
152_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage)
155class PayloadState(IntEnum):
156 PAYLOAD_COMPLETE = 0
157 PAYLOAD_NEEDS_INPUT = 1
158 PAYLOAD_HAS_PENDING_INPUT = 2
161class ParseState(IntEnum):
162 PARSE_NONE = 0
163 PARSE_LENGTH = 1
164 PARSE_CHUNKED = 2
165 PARSE_UNTIL_EOF = 3
168class ChunkState(IntEnum):
169 PARSE_CHUNKED_SIZE = 0
170 PARSE_CHUNKED_CHUNK = 1
171 PARSE_CHUNKED_CHUNK_EOF = 2
172 PARSE_TRAILERS = 4
175class HeadersParser:
176 def __init__(self, max_field_size: int = 8190, lax: bool = False) -> None:
177 self.max_field_size = max_field_size
178 self._lax = lax
180 def parse_headers(self, lines: list[bytes]) -> tuple[HeadersDictProxy, RawHeaders]:
181 headers: CIMultiDict[str] = CIMultiDict()
182 # note: "raw" does not mean inclusion of OWS before/after the field value
183 raw_headers = []
185 lines_idx = 0
186 line = lines[lines_idx]
187 line_count = len(lines)
189 while line:
190 # Parse initial header name : value pair.
191 try:
192 bname, bvalue = line.split(b":", 1)
193 except ValueError:
194 raise InvalidHeader(line) from None
196 if len(bname) == 0:
197 raise InvalidHeader(bname)
199 # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2
200 if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"}
201 raise InvalidHeader(line)
203 bvalue = bvalue.lstrip(b" \t")
204 name = bname.decode("utf-8", "surrogateescape")
205 if not TOKENRE.fullmatch(name):
206 raise InvalidHeader(bname)
208 # next line
209 lines_idx += 1
210 line = lines[lines_idx]
212 # consume continuation lines
213 continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t')
215 # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding
216 if continuation:
217 header_length = len(bvalue)
218 bvalue_lst = [bvalue]
219 while continuation:
220 header_length += len(line)
221 if header_length > self.max_field_size:
222 header_line = bname + b": " + b"".join(bvalue_lst)
223 raise LineTooLong(
224 header_line[:100] + b"...", self.max_field_size
225 )
226 bvalue_lst.append(line)
228 # next line
229 lines_idx += 1
230 if lines_idx < line_count:
231 line = lines[lines_idx]
232 if line:
233 continuation = line[0] in (32, 9) # (' ', '\t')
234 else:
235 line = b""
236 break
237 bvalue = b"".join(bvalue_lst)
239 bvalue = bvalue.strip(b" \t")
240 value = bvalue.decode("utf-8", "surrogateescape")
242 # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5
243 if self._lax:
244 if "\n" in value or "\r" in value or "\x00" in value:
245 raise InvalidHeader(bvalue)
246 elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value):
247 raise InvalidHeader(bvalue)
249 if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS:
250 raise BadHttpMessage(f"Duplicate '{name}' header found.")
251 headers.add(name, value)
252 raw_headers.append((bname, bvalue))
254 return (HeadersDictProxy(headers), tuple(raw_headers))
257def _is_supported_upgrade(headers: HeadersDictProxy) -> bool:
258 """Check if the upgrade header is supported."""
259 u = headers.get(hdrs.UPGRADE, "")
260 # .lower() can transform non-ascii characters.
261 return u.isascii() and u.lower() in {"tcp", "websocket"}
264class HttpParser(abc.ABC, Generic[_MsgT]):
265 lax: ClassVar[bool] = False
267 def __init__(
268 self,
269 protocol: BaseProtocol,
270 loop: asyncio.AbstractEventLoop,
271 limit: int,
272 max_line_size: int = 8190,
273 max_headers: int = 128,
274 max_field_size: int = 8190,
275 timer: BaseTimerContext | None = None,
276 code: int | None = None,
277 method: str | None = None,
278 payload_exception: type[BaseException] | None = None,
279 response_with_body: bool = True,
280 read_until_eof: bool = False,
281 auto_decompress: bool = True,
282 max_msg_queue_size: int = 0,
283 ) -> None:
284 self.protocol = protocol
285 self.loop = loop
286 self.max_line_size = max_line_size
287 self.max_field_size = max_field_size
288 self.max_headers = max_headers
289 self.timer = timer
290 self.code = code
291 self.method = method
292 self.payload_exception = payload_exception
293 self.response_with_body = response_with_body
294 self.read_until_eof = read_until_eof
296 self._lines: list[bytes] = []
297 self._tail = b""
298 self._upgraded = False
299 self._pending_upgrade = False
300 self._payload = None
301 self._payload_parser: HttpPayloadParser | None = None
302 self._payload_has_more_data = False
303 self._auto_decompress = auto_decompress
304 self._limit = limit
305 self._headers_parser = HeadersParser(max_field_size, self.lax)
306 # Stop emitting messages once this many are queued unconsumed (0 = off).
307 self._max_msg_queue_size = max_msg_queue_size
308 self._msg_in_flight = 0
310 @abc.abstractmethod
311 def parse_message(self, lines: list[bytes]) -> _MsgT: ...
313 @abc.abstractmethod
314 def _is_chunked_te(self, te: str) -> bool: ...
316 def pause_reading(self) -> None:
317 assert self._payload_parser is not None
318 self._payload_parser.pause_reading()
320 def message_consumed(self) -> None:
321 """Protocol drained a queued message; free a slot for parsing."""
322 if self._msg_in_flight > 0:
323 self._msg_in_flight -= 1
325 def feed_eof(self) -> _MsgT | None:
326 if self._payload_parser is not None:
327 self._payload_parser.feed_eof()
328 if self._payload_parser.done:
329 self._payload_parser = None
330 else:
331 # try to extract partial message
332 if self._tail:
333 self._lines.append(self._tail)
335 if self._lines:
336 if self._lines[-1] != "\r\n":
337 self._lines.append(b"")
338 with suppress(Exception):
339 return self.parse_message(self._lines)
340 return None
342 def feed_data(
343 self,
344 data: bytes,
345 SEP: _SEP = b"\r\n",
346 EMPTY: bytes = b"",
347 CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH,
348 METH_CONNECT: str = hdrs.METH_CONNECT,
349 SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1,
350 ) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]:
351 messages = []
353 if self._tail:
354 data, self._tail = self._tail + data, b""
356 data_len = len(data)
357 start_pos = 0
358 loop = self.loop
359 max_line_length = self.max_line_size
361 should_close = False
362 while start_pos < data_len or self._payload_has_more_data:
363 # read HTTP message (request/response line + headers), \r\n\r\n
364 # and split by lines
365 if self._payload_parser is None and not self._upgraded:
366 if (
367 self._max_msg_queue_size
368 and self._msg_in_flight >= self._max_msg_queue_size
369 ):
370 # Queue full: buffer the rest and stop. Safe pause point;
371 # any preceding body is consumed before the next request
372 # line. Resumes via feed_data(b"") when the queue drains.
373 self._tail = data[start_pos:]
374 # The remainder now lives in self._tail only. Don't return it.
375 data = EMPTY
376 break
377 pos = data.find(SEP, start_pos)
378 # consume \r\n
379 if pos == start_pos and not self._lines:
380 start_pos = pos + len(SEP)
381 continue
383 if pos >= start_pos:
384 if should_close:
385 raise BadHttpMessage("Data after `Connection: close`")
387 # line found
388 line = data[start_pos:pos]
389 if SEP == b"\n": # For lax response parsing
390 line = line.rstrip(b"\r")
391 if len(line) > max_line_length:
392 raise LineTooLong(line[:100] + b"...", max_line_length)
394 self._lines.append(line)
395 # After processing the status/request line, everything is a header.
396 max_line_length = self.max_field_size
398 if len(self._lines) > self.max_headers:
399 raise BadHttpMessage("Too many headers received")
401 start_pos = pos + len(SEP)
403 # \r\n\r\n found
404 if self._lines[-1] == EMPTY:
405 max_trailers = self.max_headers - len(self._lines)
406 try:
407 msg: _MsgT = self.parse_message(self._lines)
408 finally:
409 self._lines.clear()
411 def get_content_length() -> int | None:
412 # payload length
413 length_hdr = msg.headers.get(CONTENT_LENGTH)
414 if length_hdr is None:
415 return None
417 # Shouldn't allow +/- or other number formats.
418 # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2
419 # msg.headers is already stripped of leading/trailing wsp
420 if not DIGITS.fullmatch(length_hdr):
421 raise InvalidHeader(CONTENT_LENGTH)
423 return int(length_hdr)
425 length = get_content_length()
426 # do not support old websocket spec
427 if SEC_WEBSOCKET_KEY1 in msg.headers:
428 raise InvalidHeader(SEC_WEBSOCKET_KEY1)
430 upgraded = msg.upgrade and _is_supported_upgrade(msg.headers)
432 method = getattr(msg, "method", self.method)
433 # code is only present on responses
434 code = getattr(msg, "code", 0)
436 assert self.protocol is not None
437 # calculate payload
438 # https://www.rfc-editor.org/info/rfc9112/#name-message-body-length
439 # https://www.rfc-editor.org/info/rfc9110/#section-9.3.1-6
440 # EMPTY_BODY_METHODS should only apply to responses.
441 # self.method is None on request parser.
442 empty_body = code in EMPTY_BODY_STATUS_CODES or bool(
443 self.method and self.method in EMPTY_BODY_METHODS
444 )
445 if not empty_body and (
446 (length is not None and length > 0) or msg.chunked
447 ):
448 payload = StreamReader(
449 self.protocol,
450 timer=self.timer,
451 loop=loop,
452 limit=self._limit,
453 )
454 payload_parser = HttpPayloadParser(
455 payload,
456 length=length,
457 chunked=msg.chunked,
458 method=method,
459 compression=msg.compression,
460 code=self.code,
461 response_with_body=self.response_with_body,
462 auto_decompress=self._auto_decompress,
463 lax=self.lax,
464 headers_parser=self._headers_parser,
465 max_line_size=self.max_line_size,
466 max_field_size=self.max_field_size,
467 max_trailers=max_trailers,
468 limit=self._limit,
469 )
470 if not payload_parser.done:
471 self._payload_parser = payload_parser
472 # https://www.rfc-editor.org/info/rfc9110/#section-7.8-15
473 # Defer any requested upgrade until the
474 # complete request has been read.
475 self._pending_upgrade = upgraded
476 elif method == METH_CONNECT:
477 assert isinstance(msg, RawRequestMessage)
478 payload = StreamReader(
479 self.protocol,
480 timer=self.timer,
481 loop=loop,
482 limit=self._limit,
483 )
484 self._upgraded = True
485 self._payload_parser = HttpPayloadParser(
486 payload,
487 method=msg.method,
488 compression=msg.compression,
489 auto_decompress=self._auto_decompress,
490 lax=self.lax,
491 headers_parser=self._headers_parser,
492 max_line_size=self.max_line_size,
493 max_field_size=self.max_field_size,
494 max_trailers=max_trailers,
495 limit=self._limit,
496 )
497 elif not empty_body and length is None and self.read_until_eof:
498 payload = StreamReader(
499 self.protocol,
500 timer=self.timer,
501 loop=loop,
502 limit=self._limit,
503 )
504 payload_parser = HttpPayloadParser(
505 payload,
506 length=length,
507 chunked=msg.chunked,
508 method=method,
509 compression=msg.compression,
510 code=self.code,
511 response_with_body=self.response_with_body,
512 auto_decompress=self._auto_decompress,
513 lax=self.lax,
514 headers_parser=self._headers_parser,
515 max_line_size=self.max_line_size,
516 max_field_size=self.max_field_size,
517 max_trailers=max_trailers,
518 limit=self._limit,
519 )
520 if not payload_parser.done:
521 self._payload_parser = payload_parser
522 elif upgraded:
523 # No body to read, so the connection switches to
524 # the upgraded protocol immediately.
525 self._upgraded = True
526 payload = EMPTY_PAYLOAD
527 else:
528 payload = EMPTY_PAYLOAD
530 messages.append((msg, payload))
531 if self._max_msg_queue_size:
532 self._msg_in_flight += 1
533 should_close = msg.should_close
534 else:
535 self._tail = data[start_pos:]
536 # A bare LF here means CRLF was required:
537 # reject instead of buffering, else a following request's
538 # bytes get appended to this line and leak in the error.
539 if b"\n" in self._tail:
540 raise BadHttpMessage("Bad line ending, expected CRLF")
541 if len(self._tail) > self.max_line_size:
542 raise LineTooLong(self._tail[:100] + b"...", self.max_line_size)
543 data = EMPTY
544 break
546 # no parser, just store
547 elif self._payload_parser is None and self._upgraded:
548 assert not self._lines
549 break
551 # feed payload
552 else:
553 assert not self._lines
554 assert self._payload_parser is not None
555 try:
556 payload_state, data = self._payload_parser.feed_data(
557 data[start_pos:], SEP
558 )
559 except Exception as underlying_exc:
560 reraised_exc: BaseException = underlying_exc
561 if self.payload_exception is not None:
562 reraised_exc = self.payload_exception(str(underlying_exc))
564 set_exception(
565 self._payload_parser.payload,
566 reraised_exc,
567 underlying_exc,
568 )
570 payload_state = PayloadState.PAYLOAD_COMPLETE
571 data = b""
572 if isinstance(
573 underlying_exc, (InvalidHeader, TransferEncodingError)
574 ):
575 raise
577 self._payload_has_more_data = (
578 payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT
579 )
581 if payload_state is not PayloadState.PAYLOAD_COMPLETE:
582 # We've either consumed all available data, or we're pausing
583 # until the reader buffer is freed up.
584 break
586 start_pos = 0
587 data_len = len(data)
588 self._payload_parser = None
589 if self._pending_upgrade:
590 # Body fully read: the deferred upgrade takes effect and
591 # the rest of the connection is the upgraded protocol.
592 self._upgraded = True
593 self._pending_upgrade = False
595 if data and start_pos < data_len:
596 data = data[start_pos:]
597 else:
598 data = EMPTY
600 return messages, self._upgraded, data
602 def parse_headers(
603 self, lines: list[bytes]
604 ) -> tuple[HeadersDictProxy, RawHeaders, bool | None, str | None, bool, bool]:
605 """Parses RFC 5322 headers from a stream.
607 Line continuations are supported. Returns list of header name
608 and value pairs. Header name is in upper case.
609 """
610 headers, raw_headers = self._headers_parser.parse_headers(lines)
611 close_conn = None
612 encoding = None
613 upgrade = False
614 chunked = False
616 # keep-alive and protocol switching
617 # RFC 9110 section 7.6.1 defines Connection as a comma-separated list.
618 # We use a simple comma split here rather than getall() for performance,
619 # as the target tokens (close, keep-alive, upgrade) are simple ASCII
620 # values that never contain commas.
621 conn_values = headers.get(hdrs.CONNECTION)
622 if conn_values:
623 conn_tokens = {
624 token.lower()
625 for token in (part.strip(" \t") for part in conn_values.split(","))
626 if token and token.isascii()
627 }
629 if "close" in conn_tokens:
630 close_conn = True
631 elif "keep-alive" in conn_tokens:
632 close_conn = False
634 # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols
635 if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE):
636 upgrade = True
638 # encoding
639 enc = headers.get(hdrs.CONTENT_ENCODING, "")
640 if enc.isascii(): # Must be checked before .lower()
641 enc = enc.lower()
642 if enc in {"gzip", "deflate", "br", "zstd"}:
643 encoding = enc
645 # chunking
646 te = headers.get(hdrs.TRANSFER_ENCODING)
647 if te is not None:
648 if self._is_chunked_te(te):
649 chunked = True
651 if hdrs.CONTENT_LENGTH in headers:
652 raise BadHttpMessage(
653 "Transfer-Encoding can't be present with Content-Length",
654 )
656 return (headers, raw_headers, close_conn, encoding, upgrade, chunked)
658 def set_upgraded(self, val: bool) -> None:
659 """Set connection upgraded (to websocket) mode.
661 :param bool val: new state.
662 """
663 self._upgraded = val
666class HttpRequestParser(HttpParser[RawRequestMessage]):
667 """Read request status line.
669 Exception .http_exceptions.BadStatusLine
670 could be raised in case of any errors in status line.
671 Returns RawRequestMessage.
672 """
674 def parse_message(self, lines: list[bytes]) -> RawRequestMessage:
675 # request line
676 line = lines[0].decode("utf-8", "surrogateescape")
677 try:
678 method, path, version = line.split(" ", maxsplit=2)
679 except ValueError:
680 raise BadHttpMethod(line) from None
682 # method
683 if not TOKENRE.fullmatch(method):
684 raise BadHttpMethod(method)
685 method = method.upper()
687 # https://www.rfc-editor.org/rfc/rfc9112#section-3.2-4
688 if _TARGET_FORBIDDEN_CTL_RE.search(path):
689 raise InvalidURLError(
690 path.encode(errors="surrogateescape").decode("latin1")
691 )
693 # version
694 match = VERSRE.fullmatch(version)
695 if match is None:
696 raise BadStatusLine(line)
697 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
699 if method == "CONNECT":
700 # authority-form,
701 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
702 try:
703 url = URL.build(authority=path, encoded=True)
704 host = url.raw_host
705 except ValueError:
706 # Duplicated so mypy understands that url must be defined below.
707 raise InvalidURLError(
708 path.encode(errors="surrogateescape").decode("latin1")
709 )
710 if host is None:
711 raise InvalidURLError(
712 path.encode(errors="surrogateescape").decode("latin1")
713 )
714 elif path.startswith("/"):
715 # origin-form,
716 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
717 path_part, _hash_separator, url_fragment = path.partition("#")
718 path_part, _question_mark_separator, qs_part = path_part.partition("?")
720 # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based
721 # NOTE: parser does, otherwise it results into the same
722 # NOTE: HTTP Request-Line input producing different
723 # NOTE: `yarl.URL()` objects
724 url = URL.build(
725 path=path_part,
726 query_string=qs_part,
727 fragment=url_fragment,
728 encoded=True,
729 )
730 elif path == "*" and method == "OPTIONS":
731 # asterisk-form,
732 url = URL(path, encoded=True)
733 else:
734 # absolute-form for proxy maybe,
735 # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
736 if not _has_authority(path):
737 raise InvalidURLError(
738 path.encode(errors="surrogateescape").decode("latin1")
739 )
740 try:
741 url = URL(path, encoded=True)
742 host = url.raw_host
743 except ValueError:
744 # Duplicated so mypy understands that url must be defined below.
745 raise InvalidURLError(
746 path.encode(errors="surrogateescape").decode("latin1")
747 )
748 # https://www.rfc-editor.org/rfc/rfc9110#section-4.2.1-4
749 if host is None:
750 raise InvalidURLError(
751 path.encode(errors="surrogateescape").decode("latin1")
752 )
754 # read headers
755 (
756 headers,
757 raw_headers,
758 close,
759 compression,
760 upgrade,
761 chunked,
762 ) = self.parse_headers(lines[1:])
764 if version_o == HttpVersion11 and hdrs.HOST not in headers:
765 raise BadHttpMessage("Missing 'Host' header in request.")
767 if close is None: # then the headers weren't set in the request
768 if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close
769 close = True
770 else: # HTTP 1.1 must ask to close.
771 close = False
773 return RawRequestMessage(
774 method,
775 path,
776 version_o,
777 headers,
778 raw_headers,
779 close,
780 compression,
781 upgrade,
782 chunked,
783 url,
784 )
786 def _is_chunked_te(self, te: str) -> bool:
787 # https://www.rfc-editor.org/rfc/rfc9112#section-7.1-3
788 # "A sender MUST NOT apply the chunked transfer coding more
789 # than once to a message body"
790 parts = [p.strip(" \t") for p in te.split(",")]
791 chunked_count = sum(1 for p in parts if p.isascii() and p.lower() == "chunked")
792 if chunked_count > 1:
793 raise BadHttpMessage("Request has duplicate `chunked` Transfer-Encoding")
794 last = parts[-1]
795 # .lower() transforms some non-ascii chars, so must check first.
796 if last.isascii() and last.lower() == "chunked":
797 return True
798 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3
799 raise BadHttpMessage("Request has invalid `Transfer-Encoding`")
802class HttpResponseParser(HttpParser[RawResponseMessage]):
803 """Read response status line and headers.
805 BadStatusLine could be raised in case of any errors in status line.
806 Returns RawResponseMessage.
807 """
809 protocol: "ResponseHandler"
811 # Lax mode should only be enabled on response parser.
812 lax = not DEBUG
814 def feed_data(
815 self,
816 data: bytes,
817 SEP: _SEP | None = None,
818 *args: Any,
819 **kwargs: Any,
820 ) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]:
821 if SEP is None:
822 SEP = b"\r\n" if DEBUG else b"\n"
823 return super().feed_data(data, SEP, *args, **kwargs)
825 def parse_message(self, lines: list[bytes]) -> RawResponseMessage:
826 line = lines[0].decode("utf-8", "surrogateescape")
827 try:
828 version, status = line.split(maxsplit=1)
829 except ValueError:
830 raise BadStatusLine(line) from None
832 try:
833 status, reason = status.split(maxsplit=1)
834 except ValueError:
835 status = status.strip()
836 reason = ""
838 # version
839 match = VERSRE.fullmatch(version)
840 if match is None:
841 raise BadStatusLine(line)
842 version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
844 # The status code is a three-digit ASCII number, no padding
845 if len(status) != 3 or not DIGITS.fullmatch(status):
846 raise BadStatusLine(line)
847 status_i = int(status)
849 # read headers
850 (
851 headers,
852 raw_headers,
853 close,
854 compression,
855 upgrade,
856 chunked,
857 ) = self.parse_headers(lines[1:])
859 if close is None:
860 if version_o <= HttpVersion10:
861 close = True
862 # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length
863 elif 100 <= status_i < 200 or status_i in {204, 304}:
864 close = False
865 elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers:
866 close = False
867 else:
868 # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8
869 close = True
871 return RawResponseMessage(
872 version_o,
873 status_i,
874 reason.strip(),
875 headers,
876 raw_headers,
877 close,
878 compression,
879 upgrade,
880 chunked,
881 )
883 def _is_chunked_te(self, te: str) -> bool:
884 # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2
885 return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked"
888class HttpPayloadParser:
889 def __init__(
890 self,
891 payload: StreamReader,
892 length: int | None = None,
893 chunked: bool = False,
894 compression: str | None = None,
895 code: int | None = None,
896 method: str | None = None,
897 response_with_body: bool = True,
898 auto_decompress: bool = True,
899 lax: bool = False,
900 *,
901 headers_parser: HeadersParser,
902 max_line_size: int = 8190,
903 max_field_size: int = 8190,
904 max_trailers: int = 128,
905 limit: int = DEFAULT_CHUNK_SIZE,
906 ) -> None:
907 self._length = 0
908 self._paused = False
909 self._type = ParseState.PARSE_UNTIL_EOF
910 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
911 self._chunk_size = 0
912 self._chunk_tail = b""
913 self._auto_decompress = auto_decompress
914 self._lax = lax
915 self._headers_parser = headers_parser
916 self._max_line_size = max_line_size
917 self._max_field_size = max_field_size
918 self._max_trailers = max_trailers
919 self._more_data_available = False
920 self._trailer_lines: list[bytes] = []
921 self.done = False
922 self._eof_pending = False
924 # payload decompression wrapper
925 if response_with_body and compression and self._auto_decompress:
926 real_payload: StreamReader | DeflateBuffer = DeflateBuffer(
927 payload, compression, max_decompress_size=limit
928 )
929 else:
930 real_payload = payload
932 # payload parser
933 if not response_with_body:
934 # don't parse payload if it's not expected to be received
935 self._type = ParseState.PARSE_NONE
936 real_payload.feed_eof()
937 self.done = True
938 elif chunked:
939 self._type = ParseState.PARSE_CHUNKED
940 elif length is not None:
941 self._type = ParseState.PARSE_LENGTH
942 self._length = length
943 self._length_expected = length
944 if self._length == 0:
945 real_payload.feed_eof()
946 self.done = True
948 self.payload = real_payload
950 def pause_reading(self) -> None:
951 self._paused = True
953 def feed_eof(self) -> None:
954 if self._type == ParseState.PARSE_UNTIL_EOF:
955 self._eof_pending = True
956 while self._more_data_available:
957 if self._paused:
958 self._paused = False
959 return # Will resume via feed_data(b"") later
960 self._more_data_available = self.payload.feed_data(b"")
961 self.payload.feed_eof()
962 self.done = True
963 self._eof_pending = False
964 elif self._type == ParseState.PARSE_LENGTH:
965 if self._length:
966 received = self._length_expected - self._length
967 raise ContentLengthError(
968 f"Not enough data to satisfy content length header "
969 f"(received {received} of {self._length_expected} bytes)."
970 )
971 # Body has already been received, but parser paused.
972 while self._more_data_available:
973 if self._paused:
974 self._paused = False
975 return # Will resume via feed_data(b"") later
976 self._more_data_available = self.payload.feed_data(b"")
977 self.payload.feed_eof()
978 self.done = True
979 elif self._type == ParseState.PARSE_CHUNKED:
980 raise TransferEncodingError(
981 "Not enough data to satisfy transfer length header."
982 )
984 def feed_data(
985 self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";"
986 ) -> tuple[PayloadState, bytes]:
987 """Receive a chunk of data to process.
989 Return:
990 PayloadState - The current state of payload processing.
991 This function may be called with empty bytes after returning
992 PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause.
993 bytes - If payload is complete, this is the unconsumed bytes intended for the
994 next message/payload, b"" otherwise.
995 """
996 # Read specified amount of bytes
997 if self._type == ParseState.PARSE_LENGTH:
998 if self._chunk_tail:
999 chunk = self._chunk_tail + chunk
1000 self._chunk_tail = b""
1002 required = self._length
1003 self._length = max(required - len(chunk), 0)
1004 self._more_data_available = self.payload.feed_data(chunk[:required])
1005 while self._more_data_available:
1006 if self._paused:
1007 self._paused = False
1008 self._chunk_tail = chunk[required:]
1009 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1010 self._more_data_available = self.payload.feed_data(b"")
1012 if self._length == 0:
1013 self.payload.feed_eof()
1014 return PayloadState.PAYLOAD_COMPLETE, chunk[required:]
1015 # Chunked transfer encoding parser
1016 elif self._type == ParseState.PARSE_CHUNKED:
1017 if self._chunk_tail:
1018 # We should check the length is sane when not processing payload body.
1019 if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK:
1020 max_line_length = self._max_line_size
1021 if self._chunk == ChunkState.PARSE_TRAILERS:
1022 max_line_length = self._max_field_size
1023 if len(self._chunk_tail) > max_line_length:
1024 raise LineTooLong(
1025 self._chunk_tail[:100] + b"...", max_line_length
1026 )
1028 chunk = self._chunk_tail + chunk
1029 self._chunk_tail = b""
1031 while chunk or self._more_data_available:
1032 # read next chunk size
1033 if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
1034 pos = chunk.find(SEP)
1035 if pos >= 0:
1036 # Only chunk-size lines reach here; trailers enforce
1037 # _max_field_size separately in PARSE_TRAILERS below.
1038 if pos > self._max_line_size:
1039 raise LineTooLong(chunk[:100] + b"...", self._max_line_size)
1040 i = chunk.find(CHUNK_EXT, 0, pos)
1041 if i >= 0:
1042 size_b = chunk[:i] # strip chunk-extensions
1043 # Verify no LF in the chunk-extension
1044 if b"\n" in (ext := chunk[i:pos]):
1045 exc = TransferEncodingError(
1046 f"Unexpected LF in chunk-extension: {ext!r}"
1047 )
1048 set_exception(self.payload, exc)
1049 raise exc
1050 else:
1051 size_b = chunk[:pos]
1053 if self._lax: # Allow whitespace in lax mode.
1054 size_b = size_b.strip()
1056 if not re.fullmatch(HEXDIGITS, size_b):
1057 exc = TransferEncodingError(
1058 chunk[:pos].decode("ascii", "surrogateescape")
1059 )
1060 set_exception(self.payload, exc)
1061 raise exc
1062 size = int(bytes(size_b), 16)
1064 chunk = chunk[pos + len(SEP) :]
1065 if size == 0: # eof marker
1066 self._chunk = ChunkState.PARSE_TRAILERS
1067 if self._lax and chunk.startswith(b"\r"):
1068 chunk = chunk[1:]
1069 else:
1070 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK
1071 self._chunk_size = size
1072 self.payload.begin_http_chunk_receiving()
1073 else:
1074 if b"\n" in chunk:
1075 exc = TransferEncodingError(
1076 "Bad chunk-size line ending, expected CRLF"
1077 )
1078 set_exception(self.payload, exc)
1079 raise exc
1080 self._chunk_tail = chunk
1081 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1083 # read chunk and feed buffer
1084 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK:
1085 if self._paused:
1086 self._paused = False
1087 self._chunk_tail = chunk
1088 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1090 required = self._chunk_size
1091 self._chunk_size = max(required - len(chunk), 0)
1092 self._more_data_available = self.payload.feed_data(chunk[:required])
1093 chunk = chunk[required:]
1095 if self._more_data_available:
1096 continue
1098 if self._chunk_size:
1099 self._paused = False
1100 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1101 self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
1102 self.payload.end_http_chunk_receiving()
1104 # toss the CRLF at the end of the chunk
1105 if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF:
1106 if self._lax and chunk.startswith(b"\r"):
1107 chunk = chunk[1:]
1108 if chunk[: len(SEP)] == SEP:
1109 chunk = chunk[len(SEP) :]
1110 self._chunk = ChunkState.PARSE_CHUNKED_SIZE
1111 elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]:
1112 exc = TransferEncodingError(
1113 "Chunk size mismatch: expected CRLF after chunk data"
1114 )
1115 set_exception(self.payload, exc)
1116 raise exc
1117 else:
1118 self._chunk_tail = chunk
1119 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1121 if self._chunk == ChunkState.PARSE_TRAILERS:
1122 pos = chunk.find(SEP)
1123 if pos < 0: # No line found
1124 if b"\n" in chunk:
1125 exc = TransferEncodingError(
1126 "Bad trailer line ending, expected CRLF"
1127 )
1128 set_exception(self.payload, exc)
1129 raise exc
1130 self._chunk_tail = chunk
1131 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1133 line = chunk[:pos]
1134 chunk = chunk[pos + len(SEP) :]
1135 if SEP == b"\n": # For lax response parsing
1136 line = line.rstrip(b"\r")
1138 if len(line) > self._max_field_size:
1139 raise LineTooLong(line[:100] + b"...", self._max_field_size)
1141 self._trailer_lines.append(line)
1143 if len(self._trailer_lines) > self._max_trailers:
1144 raise BadHttpMessage("Too many trailers received")
1146 # \r\n\r\n found, end of stream
1147 if self._trailer_lines[-1] == b"":
1148 # Headers and trailers are defined the same way,
1149 # so we reuse the HeadersParser here.
1150 try:
1151 trailers, raw_trailers = self._headers_parser.parse_headers(
1152 self._trailer_lines
1153 )
1154 finally:
1155 self._trailer_lines.clear()
1156 self.payload.feed_eof()
1157 return PayloadState.PAYLOAD_COMPLETE, chunk
1159 # Read all bytes until eof
1160 elif self._type == ParseState.PARSE_UNTIL_EOF:
1161 self._more_data_available = self.payload.feed_data(chunk)
1162 while self._more_data_available:
1163 if self._paused:
1164 self._paused = False
1165 return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
1166 self._more_data_available = self.payload.feed_data(b"")
1168 if self._eof_pending:
1169 self.payload.feed_eof()
1170 self.done = True
1171 self._eof_pending = False
1172 return PayloadState.PAYLOAD_COMPLETE, b""
1174 return PayloadState.PAYLOAD_NEEDS_INPUT, b""
1177class DeflateBuffer:
1178 """DeflateStream decompress stream and feed data into specified stream."""
1180 def __init__(
1181 self,
1182 out: StreamReader,
1183 encoding: str | None,
1184 max_decompress_size: int = DEFAULT_CHUNK_SIZE,
1185 ) -> None:
1186 self.out = out
1187 self.size = 0
1188 out.total_compressed_bytes = self.size
1189 self.encoding = encoding
1190 self._started_decoding = False
1192 self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor
1193 if encoding == "br":
1194 if not HAS_BROTLI:
1195 raise ContentEncodingError(
1196 "Can not decode content-encoding: brotli (br). "
1197 "Please install `Brotli`"
1198 )
1199 self.decompressor = BrotliDecompressor()
1200 elif encoding == "zstd":
1201 if not HAS_ZSTD:
1202 raise ContentEncodingError(
1203 "Can not decode content-encoding: zstandard (zstd). "
1204 "Please install `backports.zstd`"
1205 )
1206 self.decompressor = ZSTDDecompressor()
1207 else:
1208 self.decompressor = ZLibDecompressor(encoding=encoding)
1210 self._max_decompress_size = max_decompress_size
1212 def set_exception(
1213 self,
1214 exc: type[BaseException] | BaseException,
1215 exc_cause: BaseException = _EXC_SENTINEL,
1216 ) -> None:
1217 set_exception(self.out, exc, exc_cause)
1219 def feed_data(self, chunk: bytes) -> bool:
1220 """Return True if more data is available and this method should be called again with b""."""
1221 self.size += len(chunk)
1222 self.out.total_compressed_bytes = self.size
1224 # Inspect the first real byte once to choose the decompressor. An empty
1225 # chunk (e.g. a chunk-size line arriving without body bytes) has no
1226 # header to sniff, so skip it and wait for the first data byte.
1227 if not self._started_decoding and chunk:
1228 # RFC1950
1229 # bits 0..3 = CM = 0b1000 = 8 = "deflate"
1230 # bits 4..7 = CINFO = 1..7 = windows size.
1231 if self.encoding == "deflate" and chunk[0] & 0xF != 8:
1232 # Change the decoder to decompress incorrectly compressed data
1233 # Actually we should issue a warning about non-RFC-compliant data.
1234 self.decompressor = ZLibDecompressor(
1235 encoding=self.encoding, suppress_deflate_header=True
1236 )
1237 self._started_decoding = True
1239 low_water = self.out._low_water
1240 max_length = (
1241 0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water)
1242 )
1243 try:
1244 chunk = self.decompressor.decompress_sync(chunk, max_length=max_length)
1245 except Exception:
1246 raise ContentEncodingError(
1247 "Can not decode content-encoding: %s" % self.encoding
1248 )
1250 if chunk:
1251 self.out.feed_data(chunk)
1252 return self.decompressor.data_available
1254 def feed_eof(self) -> None:
1255 chunk = self.decompressor.flush()
1256 # This should never contain data as we defer the call until exhausting
1257 # the decompression. If .flush() is returning data, this may indicate a
1258 # zip bomb vulnerability as it will decompress all remaining data at once.
1259 assert not chunk
1261 if self.size > 0:
1262 # decompressor is not brotli unless encoding is "br"
1263 if self.encoding == "deflate" and not self.decompressor.eof: # type: ignore[union-attr]
1264 raise ContentEncodingError("deflate")
1266 self.out.feed_eof()
1268 def begin_http_chunk_receiving(self) -> None:
1269 self.out.begin_http_chunk_receiving()
1271 def end_http_chunk_receiving(self) -> None:
1272 self.out.end_http_chunk_receiving()
1275HttpRequestParserPy = HttpRequestParser
1276HttpResponseParserPy = HttpResponseParser
1277RawRequestMessagePy = RawRequestMessage
1278RawResponseMessagePy = RawResponseMessage
1280with suppress(ImportError):
1281 if not NO_EXTENSIONS:
1282 from ._http_parser import ( # type: ignore[import-not-found,no-redef]
1283 HttpRequestParser,
1284 HttpResponseParser,
1285 RawRequestMessage,
1286 RawResponseMessage,
1287 )
1289 HttpRequestParserC = HttpRequestParser
1290 HttpResponseParserC = HttpResponseParser
1291 RawRequestMessageC = RawRequestMessage
1292 RawResponseMessageC = RawResponseMessage