Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import logging
4import os
5import shutil
6import sys
7import tempfile
8from enum import IntEnum
9from io import BufferedRandom, BytesIO
10from numbers import Number
11from typing import TYPE_CHECKING, cast
13from .decoders import Base64Decoder, QuotedPrintableDecoder
14from .exceptions import FileError, FormParserError, MultipartParseError, QuerystringParseError
16if TYPE_CHECKING:
17 from collections.abc import Callable
18 from typing import Any, Literal, Protocol, TypeAlias, TypedDict
20 class SupportsRead(Protocol):
21 def read(self, __n: int) -> bytes: ...
23 class QuerystringCallbacks(TypedDict, total=False):
24 on_field_start: Callable[[], None]
25 on_field_name: Callable[[bytes, int, int], None]
26 on_field_data: Callable[[bytes, int, int], None]
27 on_field_end: Callable[[], None]
28 on_end: Callable[[], None]
30 class OctetStreamCallbacks(TypedDict, total=False):
31 on_start: Callable[[], None]
32 on_data: Callable[[bytes, int, int], None]
33 on_end: Callable[[], None]
35 class MultipartCallbacks(TypedDict, total=False):
36 on_part_begin: Callable[[], None]
37 on_part_data: Callable[[bytes, int, int], None]
38 on_part_end: Callable[[], None]
39 on_header_begin: Callable[[], None]
40 on_header_field: Callable[[bytes, int, int], None]
41 on_header_value: Callable[[bytes, int, int], None]
42 on_header_end: Callable[[], None]
43 on_headers_finished: Callable[[], None]
44 on_end: Callable[[], None]
46 class FileConfig(TypedDict, total=False):
47 UPLOAD_DIR: str | bytes | None
48 UPLOAD_DELETE_TMP: bool
49 UPLOAD_KEEP_FILENAME: bool
50 UPLOAD_KEEP_EXTENSIONS: bool
51 MAX_MEMORY_FILE_SIZE: int
53 class FormParserConfig(FileConfig):
54 UPLOAD_ERROR_ON_BAD_CTE: bool
55 MAX_BODY_SIZE: float
56 MAX_HEADER_COUNT: int
57 MAX_HEADER_SIZE: int
59 CallbackName: TypeAlias = Literal[
60 "start",
61 "data",
62 "end",
63 "field_start",
64 "field_name",
65 "field_data",
66 "field_end",
67 "part_begin",
68 "part_data",
69 "part_end",
70 "header_begin",
71 "header_field",
72 "header_value",
73 "header_end",
74 "headers_finished",
75 ]
77# Unique missing object.
78_missing = object()
81def _noop_event() -> None:
82 pass
85def _noop_data(_data: bytes, _start: int, _end: int) -> None:
86 pass
89class QuerystringState(IntEnum):
90 """Querystring parser states.
92 These are used to keep track of the state of the parser, and are used to determine
93 what to do when new data is encountered.
94 """
96 BEFORE_FIELD = 0
97 FIELD_NAME = 1
98 FIELD_DATA = 2
101class MultipartState(IntEnum):
102 """Multipart parser states.
104 These are used to keep track of the state of the parser, and are used to determine
105 what to do when new data is encountered.
106 """
108 START = 0
109 START_BOUNDARY = 1
110 HEADER_FIELD_START = 2
111 HEADER_FIELD = 3
112 HEADER_VALUE_START = 4
113 HEADER_VALUE = 5
114 HEADER_VALUE_ALMOST_DONE = 6
115 HEADERS_ALMOST_DONE = 7
116 PART_DATA_START = 8
117 PART_DATA = 9
118 PART_DATA_END = 10
119 END_BOUNDARY = 11
120 END = 12
123# Flags for the multipart parser.
124FLAG_PART_BOUNDARY = 1
125FLAG_LAST_BOUNDARY = 2
127# Get constants. Since iterating over a str on Python 2 gives you a 1-length
128# string, but iterating over a bytes object on Python 3 gives you an integer,
129# we need to save these constants.
130CR = b"\r"[0]
131LF = b"\n"[0]
132COLON = b":"[0]
133SPACE = b" "[0]
134HYPHEN = b"-"[0]
135AMPERSAND = b"&"[0]
136LOWER_A = b"a"[0]
137LOWER_Z = b"z"[0]
138NULL = b"\x00"[0]
140# fmt: off
141# Mask for ASCII characters that can be http tokens.
142# Per RFC7230 - 3.2.6, this is all alpha-numeric characters
143# and these: !#$%&'*+-.^_`|~
144TOKEN_CHARS = (
145 b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
146 b"abcdefghijklmnopqrstuvwxyz"
147 b"0123456789"
148 b"!#$%&'*+-.^_`|~")
149TOKEN_CHARS_SET = frozenset(TOKEN_CHARS)
150# fmt: on
152DEFAULT_MAX_HEADER_COUNT = 8
153"""Default maximum number of headers allowed per multipart part."""
155DEFAULT_MAX_HEADER_SIZE = 4096 + 128
156"""Default maximum size of a single multipart header line, including syntax overhead."""
158MAX_BOUNDARY_LENGTH = 256
159"""Maximum allowed length of a multipart boundary.
161[RFC 2046 §5.1.1](https://datatracker.ietf.org/doc/html/rfc2046#section-5.1.1)
162recommends boundaries be at most 70 bytes. 256 bytes is generous headroom over
163every HTTP client.
164"""
167def _parseparam(s: str) -> list[str]:
168 # Vendored from the standard library's
169 # [`email.message._parseparam`](https://github.com/python/cpython/blob/v3.14.2/Lib/email/message.py#L73-L96)
170 # to split a header into its `;`-separated parts without treating a `;` inside a double-quoted string as a
171 # separator - and without the RFC 2231 decoding that `email.message.Message.get_params` would apply on top.
172 s = ";" + s
173 plist: list[str] = []
174 start = 0
175 while s.find(";", start) == start:
176 start += 1
177 end = s.find(";", start)
178 ind, diff = start, 0
179 while end > 0:
180 diff += s.count('"', ind, end) - s.count('\\"', ind, end)
181 if diff % 2 == 0:
182 break
183 end, ind = ind, s.find(";", end + 1)
184 if end < 0:
185 end = len(s)
186 i = s.find("=", start, end)
187 if i == -1:
188 f = s[start:end]
189 else:
190 f = s[start:i].rstrip().lower() + "=" + s[i + 1 : end].lstrip()
191 plist.append(f.strip())
192 start = end
193 return plist
196def parse_options_header(value: str | bytes | None) -> tuple[bytes, dict[bytes, bytes]]:
197 """Parses a Content-Type header into a value in the following format: (content_type, {parameters})."""
198 if not value:
199 return (b"", {})
201 # If we are passed bytes, we assume that it conforms to WSGI, encoding in latin-1.
202 if isinstance(value, bytes): # pragma: no cover
203 value = value.decode("latin-1")
205 # For types
206 assert isinstance(value, str), "Value should be a string by now"
208 # If we have no options, return the string as-is.
209 if ";" not in value:
210 return (value.lower().strip().encode("latin-1"), {})
212 ctype, *segments = _parseparam(value)
213 options: dict[bytes, bytes] = {}
214 for segment in segments:
215 key, _, val = segment.partition("=")
216 # [RFC 7578 §4.2](https://datatracker.ietf.org/doc/html/rfc7578#section-4.2)
217 # forbids the RFC 5987/2231 extended syntax (`key*=`, `key*0`, ...) in
218 # multipart/form-data, so we ignore those parameters and keep the plain
219 # `key` authoritative.
220 if "*" in key:
221 continue
222 if len(val) >= 2 and val[0] == '"' and val[-1] == '"':
223 val = val[1:-1].replace("\\\\", "\\").replace('\\"', '"')
224 # Work around an IE6 bug where the full file path is sent instead of
225 # just the filename.
226 if key == "filename" and (val[1:3] == ":\\" or val[:2] == "\\\\"):
227 val = val.split("\\")[-1]
228 options[key.encode("latin-1")] = val.encode("latin-1")
229 return ctype.encode("latin-1"), options
232class Field:
233 """A Field object represents a (parsed) form field. It represents a single
234 field with a corresponding name and value.
236 The name that a :class:`Field` will be instantiated with is the same name
237 that would be found in the following HTML::
239 <input name="name_goes_here" type="text"/>
241 This class defines two methods, :meth:`on_data` and :meth:`on_end`, that
242 will be called when data is written to the Field, and when the Field is
243 finalized, respectively.
245 Args:
246 name: The name of the form field.
247 content_type: The value of the Content-Type header for this field.
248 """
250 def __init__(self, name: bytes | None, *, content_type: str | None = None) -> None:
251 self._name = name
252 self._value: list[bytes] = []
253 self._content_type = content_type
255 # We cache the joined version of _value for speed.
256 self._cache = _missing
258 @classmethod
259 def from_value(cls, name: bytes, value: bytes | None) -> Field:
260 """Create an instance of a :class:`Field`, and set the corresponding
261 value - either None or an actual value. This method will also
262 finalize the Field itself.
264 Args:
265 name: the name of the form field.
266 value: the value of the form field - either a bytestring or None.
268 Returns:
269 A new instance of a [`Field`][python_multipart.Field].
270 """
272 f = cls(name)
273 if value is None:
274 f.set_none()
275 else:
276 f.write(value)
277 f.finalize()
278 return f
280 def write(self, data: bytes) -> int:
281 """Write some data into the form field.
283 Args:
284 data: The data to write to the field.
286 Returns:
287 The number of bytes written.
288 """
289 return self.on_data(data)
291 def on_data(self, data: bytes) -> int:
292 """This method is a callback that will be called whenever data is
293 written to the Field.
295 Args:
296 data: The data to write to the field.
298 Returns:
299 The number of bytes written.
300 """
301 self._value.append(data)
302 self._cache = _missing
303 return len(data)
305 def on_end(self) -> None:
306 """This method is called whenever the Field is finalized."""
307 if self._cache is _missing:
308 self._cache = b"".join(self._value)
310 def finalize(self) -> None:
311 """Finalize the form field."""
312 self.on_end()
314 def close(self) -> None:
315 """Close the Field object. This will free any underlying cache."""
316 # Free our value array.
317 if self._cache is _missing:
318 self._cache = b"".join(self._value)
320 del self._value
322 def set_none(self) -> None:
323 """Some fields in a querystring can possibly have a value of None - for
324 example, the string "foo&bar=&baz=asdf" will have a field with the
325 name "foo" and value None, one with name "bar" and value "", and one
326 with name "baz" and value "asdf". Since the write() interface doesn't
327 support writing None, this function will set the field value to None.
328 """
329 self._cache = None
331 @property
332 def field_name(self) -> bytes | None:
333 """This property returns the name of the field."""
334 return self._name
336 @property
337 def value(self) -> bytes | None:
338 """This property returns the value of the form field."""
339 if self._cache is _missing:
340 self._cache = b"".join(self._value)
342 assert isinstance(self._cache, bytes) or self._cache is None
343 return self._cache
345 @property
346 def content_type(self) -> str | None:
347 """This property returns the content_type value of the field."""
348 return self._content_type
350 def __eq__(self, other: object) -> bool:
351 if isinstance(other, Field):
352 return self.field_name == other.field_name and self.value == other.value
353 else:
354 return NotImplemented
356 def __repr__(self) -> str:
357 if self.value is not None and len(self.value) > 97:
358 # We get the repr, and then insert three dots before the final
359 # quote.
360 v = repr(self.value[:97])[:-1] + "...'"
361 else:
362 v = repr(self.value)
364 return f"{self.__class__.__name__}(field_name={self.field_name!r}, value={v})"
367class File:
368 """This class represents an uploaded file. It handles writing file data to
369 either an in-memory file or a temporary file on-disk, if the optional
370 threshold is passed.
372 There are some options that can be passed to the File to change behavior
373 of the class. Valid options are as follows:
375 | Name | Type | Default | Description |
376 |-----------------------|-------|---------|-------------|
377 | UPLOAD_DIR | `str` | None | The directory to store uploaded files in. If this is None, a temporary file will be created in the system's standard location. |
378 | UPLOAD_DELETE_TMP | `bool`| True | Delete automatically created TMP file |
379 | UPLOAD_KEEP_FILENAME | `bool`| False | Whether or not to keep the filename of the uploaded file. If True, then the filename is reduced to its basename (directory components are stripped) and the file is saved with that name. Otherwise, a temporary name will be used. |
380 | UPLOAD_KEEP_EXTENSIONS| `bool`| False | Whether or not to keep the uploaded file's extension. If False, the file will be saved with the default temporary extension (usually ".tmp"). Otherwise, the file's extension will be maintained. Note that this will properly combine with the UPLOAD_KEEP_FILENAME setting. |
381 | MAX_MEMORY_FILE_SIZE | `int` | 1 MiB | The maximum number of bytes of a File to keep in memory. By default, the contents of a File are kept into memory until a certain limit is reached, after which the contents of the File are written to a temporary file. This behavior can be disabled by setting this value to an appropriately large value (or, for example, infinity, such as `float('inf')`. |
383 Args:
384 file_name: The name of the file that this [`File`][python_multipart.File] represents.
385 field_name: The name of the form field that this file was uploaded with. This can be None, if, for example,
386 the file was uploaded with Content-Type application/octet-stream.
387 config: The configuration for this File. See above for valid configuration keys and their corresponding values.
388 content_type: The value of the Content-Type header.
389 """ # noqa: E501
391 def __init__(
392 self,
393 file_name: bytes | None,
394 field_name: bytes | None = None,
395 config: FileConfig = {},
396 *,
397 content_type: str | None = None,
398 ) -> None:
399 # Save configuration, set other variables default.
400 self.logger = logging.getLogger(__name__)
401 self._config = config
402 self._in_memory = True
403 self._bytes_written = 0
404 self._fileobj: BytesIO | BufferedRandom = BytesIO()
406 # Save the provided field/file name and content type.
407 self._field_name = field_name
408 self._file_name = file_name
409 self._content_type = content_type
411 # Our actual file name is None by default, since, depending on our
412 # config, we may not actually use the provided name.
413 self._actual_file_name: bytes | None = None
415 # Split the extension from the filename.
416 if file_name is not None:
417 # Extract just the basename to avoid directory traversal
418 basename = os.path.basename(file_name)
419 base, ext = os.path.splitext(basename)
420 self._file_base = base
421 self._ext = ext
423 @property
424 def field_name(self) -> bytes | None:
425 """The form field associated with this file. May be None if there isn't
426 one, for example when we have an application/octet-stream upload.
427 """
428 return self._field_name
430 @property
431 def file_name(self) -> bytes | None:
432 """The file name given in the upload request."""
433 return self._file_name
435 @property
436 def actual_file_name(self) -> bytes | None:
437 """The file name that this file is saved as. Will be None if it's not
438 currently saved on disk.
439 """
440 return self._actual_file_name
442 @property
443 def file_object(self) -> BytesIO | BufferedRandom:
444 """The file object that we're currently writing to. Note that this
445 will either be an instance of a :class:`io.BytesIO`, or a regular file
446 object.
447 """
448 return self._fileobj
450 @property
451 def size(self) -> int:
452 """The total size of this file, counted as the number of bytes that
453 currently have been written to the file.
454 """
455 return self._bytes_written
457 @property
458 def in_memory(self) -> bool:
459 """A boolean representing whether or not this file object is currently
460 stored in-memory or on-disk.
461 """
462 return self._in_memory
464 @property
465 def content_type(self) -> str | None:
466 """The Content-Type value for this part, if it was set."""
467 return self._content_type
469 def flush_to_disk(self) -> None:
470 """If the file is already on-disk, do nothing. Otherwise, copy from
471 the in-memory buffer to a disk file, and then reassign our internal
472 file object to this new disk file.
474 Note that if you attempt to flush a file that is already on-disk, a
475 warning will be logged to this module's logger.
476 """
477 if not self._in_memory:
478 self.logger.warning("Trying to flush to disk when we're not in memory")
479 return
481 # Go back to the start of our file.
482 self._fileobj.seek(0)
484 # Open a new file.
485 new_file = self._get_disk_file()
487 # Copy the file objects.
488 shutil.copyfileobj(self._fileobj, new_file)
490 # Seek to the new position in our new file.
491 new_file.seek(self._bytes_written)
493 # Reassign the fileobject.
494 old_fileobj = self._fileobj
495 self._fileobj = new_file
497 # We're no longer in memory.
498 self._in_memory = False
500 # Close the old file object.
501 old_fileobj.close()
503 def _get_disk_file(self) -> BufferedRandom:
504 """This function is responsible for getting a file object on-disk for us."""
505 self.logger.info("Opening a file on disk")
507 file_dir = self._config.get("UPLOAD_DIR")
508 keep_filename = self._config.get("UPLOAD_KEEP_FILENAME", False)
509 keep_extensions = self._config.get("UPLOAD_KEEP_EXTENSIONS", False)
510 delete_tmp = self._config.get("UPLOAD_DELETE_TMP", True)
511 tmp_file: None | BufferedRandom = None
513 # If we have a directory and are to keep the filename...
514 if file_dir is not None and keep_filename:
515 self.logger.info("Saving with filename in: %r", file_dir)
517 # Build our filename.
518 # TODO: what happens if we don't have a filename?
519 fname = self._file_base + self._ext if keep_extensions else self._file_base
521 path = os.path.join(file_dir, fname) # type: ignore[arg-type]
522 try:
523 self.logger.info("Opening file: %r", path)
524 tmp_file = open(path, "w+b")
525 except OSError:
526 tmp_file = None
528 self.logger.exception("Error opening temporary file")
529 raise FileError("Error opening temporary file: %r" % path)
530 else:
531 # Build options array.
532 # Note that on Python 3, tempfile doesn't support byte names. We
533 # encode our paths using the default filesystem encoding.
534 suffix = self._ext.decode(sys.getfilesystemencoding()) if keep_extensions else None
536 if file_dir is None:
537 dir = None
538 elif isinstance(file_dir, bytes):
539 dir = file_dir.decode(sys.getfilesystemencoding())
540 else:
541 dir = file_dir # pragma: no cover
543 # Create a temporary (named) file with the appropriate settings.
544 self.logger.info(
545 "Creating a temporary file with options: %r", {"suffix": suffix, "delete": delete_tmp, "dir": dir}
546 )
547 try:
548 tmp_file = cast(BufferedRandom, tempfile.NamedTemporaryFile(suffix=suffix, delete=delete_tmp, dir=dir))
549 except OSError:
550 self.logger.exception("Error creating named temporary file")
551 raise FileError("Error creating named temporary file")
553 assert tmp_file is not None
554 # Encode filename as bytes.
555 if isinstance(tmp_file.name, str):
556 fname = tmp_file.name.encode(sys.getfilesystemencoding())
557 else:
558 fname = cast(bytes, tmp_file.name) # pragma: no cover
560 self._actual_file_name = fname
561 return tmp_file
563 def write(self, data: bytes) -> int:
564 """Write some data to the File.
566 :param data: a bytestring
567 """
568 return self.on_data(data)
570 def on_data(self, data: bytes) -> int:
571 """This method is a callback that will be called whenever data is
572 written to the File.
574 Args:
575 data: The data to write to the file.
577 Returns:
578 The number of bytes written.
579 """
580 bwritten = self._fileobj.write(data)
582 # If the bytes written isn't the same as the length, just return.
583 if bwritten != len(data):
584 self.logger.warning("bwritten != len(data) (%d != %d)", bwritten, len(data))
585 return bwritten
587 # Keep track of how many bytes we've written.
588 self._bytes_written += bwritten
590 # If we're in-memory and are over our limit, we create a file.
591 max_memory_file_size = self._config.get("MAX_MEMORY_FILE_SIZE")
592 if self._in_memory and max_memory_file_size is not None and (self._bytes_written > max_memory_file_size):
593 self.logger.info("Flushing to disk")
594 self.flush_to_disk()
596 # Return the number of bytes written.
597 return bwritten
599 def on_end(self) -> None:
600 """This method is called whenever the Field is finalized."""
601 # Flush the underlying file object
602 self._fileobj.flush()
604 def finalize(self) -> None:
605 """Finalize the form file. This will not close the underlying file,
606 but simply signal that we are finished writing to the File.
607 """
608 self.on_end()
610 def close(self) -> None:
611 """Close the File object. This will actually close the underlying
612 file object (whether it's a :class:`io.BytesIO` or an actual file
613 object).
614 """
615 self._fileobj.close()
617 def __repr__(self) -> str:
618 return f"{self.__class__.__name__}(file_name={self.file_name!r}, field_name={self.field_name!r})"
621class BaseParser:
622 """This class is the base class for all parsers. It contains the logic for
623 calling and adding callbacks.
625 A callback can be one of two different forms. "Notification callbacks" are
626 callbacks that are called when something happens - for example, when a new
627 part of a multipart message is encountered by the parser. "Data callbacks"
628 are called when we get some sort of data - for example, part of the body of
629 a multipart chunk. Notification callbacks are called with no parameters,
630 whereas data callbacks are called with three, as follows::
632 data_callback(data, start, end)
634 The "data" parameter is a bytestring (i.e. "foo" on Python 2, or b"foo" on
635 Python 3). "start" and "end" are integer indexes into the "data" string
636 that represent the data of interest. Thus, in a data callback, the slice
637 `data[start:end]` represents the data that the callback is "interested in".
638 The callback is not passed a copy of the data, since copying severely hurts
639 performance.
640 """
642 def __init__(self) -> None:
643 self.logger = logging.getLogger(__name__)
644 self.callbacks: QuerystringCallbacks | OctetStreamCallbacks | MultipartCallbacks = {}
646 def callback(
647 self, name: CallbackName, data: bytes | None = None, start: int | None = None, end: int | None = None
648 ) -> None:
649 """This function calls a provided callback with some data. If the
650 callback is not set, will do nothing.
652 Args:
653 name: The name of the callback to call (as a string).
654 data: Data to pass to the callback. If None, then it is assumed that the callback is a notification
655 callback, and no parameters are given.
656 end: An integer that is passed to the data callback.
657 start: An integer that is passed to the data callback.
658 """
659 func = self.callbacks.get("on_" + name)
660 if func is None:
661 return
662 func = cast("Callable[..., Any]", func)
663 # Depending on whether we're given a buffer...
664 if data is not None:
665 # Don't do anything if we have start == end.
666 if start is not None and start == end:
667 return
668 func(data, start, end)
669 else:
670 func()
672 def set_callback(self, name: CallbackName, new_func: Callable[..., Any] | None) -> None:
673 """Update the function for a callback. Removes from the callbacks dict
674 if new_func is None.
676 :param name: The name of the callback to call (as a string).
678 :param new_func: The new function for the callback. If None, then the
679 callback will be removed (with no error if it does not
680 exist).
681 """
682 if new_func is None:
683 self.callbacks.pop("on_" + name, None) # type: ignore[misc]
684 else:
685 self.callbacks["on_" + name] = new_func # type: ignore[literal-required]
687 def close(self) -> None:
688 pass # pragma: no cover
690 def finalize(self) -> None:
691 pass # pragma: no cover
693 def __repr__(self) -> str:
694 return "%s()" % self.__class__.__name__
697class OctetStreamParser(BaseParser):
698 """This parser parses an octet-stream request body and calls callbacks when
699 incoming data is received. Callbacks are as follows:
701 | Callback Name | Parameters | Description |
702 |----------------|-----------------|-----------------------------------------------------|
703 | on_start | None | Called when the first data is parsed. |
704 | on_data | data, start, end| Called for each data chunk that is parsed. |
705 | on_end | None | Called when the parser is finished parsing all data.|
707 Args:
708 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
709 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
710 """
712 def __init__(self, callbacks: OctetStreamCallbacks = {}, max_size: float = float("inf")):
713 super().__init__()
714 self.callbacks = callbacks
715 self._started = False
717 if not isinstance(max_size, Number) or max_size < 1:
718 raise ValueError("max_size must be a positive number, not %r" % max_size)
719 self.max_size: int | float = max_size
720 self._current_size = 0
722 def write(self, data: bytes) -> int:
723 """Write some data to the parser, which will perform size verification,
724 and then pass the data to the underlying callback.
726 Args:
727 data: The data to write to the parser.
729 Returns:
730 The number of bytes written.
731 """
732 if not self._started:
733 self.callback("start")
734 self._started = True
736 # Truncate data length.
737 data_len = len(data)
738 if (self._current_size + data_len) > self.max_size:
739 # We truncate the length of data that we are to process.
740 new_size = int(self.max_size - self._current_size)
741 self.logger.warning(
742 "Current size is %d (max %d), so truncating data length from %d to %d",
743 self._current_size,
744 self.max_size,
745 data_len,
746 new_size,
747 )
748 data_len = new_size
750 # Increment size, then callback, in case there's an exception.
751 self._current_size += data_len
752 self.callback("data", data, 0, data_len)
753 return data_len
755 def finalize(self) -> None:
756 """Finalize this parser, which signals to that we are finished parsing,
757 and sends the on_end callback.
758 """
759 self.callback("end")
761 def __repr__(self) -> str:
762 return "%s()" % self.__class__.__name__
765class QuerystringParser(BaseParser):
766 """This is a streaming querystring parser. It will consume data, and call
767 the callbacks given when it has data.
769 | Callback Name | Parameters | Description |
770 |----------------|-----------------|-----------------------------------------------------|
771 | on_field_start | None | Called when a new field is encountered. |
772 | on_field_name | data, start, end| Called when a portion of a field's name is encountered. |
773 | on_field_data | data, start, end| Called when a portion of a field's data is encountered. |
774 | on_field_end | None | Called when the end of a field is encountered. |
775 | on_end | None | Called when the parser is finished parsing all data.|
777 Args:
778 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
779 strict_parsing: Whether or not to parse the body strictly. Defaults to False. If this is set to True, then the
780 behavior of the parser changes as the following: if a field has a value with an equal sign
781 (e.g. "foo=bar", or "foo="), it is always included. If a field has no equals sign (e.g. "...&name&..."),
782 it will be treated as an error if 'strict_parsing' is True, otherwise included. If an error is encountered,
783 then a [`QuerystringParseError`][python_multipart.exceptions.QuerystringParseError] will be raised.
784 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
785 """ # noqa: E501
787 state: QuerystringState
789 def __init__(
790 self, callbacks: QuerystringCallbacks = {}, strict_parsing: bool = False, max_size: float = float("inf")
791 ) -> None:
792 super().__init__()
793 self.state = QuerystringState.BEFORE_FIELD
794 self._found_sep = False
796 self.callbacks = callbacks
798 # Max-size stuff
799 if not isinstance(max_size, Number) or max_size < 1:
800 raise ValueError("max_size must be a positive number, not %r" % max_size)
801 self.max_size: int | float = max_size
802 self._current_size = 0
804 # Should parsing be strict?
805 self.strict_parsing = strict_parsing
807 def write(self, data: bytes) -> int:
808 """Write some data to the parser, which will perform size verification,
809 parse into either a field name or value, and then pass the
810 corresponding data to the underlying callback. If an error is
811 encountered while parsing, a QuerystringParseError will be raised. The
812 "offset" attribute of the raised exception will be set to the offset in
813 the input data chunk (NOT the overall stream) that caused the error.
815 Args:
816 data: The data to write to the parser.
818 Returns:
819 The number of bytes written.
820 """
821 # Handle sizing.
822 data_len = len(data)
823 if (self._current_size + data_len) > self.max_size:
824 # We truncate the length of data that we are to process.
825 new_size = int(self.max_size - self._current_size)
826 self.logger.warning(
827 "Current size is %d (max %d), so truncating data length from %d to %d",
828 self._current_size,
829 self.max_size,
830 data_len,
831 new_size,
832 )
833 data_len = new_size
835 l = 0
836 try:
837 l = self._internal_write(data, data_len)
838 finally:
839 self._current_size += l
841 return l
843 def _internal_write(self, data: bytes, length: int) -> int:
844 state = self.state
845 strict_parsing = self.strict_parsing
846 found_sep = self._found_sep
847 callbacks = cast("QuerystringCallbacks", self.callbacks)
848 on_field_start = callbacks.get("on_field_start")
849 on_field_name = callbacks.get("on_field_name")
850 on_field_data = callbacks.get("on_field_data")
851 on_field_end = callbacks.get("on_field_end")
852 if on_field_start is None:
853 on_field_start = _noop_event
854 if on_field_name is None:
855 on_field_name = _noop_data
856 if on_field_data is None:
857 on_field_data = _noop_data
858 if on_field_end is None:
859 on_field_end = _noop_event
861 i = 0
862 while i < length:
863 ch = data[i]
865 # Depending on our state...
866 if state == QuerystringState.BEFORE_FIELD:
867 # If the 'found_sep' flag is set, we've already encountered
868 # and skipped a single separator. If so, we check our strict
869 # parsing flag and decide what to do. Otherwise, we haven't
870 # yet reached a separator, and thus, if we do, we need to skip
871 # it as it will be the boundary between fields that's supposed
872 # to be there.
873 if ch == AMPERSAND:
874 if found_sep:
875 # If we're parsing strictly, we disallow blank chunks.
876 if strict_parsing:
877 raise QuerystringParseError("Skipping duplicate ampersand at %d" % i, offset=i)
878 else:
879 self.logger.debug("Skipping duplicate ampersand at %d", i)
880 else:
881 # This case is when we're skipping the (first)
882 # separator between fields, so we just set our flag
883 # and continue on.
884 found_sep = True
885 else:
886 # Emit a field-start event, and go to that state. Also,
887 # reset the "found_sep" flag, for the next time we get to
888 # this state.
889 on_field_start()
890 i -= 1
891 state = QuerystringState.FIELD_NAME
892 found_sep = False
894 elif state == QuerystringState.FIELD_NAME:
895 # Try and find a separator - we ensure that, if we do, we only
896 # look for the equal sign before it.
897 sep_pos = data.find(b"&", i, length)
899 # See if we can find an equals sign in the remaining data. If
900 # so, we can immediately emit the field name and jump to the
901 # data state.
902 if sep_pos != -1:
903 equals_pos = data.find(b"=", i, sep_pos)
904 else:
905 equals_pos = data.find(b"=", i, length)
907 if equals_pos != -1:
908 # Emit this name.
909 if i != equals_pos:
910 on_field_name(data, i, equals_pos)
912 # Jump i to this position. Note that it will then have 1
913 # added to it below, which means the next iteration of this
914 # loop will inspect the character after the equals sign.
915 i = equals_pos
916 state = QuerystringState.FIELD_DATA
917 else:
918 # No equals sign found.
919 if not strict_parsing:
920 # See also comments in the QuerystringState.FIELD_DATA case below.
921 # If we found the separator, we emit the name and just
922 # end - there's no data callback at all (not even with
923 # a blank value).
924 if sep_pos != -1:
925 if i != sep_pos:
926 on_field_name(data, i, sep_pos)
927 on_field_end()
929 i = sep_pos - 1
930 state = QuerystringState.BEFORE_FIELD
931 else:
932 # Otherwise, no separator in this block, so the
933 # rest of this chunk must be a name.
934 if i != length:
935 on_field_name(data, i, length)
936 i = length
938 else:
939 # We're parsing strictly. If we find a separator,
940 # this is an error - we require an equals sign.
941 if sep_pos != -1:
942 raise QuerystringParseError(
943 "When strict_parsing is True, we require an "
944 "equals sign in all field chunks. Did not "
945 "find one in the chunk that starts at %d" % (i,),
946 offset=i,
947 )
949 # No separator in the rest of this chunk, so it's just
950 # a field name.
951 if i != length:
952 on_field_name(data, i, length)
953 i = length
955 elif state == QuerystringState.FIELD_DATA:
956 # Try finding an ampersand after this position.
957 sep_pos = data.find(b"&", i, length)
959 # If we found it, callback this bit as data and then go back
960 # to expecting to find a field.
961 if sep_pos != -1:
962 if i != sep_pos:
963 on_field_data(data, i, sep_pos)
964 on_field_end()
966 # Note that we go to the separator, which brings us to the
967 # "before field" state. This allows us to properly emit
968 # "field_start" events only when we actually have data for
969 # a field of some sort.
970 i = sep_pos - 1
971 state = QuerystringState.BEFORE_FIELD
973 # Otherwise, emit the rest as data and finish.
974 else:
975 if i != length:
976 on_field_data(data, i, length)
977 i = length
979 else: # pragma: no cover (error case)
980 msg = "Reached an unknown state %d at %d" % (state, i)
981 self.logger.warning(msg)
982 raise QuerystringParseError(msg, offset=i)
984 i += 1
986 self.state = state
987 self._found_sep = found_sep
988 return length
990 def finalize(self) -> None:
991 """Finalize this parser, which signals to that we are finished parsing,
992 if we're still in the middle of a field, an on_field_end callback, and
993 then the on_end callback.
994 """
995 callbacks = cast("QuerystringCallbacks", self.callbacks)
996 # If we're currently in the middle of a field, we finish it.
997 if self.state in (QuerystringState.FIELD_DATA, QuerystringState.FIELD_NAME):
998 on_field_end = callbacks.get("on_field_end")
999 if on_field_end is None:
1000 on_field_end = _noop_event
1001 on_field_end()
1002 on_end = callbacks.get("on_end")
1003 if on_end is None:
1004 on_end = _noop_event
1005 on_end()
1007 def __repr__(self) -> str:
1008 return "{}(strict_parsing={!r}, max_size={!r})".format(
1009 self.__class__.__name__, self.strict_parsing, self.max_size
1010 )
1013class MultipartParser(BaseParser):
1014 """This class is a streaming multipart/form-data parser.
1016 | Callback Name | Parameters | Description |
1017 |--------------------|-----------------|-------------|
1018 | on_part_begin | None | Called when a new part of the multipart message is encountered. |
1019 | on_part_data | data, start, end| Called when a portion of a part's data is encountered. |
1020 | on_part_end | None | Called when the end of a part is reached. |
1021 | on_header_begin | None | Called when we've found a new header in a part of a multipart message |
1022 | on_header_field | data, start, end| Called each time an additional portion of a header is read (i.e. the part of the header that is before the colon; the "Foo" in "Foo: Bar"). |
1023 | on_header_value | data, start, end| Called when we get data for a header. |
1024 | on_header_end | None | Called when the current header is finished - i.e. we've reached the newline at the end of the header. |
1025 | on_headers_finished| None | Called when all headers are finished, and before the part data starts. |
1026 | on_end | None | Called when the parser is finished parsing all data. |
1028 Args:
1029 boundary: The multipart boundary. This is required, and must match what is given in the HTTP request - usually in the Content-Type header.
1030 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
1031 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
1032 max_header_count: The maximum number of headers allowed per part.
1033 max_header_size: The maximum size of a single header line (excluding the trailing CRLF).
1034 """ # noqa: E501
1036 def __init__(
1037 self,
1038 boundary: bytes | str,
1039 callbacks: MultipartCallbacks = {},
1040 max_size: float = float("inf"),
1041 *,
1042 max_header_count: int = DEFAULT_MAX_HEADER_COUNT,
1043 max_header_size: int = DEFAULT_MAX_HEADER_SIZE,
1044 ) -> None:
1045 # Initialize parser state.
1046 super().__init__()
1047 self.state = MultipartState.START
1048 self.index = self.flags = 0
1050 self.callbacks = callbacks
1052 if not isinstance(max_size, Number) or max_size < 1:
1053 raise ValueError("max_size must be a positive number, not %r" % max_size)
1054 self.max_size = max_size
1055 self._current_size = 0
1057 self.max_header_count = max_header_count
1058 self._current_header_count = 0
1060 self.max_header_size = max_header_size
1061 self._current_header_size = 0
1063 # Setup marks. These are used to track the state of data received.
1064 self.marks: dict[str, int] = {}
1066 # Save our boundary.
1067 if isinstance(boundary, str): # pragma: no cover
1068 boundary = boundary.encode("latin-1")
1069 if len(boundary) > MAX_BOUNDARY_LENGTH:
1070 raise FormParserError(f"Boundary length {len(boundary)} exceeds maximum of {MAX_BOUNDARY_LENGTH}")
1071 self.boundary = b"\r\n--" + boundary
1073 def write(self, data: bytes) -> int:
1074 """Write some data to the parser, which will perform size verification,
1075 and then parse the data into the appropriate location (e.g. header,
1076 data, etc.), and pass this on to the underlying callback. If an error
1077 is encountered, a MultipartParseError will be raised. The "offset"
1078 attribute on the raised exception will be set to the offset of the byte
1079 in the input chunk that caused the error.
1081 Args:
1082 data: The data to write to the parser.
1084 Returns:
1085 The number of bytes written.
1086 """
1087 # Handle sizing.
1088 data_len = len(data)
1089 if (self._current_size + data_len) > self.max_size:
1090 # We truncate the length of data that we are to process.
1091 new_size = int(self.max_size - self._current_size)
1092 self.logger.warning(
1093 "Current size is %d (max %d), so truncating data length from %d to %d",
1094 self._current_size,
1095 self.max_size,
1096 data_len,
1097 new_size,
1098 )
1099 data_len = new_size
1101 l = 0
1102 try:
1103 l = self._internal_write(data, data_len)
1104 finally:
1105 self._current_size += l
1107 return l
1109 def _internal_write(self, data: bytes, length: int) -> int:
1110 # Get values from locals.
1111 boundary = self.boundary
1112 boundary_length = len(boundary)
1114 # Get our state, flags and index. These are persisted between calls to
1115 # this function.
1116 state = self.state
1117 index = self.index
1118 flags = self.flags
1119 current_header_count = self._current_header_count
1120 current_header_size = self._current_header_size
1122 # Our index defaults to 0.
1123 i = 0
1125 def advance_header_size(amount: int = 1) -> None:
1126 nonlocal current_header_size
1127 current_header_size += amount
1128 if current_header_size > self.max_header_size:
1129 raise MultipartParseError("Maximum header size exceeded", offset=i)
1131 # Set a mark.
1132 def set_mark(name: str) -> None:
1133 self.marks[name] = i
1135 # Remove a mark.
1136 def delete_mark(name: str, reset: bool = False) -> None:
1137 self.marks.pop(name, None)
1139 # Helper function that makes calling a callback with data easier. The
1140 # 'remaining' parameter will callback from the marked value until the
1141 # end of the buffer, and reset the mark, instead of deleting it. This
1142 # is used at the end of the function to call our callbacks with any
1143 # remaining data in this chunk.
1144 def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None:
1145 marked_index = self.marks.get(name)
1146 if marked_index is None:
1147 return
1149 # Otherwise, we call it from the mark to the current byte we're
1150 # processing.
1151 if end_i <= marked_index:
1152 # There is no additional data to send.
1153 pass
1154 elif marked_index >= 0:
1155 # We are emitting data from the local buffer.
1156 self.callback(name, data, marked_index, end_i)
1157 else:
1158 # Some of the data comes from a partial boundary match.
1159 # and requires look-behind.
1160 # We need to use self.flags (and not flags) because we care about
1161 # the state when we entered the loop.
1162 lookbehind_len = -marked_index
1163 if lookbehind_len <= boundary_length:
1164 self.callback(name, boundary, 0, lookbehind_len)
1165 elif self.flags & FLAG_PART_BOUNDARY:
1166 lookback = boundary + b"\r\n"
1167 self.callback(name, lookback, 0, lookbehind_len)
1168 elif self.flags & FLAG_LAST_BOUNDARY:
1169 lookback = boundary + b"--\r\n"
1170 self.callback(name, lookback, 0, lookbehind_len)
1171 else: # pragma: no cover (error case)
1172 self.logger.warning("Look-back buffer error")
1174 if end_i > 0:
1175 self.callback(name, data, 0, end_i)
1176 # If we're getting remaining data, we have got all the data we
1177 # can be certain is not a boundary, leaving only a partial boundary match.
1178 if remaining:
1179 self.marks[name] = end_i - length
1180 else:
1181 self.marks.pop(name, None)
1183 # For each byte...
1184 while i < length:
1185 c = data[i]
1187 if state == MultipartState.START:
1188 # Skip leading newlines
1189 if c == CR or c == LF:
1190 i = data.find(b"-", i)
1191 if i == -1:
1192 # No boundary candidate in this chunk, so ignore the content after the leading CR/LF.
1193 i = length
1194 break
1195 continue
1197 # index is used as in index into our boundary. Set to 0.
1198 index = 0
1200 # Move to the next state, but decrement i so that we re-process
1201 # this character.
1202 state = MultipartState.START_BOUNDARY
1203 i -= 1
1205 elif state == MultipartState.START_BOUNDARY:
1206 # Check to ensure that the last 2 characters in our boundary
1207 # are CRLF.
1208 if index == boundary_length - 2:
1209 if c == HYPHEN:
1210 # Potential empty message.
1211 state = MultipartState.END_BOUNDARY
1212 elif c != CR:
1213 # Error!
1214 msg = "Did not find CR at end of boundary (%d)" % (i,)
1215 self.logger.warning(msg)
1216 raise MultipartParseError(msg, offset=i)
1218 index += 1
1220 elif index == boundary_length - 1:
1221 if c != LF:
1222 msg = "Did not find LF at end of boundary (%d)" % (i,)
1223 self.logger.warning(msg)
1224 raise MultipartParseError(msg, offset=i)
1226 # The index is now used for indexing into our boundary.
1227 index = 0
1229 # Callback for the start of a part.
1230 self.callback("part_begin")
1231 current_header_count = 0
1232 current_header_size = 0
1234 # Move to the next character and state.
1235 state = MultipartState.HEADER_FIELD_START
1237 else:
1238 # Check to ensure our boundary matches
1239 if c != boundary[index + 2]:
1240 msg = "Expected boundary character %r, got %r at index %d" % (boundary[index + 2], c, index + 2)
1241 self.logger.warning(msg)
1242 raise MultipartParseError(msg, offset=i)
1244 # Increment index into boundary and continue.
1245 index += 1
1247 elif state == MultipartState.HEADER_FIELD_START:
1248 # Mark the start of a header field here, reset the index, and
1249 # continue parsing our header field.
1250 index = 0
1252 if c != CR:
1253 current_header_count += 1
1254 if current_header_count > self.max_header_count:
1255 raise MultipartParseError("Maximum header count exceeded", offset=i)
1256 current_header_size = 0
1258 # Set a mark of our header field.
1259 set_mark("header_field")
1261 # Notify that we're starting a header if the next character is
1262 # not a CR; a CR at the beginning of the header will cause us
1263 # to stop parsing headers in the MultipartState.HEADER_FIELD state,
1264 # below.
1265 if c != CR:
1266 self.callback("header_begin")
1268 # Move to parsing header fields.
1269 state = MultipartState.HEADER_FIELD
1270 i -= 1
1272 elif state == MultipartState.HEADER_FIELD:
1273 # If we've reached a CR at the beginning of a header, it means
1274 # that we've reached the second of 2 newlines, and so there are
1275 # no more headers to parse.
1276 if c == CR and index == 0:
1277 delete_mark("header_field")
1278 state = MultipartState.HEADERS_ALMOST_DONE
1279 i += 1
1280 continue
1282 # The field name runs until the colon; jump straight to it and
1283 # validate the whole span at once instead of byte by byte.
1284 colon = data.find(b":", i, length)
1285 end = colon if colon != -1 else length
1287 # Enforce the size limit before slicing and validating, so an oversized header
1288 # name fails fast instead of copying and scanning a potentially huge span.
1289 advance_header_size(end - i if colon == -1 else end - i + 1)
1291 field = data[i:end]
1292 if field.translate(None, TOKEN_CHARS):
1293 bad = next(b for b in field if b not in TOKEN_CHARS_SET)
1294 bad_i = i + field.index(bad)
1295 msg = "Found invalid character %r in header at %d" % (bad, bad_i)
1296 self.logger.warning(msg)
1297 raise MultipartParseError(msg, offset=bad_i)
1299 index += end - i
1300 if colon == -1:
1301 # Field name continues into the next chunk.
1302 i = length
1303 else:
1304 # A 0-length header is an error.
1305 if index == 0:
1306 msg = "Found 0-length header at %d" % (i,)
1307 self.logger.warning(msg)
1308 raise MultipartParseError(msg, offset=i)
1310 # Call our callback with the header field.
1311 i = colon
1312 data_callback("header_field", i)
1314 # Move to parsing the header value.
1315 state = MultipartState.HEADER_VALUE_START
1317 elif state == MultipartState.HEADER_VALUE_START:
1318 # Skip leading spaces.
1319 if c == SPACE:
1320 advance_header_size()
1321 i += 1
1322 continue
1324 # Mark the start of the header value.
1325 set_mark("header_value")
1327 # Move to the header-value state, reprocessing this character.
1328 state = MultipartState.HEADER_VALUE
1329 i -= 1
1331 elif state == MultipartState.HEADER_VALUE:
1332 # The value runs until the terminating CR; jump straight to it
1333 # instead of inspecting every byte.
1334 cr = data.find(b"\r", i, length)
1335 end = cr if cr != -1 else length
1336 advance_header_size(end - i)
1337 if cr != -1:
1338 i = cr
1339 data_callback("header_value", i)
1340 self.callback("header_end")
1341 current_header_size = 0
1342 state = MultipartState.HEADER_VALUE_ALMOST_DONE
1343 else:
1344 i = length
1346 elif state == MultipartState.HEADER_VALUE_ALMOST_DONE:
1347 # The last character should be a LF. If not, it's an error.
1348 if c != LF:
1349 msg = f"Did not find LF character at end of header (found {c!r})"
1350 self.logger.warning(msg)
1351 raise MultipartParseError(msg, offset=i)
1353 # Move back to the start of another header. Note that if that
1354 # state detects ANOTHER newline, it'll trigger the end of our
1355 # headers.
1356 state = MultipartState.HEADER_FIELD_START
1358 elif state == MultipartState.HEADERS_ALMOST_DONE:
1359 # We're almost done our headers. This is reached when we parse
1360 # a CR at the beginning of a header, so our next character
1361 # should be a LF, or it's an error.
1362 if c != LF:
1363 msg = f"Did not find LF at end of headers (found {c!r})"
1364 self.logger.warning(msg)
1365 raise MultipartParseError(msg, offset=i)
1367 self.callback("headers_finished")
1368 state = MultipartState.PART_DATA_START
1370 elif state == MultipartState.PART_DATA_START:
1371 # Mark the start of our part data.
1372 set_mark("part_data")
1374 # Start processing part data, including this character.
1375 state = MultipartState.PART_DATA
1376 i -= 1
1378 elif state == MultipartState.PART_DATA:
1379 # We're processing our part data right now. During this, we
1380 # need to efficiently search for our boundary, since any data
1381 # on any number of lines can be a part of the current data.
1383 # Save the current value of our index. We use this in case we
1384 # find part of a boundary, but it doesn't match fully.
1385 prev_index = index
1387 # If our index is 0, we're starting a new part, so start our
1388 # search.
1389 if index == 0:
1390 # The most common case is likely to be that the whole
1391 # boundary is present in the buffer.
1392 # Calling `find` is much faster than iterating here.
1393 i0 = data.find(boundary, i, length)
1394 if i0 >= 0:
1395 # We matched the whole boundary string.
1396 index = boundary_length - 1
1397 i = i0 + boundary_length - 1
1398 c = data[i]
1399 else:
1400 # No whole boundary, but the tail may hold a partial one
1401 # that completes in the next chunk. Boundary starts with
1402 # CR, which an RFC boundary contains nowhere else, so the
1403 # last CR in the tail is the only candidate prefix start.
1404 k = data.rfind(boundary[:1], max(i, length - boundary_length + 1), length)
1405 if k != -1 and boundary.startswith(data[k:length]):
1406 index = length - k
1407 # Carry the partial via index; the end-of-chunk flush
1408 # emits the data before it and re-marks the lookbehind.
1409 i = length
1410 continue
1412 # Now, we have a couple of cases here. If our index is before
1413 # the end of the boundary...
1414 if index < boundary_length:
1415 # If the character matches...
1416 if boundary[index] == c:
1417 # The current character matches, so continue!
1418 index += 1
1419 else:
1420 index = 0
1422 # Our index is equal to the length of our boundary!
1423 elif index == boundary_length:
1424 # First we increment it.
1425 index += 1
1427 # Now, if we've reached a newline, we need to set this as
1428 # the potential end of our boundary.
1429 if c == CR:
1430 flags |= FLAG_PART_BOUNDARY
1432 # Otherwise, if this is a hyphen, we might be at the last
1433 # of all boundaries.
1434 elif c == HYPHEN:
1435 flags |= FLAG_LAST_BOUNDARY
1437 # Otherwise, we reset our index, since this isn't either a
1438 # newline or a hyphen.
1439 else:
1440 index = 0
1442 # Our index is right after the part boundary, which should be
1443 # a LF.
1444 elif index == boundary_length + 1:
1445 # If we're at a part boundary (i.e. we've seen a CR
1446 # character already)...
1447 if flags & FLAG_PART_BOUNDARY:
1448 # We need a LF character next.
1449 if c == LF:
1450 # Unset the part boundary flag.
1451 flags &= ~FLAG_PART_BOUNDARY
1453 # We have identified a boundary, callback for any data before it.
1454 data_callback("part_data", i - index)
1455 # Callback indicating that we've reached the end of
1456 # a part, and are starting a new one.
1457 self.callback("part_end")
1458 self.callback("part_begin")
1459 current_header_count = 0
1460 current_header_size = 0
1462 # Move to parsing new headers.
1463 index = 0
1464 state = MultipartState.HEADER_FIELD_START
1465 i += 1
1466 continue
1468 # We didn't find an LF character, so no match. Reset
1469 # our index and clear our flag.
1470 index = 0
1471 flags &= ~FLAG_PART_BOUNDARY
1473 # Otherwise, if we're at the last boundary (i.e. we've
1474 # seen a hyphen already)...
1475 elif flags & FLAG_LAST_BOUNDARY:
1476 # We need a second hyphen here.
1477 if c == HYPHEN:
1478 # We have identified a boundary, callback for any data before it.
1479 data_callback("part_data", i - index)
1480 # Callback to end the current part, and then the
1481 # message.
1482 self.callback("part_end")
1483 self.callback("end")
1484 state = MultipartState.END
1485 else:
1486 # No match, so reset index.
1487 index = 0
1489 # Otherwise, our index is 0. If the previous index is not, it
1490 # means we reset something, and we need to take the data we
1491 # thought was part of our boundary and send it along as actual
1492 # data.
1493 if index == 0 and prev_index > 0:
1494 # Overwrite our previous index.
1495 prev_index = 0
1497 # Re-consider the current character, since this could be
1498 # the start of the boundary itself.
1499 i -= 1
1501 elif state == MultipartState.END_BOUNDARY:
1502 if index == boundary_length - 1:
1503 if c != HYPHEN:
1504 msg = "Did not find - at end of boundary (%d)" % (i,)
1505 self.logger.warning(msg)
1506 raise MultipartParseError(msg, offset=i)
1507 index += 1
1508 self.callback("end")
1509 state = MultipartState.END
1511 elif state == MultipartState.END:
1512 # Silently discard any epilogue data (RFC 2046 section 5.1.1 allows a CRLF and optional
1513 # epilogue after the closing boundary). Django and Werkzeug do the same.
1514 i = length
1515 break
1517 else: # pragma: no cover (error case)
1518 # We got into a strange state somehow! Just stop processing.
1519 msg = "Reached an unknown state %d at %d" % (state, i)
1520 self.logger.warning(msg)
1521 raise MultipartParseError(msg, offset=i)
1523 # Move to the next byte.
1524 i += 1
1526 # We call our callbacks with any remaining data. Note that we pass
1527 # the 'remaining' flag, which sets the mark back to 0 instead of
1528 # deleting it, if it's found. This is because, if the mark is found
1529 # at this point, we assume that there's data for one of these things
1530 # that has been parsed, but not yet emitted. And, as such, it implies
1531 # that we haven't yet reached the end of this 'thing'. So, by setting
1532 # the mark to 0, we cause any data callbacks that take place in future
1533 # calls to this function to start from the beginning of that buffer.
1534 data_callback("header_field", length, True)
1535 data_callback("header_value", length, True)
1536 data_callback("part_data", length - index, True)
1538 # Save values to locals.
1539 self.state = state
1540 self.index = index
1541 self.flags = flags
1542 self._current_header_count = current_header_count
1543 self._current_header_size = current_header_size
1545 # Return our data length to indicate no errors, and that we processed
1546 # all of it.
1547 return length
1549 def finalize(self) -> None:
1550 """Finalize this parser, which signals to that we are finished parsing.
1552 Note: It does not currently, but in the future, it will verify that we
1553 are in the final state of the parser (i.e. the end of the multipart
1554 message is well-formed), and, if not, throw an error.
1555 """
1556 # TODO: verify that we're in the state MultipartState.END, otherwise throw an
1557 # error or otherwise state that we're not finished parsing.
1558 pass
1560 def __repr__(self) -> str:
1561 return f"{self.__class__.__name__}(boundary={self.boundary!r})"
1564class FormParser:
1565 """This class is the all-in-one form parser. Given all the information
1566 necessary to parse a form, it will instantiate the correct parser, create
1567 the proper :class:`Field` and :class:`File` classes to store the data that
1568 is parsed, and call the two given callbacks with each field and file as
1569 they become available.
1571 Args:
1572 content_type: The Content-Type of the incoming request. This is used to select the appropriate parser.
1573 on_field: The callback to call when a field has been parsed and is ready for usage. See above for parameters.
1574 on_file: The callback to call when a file has been parsed and is ready for usage. See above for parameters.
1575 on_end: An optional callback to call when all fields and files in a request has been parsed. Can be None.
1576 boundary: If the request is a multipart/form-data request, this should be the boundary of the request, as given
1577 in the Content-Type header, as a bytestring.
1578 file_name: If the request is of type application/octet-stream, then the body of the request will not contain any
1579 information about the uploaded file. In such cases, you can provide the file name of the uploaded file
1580 manually.
1581 config: Configuration to use for this FormParser. The default values are taken from the DEFAULT_CONFIG value,
1582 and then any keys present in this dictionary will overwrite the default values.
1583 """
1585 #: This is the default configuration for our form parser.
1586 #: Note: all file sizes should be in bytes.
1587 DEFAULT_CONFIG: FormParserConfig = {
1588 "MAX_BODY_SIZE": float("inf"),
1589 "MAX_HEADER_COUNT": DEFAULT_MAX_HEADER_COUNT,
1590 "MAX_HEADER_SIZE": DEFAULT_MAX_HEADER_SIZE,
1591 "MAX_MEMORY_FILE_SIZE": 1 * 1024 * 1024,
1592 "UPLOAD_DIR": None,
1593 "UPLOAD_DELETE_TMP": True,
1594 "UPLOAD_KEEP_FILENAME": False,
1595 "UPLOAD_KEEP_EXTENSIONS": False,
1596 # Error on invalid Content-Transfer-Encoding?
1597 "UPLOAD_ERROR_ON_BAD_CTE": False,
1598 }
1600 def __init__(
1601 self,
1602 content_type: str,
1603 on_field: Callable[[Field], None] | None,
1604 on_file: Callable[[File], None] | None,
1605 on_end: Callable[[], None] | None = None,
1606 boundary: bytes | str | None = None,
1607 file_name: bytes | None = None,
1608 config: dict[Any, Any] = {},
1609 ) -> None:
1610 self.logger = logging.getLogger(__name__)
1612 # Save variables.
1613 self.content_type = content_type
1614 self.boundary = boundary
1615 self.bytes_received = 0
1616 self.parser = None
1618 # Save callbacks.
1619 self.on_field = on_field
1620 self.on_file = on_file
1621 self.on_end = on_end
1623 # Set configuration options.
1624 self.config: FormParserConfig = self.DEFAULT_CONFIG.copy()
1625 self.config.update(config) # type: ignore[typeddict-item]
1627 parser: OctetStreamParser | MultipartParser | QuerystringParser | None = None
1629 # Depending on the Content-Type, we instantiate the correct parser.
1630 if content_type == "application/octet-stream":
1631 file: File | None = None
1633 def on_start() -> None:
1634 nonlocal file
1635 file = File(file_name, None, config=self.config)
1637 def on_data(data: bytes, start: int, end: int) -> None:
1638 nonlocal file
1639 assert file is not None
1640 file.write(data[start:end])
1642 def _on_end() -> None:
1643 nonlocal file
1644 assert file is not None
1645 # Finalize the file itself.
1646 file.finalize()
1648 # Call our callback.
1649 if on_file:
1650 on_file(file)
1652 # Call the on-end callback.
1653 if self.on_end is not None:
1654 self.on_end()
1656 # Instantiate an octet-stream parser
1657 parser = OctetStreamParser(
1658 callbacks={"on_start": on_start, "on_data": on_data, "on_end": _on_end},
1659 max_size=self.config["MAX_BODY_SIZE"],
1660 )
1662 elif content_type == "application/x-www-form-urlencoded" or content_type == "application/x-url-encoded":
1663 name_buffer: list[bytes] = []
1665 f: Field | None = None
1667 def on_field_start() -> None:
1668 pass
1670 def on_field_name(data: bytes, start: int, end: int) -> None:
1671 name_buffer.append(data[start:end])
1673 def on_field_data(data: bytes, start: int, end: int) -> None:
1674 nonlocal f
1675 if f is None:
1676 f = Field(b"".join(name_buffer))
1677 del name_buffer[:]
1678 f.write(data[start:end])
1680 def on_field_end() -> None:
1681 nonlocal f
1682 # Finalize and call callback.
1683 if f is None:
1684 # If we get here, it's because there was no field data.
1685 # We create a field, set it to None, and then continue.
1686 f = Field(b"".join(name_buffer))
1687 del name_buffer[:]
1688 f.set_none()
1690 f.finalize()
1691 if on_field:
1692 on_field(f)
1693 f = None
1695 def _on_end() -> None:
1696 if self.on_end is not None:
1697 self.on_end()
1699 # Instantiate parser.
1700 parser = QuerystringParser(
1701 callbacks={
1702 "on_field_start": on_field_start,
1703 "on_field_name": on_field_name,
1704 "on_field_data": on_field_data,
1705 "on_field_end": on_field_end,
1706 "on_end": _on_end,
1707 },
1708 max_size=self.config["MAX_BODY_SIZE"],
1709 )
1711 elif content_type == "multipart/form-data":
1712 if boundary is None:
1713 self.logger.error("No boundary given")
1714 raise FormParserError("No boundary given")
1716 header_name: list[bytes] = []
1717 header_value: list[bytes] = []
1718 headers: dict[bytes, bytes] = {}
1720 f_multi: File | Field | None = None
1721 writer: File | Field | Base64Decoder | QuotedPrintableDecoder | None = None
1722 is_file = False
1724 def on_part_begin() -> None:
1725 # Reset headers in case this isn't the first part.
1726 nonlocal headers
1727 headers = {}
1729 def on_part_data(data: bytes, start: int, end: int) -> None:
1730 nonlocal writer
1731 assert writer is not None
1732 writer.write(data[start:end])
1733 # TODO: check for error here.
1735 def on_part_end() -> None:
1736 nonlocal f_multi, is_file
1737 assert f_multi is not None
1738 f_multi.finalize()
1739 if is_file:
1740 if on_file:
1741 assert isinstance(f_multi, File)
1742 on_file(f_multi)
1743 else:
1744 if on_field:
1745 assert isinstance(f_multi, Field)
1746 on_field(f_multi)
1748 def on_header_field(data: bytes, start: int, end: int) -> None:
1749 header_name.append(data[start:end])
1751 def on_header_value(data: bytes, start: int, end: int) -> None:
1752 header_value.append(data[start:end])
1754 def on_header_end() -> None:
1755 headers[b"".join(header_name).lower()] = b"".join(header_value)
1756 del header_name[:]
1757 del header_value[:]
1759 def on_headers_finished() -> None:
1760 nonlocal is_file, f_multi, writer
1761 # Reset the 'is file' flag.
1762 is_file = False
1764 # Parse the content-disposition header.
1765 content_disp = headers.get(b"content-disposition")
1766 disp, options = parse_options_header(content_disp)
1768 # Get the field and filename.
1769 field_name = options.get(b"name")
1770 file_name = options.get(b"filename")
1771 # RFC 7578 §4.2: each part MUST have a Content-Disposition header with a "name" parameter.
1772 if field_name is None:
1773 raise FormParserError(f'Field name not found in Content-Disposition: "{content_disp!r}"')
1775 # Create the proper class.
1776 content_type_b = headers.get(b"content-type")
1777 content_type = content_type_b.decode("latin-1") if content_type_b is not None else None
1778 if file_name is None:
1779 f_multi = Field(field_name, content_type=content_type)
1780 else:
1781 f_multi = File(file_name, field_name, config=self.config, content_type=content_type)
1782 is_file = True
1784 # Parse the given Content-Transfer-Encoding to determine what
1785 # we need to do with the incoming data.
1786 # TODO: check that we properly handle 8bit / 7bit encoding.
1787 # RFC 2045 section 6.1: Content-Transfer-Encoding values are case-insensitive.
1788 # https://www.rfc-editor.org/rfc/rfc2045#section-6.1
1789 transfer_encoding = headers.get(b"content-transfer-encoding", b"7bit").lower()
1791 if transfer_encoding in (b"binary", b"8bit", b"7bit"):
1792 writer = f_multi
1794 elif transfer_encoding == b"base64":
1795 writer = Base64Decoder(f_multi)
1797 elif transfer_encoding == b"quoted-printable":
1798 writer = QuotedPrintableDecoder(f_multi)
1800 else:
1801 self.logger.warning("Unknown Content-Transfer-Encoding: %r", transfer_encoding)
1802 if self.config["UPLOAD_ERROR_ON_BAD_CTE"]:
1803 raise FormParserError(f'Unknown Content-Transfer-Encoding "{transfer_encoding!r}"')
1804 else:
1805 # If we aren't erroring, then we just treat this as an
1806 # unencoded Content-Transfer-Encoding.
1807 writer = f_multi
1809 def _on_end() -> None:
1810 nonlocal writer
1811 if writer is not None:
1812 writer.finalize()
1813 if self.on_end is not None:
1814 self.on_end()
1816 # Instantiate a multipart parser.
1817 parser = MultipartParser(
1818 boundary,
1819 callbacks={
1820 "on_part_begin": on_part_begin,
1821 "on_part_data": on_part_data,
1822 "on_part_end": on_part_end,
1823 "on_header_field": on_header_field,
1824 "on_header_value": on_header_value,
1825 "on_header_end": on_header_end,
1826 "on_headers_finished": on_headers_finished,
1827 "on_end": _on_end,
1828 },
1829 max_size=self.config["MAX_BODY_SIZE"],
1830 max_header_count=self.config["MAX_HEADER_COUNT"],
1831 max_header_size=self.config["MAX_HEADER_SIZE"],
1832 )
1834 else:
1835 self.logger.warning("Unknown Content-Type: %r", content_type)
1836 raise FormParserError(f"Unknown Content-Type: {content_type}")
1838 self.parser = parser
1840 def write(self, data: bytes) -> int:
1841 """Write some data. The parser will forward this to the appropriate
1842 underlying parser.
1844 Args:
1845 data: The data to write.
1847 Returns:
1848 The number of bytes processed.
1849 """
1850 self.bytes_received += len(data)
1851 # TODO: check the parser's return value for errors?
1852 assert self.parser is not None
1853 return self.parser.write(data)
1855 def finalize(self) -> None:
1856 """Finalize the parser."""
1857 if self.parser is not None and hasattr(self.parser, "finalize"):
1858 self.parser.finalize()
1860 def close(self) -> None:
1861 """Close the parser."""
1862 if self.parser is not None and hasattr(self.parser, "close"):
1863 self.parser.close()
1865 def __repr__(self) -> str:
1866 return f"{self.__class__.__name__}(content_type={self.content_type!r}, parser={self.parser!r})"
1869def create_form_parser(
1870 headers: dict[str, bytes],
1871 on_field: Callable[[Field], None] | None,
1872 on_file: Callable[[File], None] | None,
1873 config: dict[Any, Any] = {},
1874) -> FormParser:
1875 """This function is a helper function to aid in creating a FormParser
1876 instances. Given a dictionary-like headers object, it will determine
1877 the correct information needed, instantiate a FormParser with the
1878 appropriate values and given callbacks, and then return the corresponding
1879 parser.
1881 Args:
1882 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1883 on_field: Callback to call with each parsed field.
1884 on_file: Callback to call with each parsed file.
1885 config: Configuration variables to pass to the FormParser.
1886 """
1887 content_type: str | bytes | None = headers.get("Content-Type")
1888 if content_type is None:
1889 logging.getLogger(__name__).warning("No Content-Type header given")
1890 raise ValueError("No Content-Type header given!")
1892 # Boundaries are optional (the FormParser will raise if one is needed
1893 # but not given).
1894 content_type, params = parse_options_header(content_type)
1895 boundary = params.get(b"boundary")
1897 # We need content_type to be a string, not a bytes object.
1898 content_type = content_type.decode("latin-1")
1900 # Instantiate a form parser.
1901 form_parser = FormParser(content_type, on_field, on_file, boundary=boundary, config=config)
1903 # Return our parser.
1904 return form_parser
1907def parse_form(
1908 headers: dict[str, bytes],
1909 input_stream: SupportsRead,
1910 on_field: Callable[[Field], None] | None,
1911 on_file: Callable[[File], None] | None,
1912 chunk_size: int = 1048576,
1913) -> None:
1914 """This function is useful if you just want to parse a request body,
1915 without too much work. Pass it a dictionary-like object of the request's
1916 headers, and a file-like object for the input stream, along with two
1917 callbacks that will get called whenever a field or file is parsed.
1919 Args:
1920 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1921 input_stream: A file-like object that represents the request body. The read() method must return bytestrings.
1922 on_field: Callback to call with each parsed field.
1923 on_file: Callback to call with each parsed file.
1924 chunk_size: The maximum size to read from the input stream and write to the parser at one time.
1925 Defaults to 1 MiB.
1926 """
1927 if chunk_size < 1:
1928 raise ValueError(f"chunk_size must be a positive number, not {chunk_size!r}")
1930 # Create our form parser.
1931 parser = create_form_parser(headers, on_field, on_file)
1933 # Read chunks of 1MiB and write to the parser, but never read more than
1934 # the given Content-Length, if any.
1935 content_length: int | float | bytes | None = headers.get("Content-Length")
1936 if content_length is not None:
1937 content_length = int(content_length)
1938 if content_length < 0:
1939 raise ValueError("Content-Length must be non-negative")
1940 else:
1941 content_length = float("inf")
1942 bytes_read = 0
1944 while True:
1945 # Read only up to the Content-Length given.
1946 max_readable = int(min(content_length - bytes_read, chunk_size))
1947 buff = input_stream.read(max_readable)
1949 # Write to the parser and update our length.
1950 parser.write(buff)
1951 bytes_read += len(buff)
1953 # If we get a buffer that's smaller than the size requested, or if we
1954 # have read up to our content length, we're done.
1955 if len(buff) != max_readable or bytes_read == content_length:
1956 break
1958 # Tell our parser that we're done writing data.
1959 parser.finalize()