Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import logging
4import os
5import shutil
6import sys
7import tempfile
8from enum import IntEnum, global_enum
9from io import BufferedRandom, BytesIO
10from numbers import Number
11from typing import TYPE_CHECKING, cast
13from .decoders import Base64Decoder, QuotedPrintableDecoder
14from .exceptions import FileError, FormParserError, MultipartParseError, QuerystringParseError
16if TYPE_CHECKING:
17 from collections.abc import Callable
18 from typing import Any, Literal, Protocol, TypeAlias, TypedDict
20 class SupportsRead(Protocol):
21 def read(self, __n: int) -> bytes: ...
23 class QuerystringCallbacks(TypedDict, total=False):
24 on_field_start: Callable[[], None]
25 on_field_name: Callable[[bytes, int, int], None]
26 on_field_data: Callable[[bytes, int, int], None]
27 on_field_end: Callable[[], None]
28 on_end: Callable[[], None]
30 class OctetStreamCallbacks(TypedDict, total=False):
31 on_start: Callable[[], None]
32 on_data: Callable[[bytes, int, int], None]
33 on_end: Callable[[], None]
35 class MultipartCallbacks(TypedDict, total=False):
36 on_part_begin: Callable[[], None]
37 on_part_data: Callable[[bytes, int, int], None]
38 on_part_end: Callable[[], None]
39 on_header_begin: Callable[[], None]
40 on_header_field: Callable[[bytes, int, int], None]
41 on_header_value: Callable[[bytes, int, int], None]
42 on_header_end: Callable[[], None]
43 on_headers_finished: Callable[[], None]
44 on_end: Callable[[], None]
46 class FileConfig(TypedDict, total=False):
47 UPLOAD_DIR: str | bytes | None
48 UPLOAD_DELETE_TMP: bool
49 UPLOAD_KEEP_FILENAME: bool
50 UPLOAD_KEEP_EXTENSIONS: bool
51 MAX_MEMORY_FILE_SIZE: int
53 class FormParserConfig(FileConfig):
54 UPLOAD_ERROR_ON_BAD_CTE: bool
55 MAX_BODY_SIZE: float
56 MAX_FIELDS: int | float
57 MAX_FIELD_SIZE: int | float
58 MAX_HEADER_COUNT: int
59 MAX_HEADER_SIZE: int
61 CallbackName: TypeAlias = Literal[
62 "start",
63 "data",
64 "end",
65 "field_start",
66 "field_name",
67 "field_data",
68 "field_end",
69 "part_begin",
70 "part_data",
71 "part_end",
72 "header_begin",
73 "header_field",
74 "header_value",
75 "header_end",
76 "headers_finished",
77 ]
79# Unique missing object.
80_missing = object()
83def _noop_event() -> None:
84 pass
87def _noop_data(_data: bytes, _start: int, _end: int) -> None:
88 pass
91@global_enum
92class QuerystringState(IntEnum):
93 """Querystring parser states.
95 These are used to keep track of the state of the parser, and are used to determine
96 what to do when new data is encountered.
97 """
99 BEFORE_FIELD = 0
100 FIELD_NAME = 1
101 FIELD_DATA = 2
104@global_enum
105class MultipartState(IntEnum):
106 """Multipart parser states.
108 These are used to keep track of the state of the parser, and are used to determine
109 what to do when new data is encountered.
110 """
112 START = 0
113 START_BOUNDARY = 1
114 HEADER_FIELD_START = 2
115 HEADER_FIELD = 3
116 HEADER_VALUE_START = 4
117 HEADER_VALUE = 5
118 HEADER_VALUE_ALMOST_DONE = 6
119 HEADERS_ALMOST_DONE = 7
120 PART_DATA_START = 8
121 PART_DATA = 9
122 PART_DATA_END = 10
123 END_BOUNDARY = 11
124 END = 12
125 BOUNDARY_PADDING = 13
128if TYPE_CHECKING:
129 BEFORE_FIELD = QuerystringState.BEFORE_FIELD
130 FIELD_NAME = QuerystringState.FIELD_NAME
131 FIELD_DATA = QuerystringState.FIELD_DATA
133 START = MultipartState.START
134 START_BOUNDARY = MultipartState.START_BOUNDARY
135 HEADER_FIELD_START = MultipartState.HEADER_FIELD_START
136 HEADER_FIELD = MultipartState.HEADER_FIELD
137 HEADER_VALUE_START = MultipartState.HEADER_VALUE_START
138 HEADER_VALUE = MultipartState.HEADER_VALUE
139 HEADER_VALUE_ALMOST_DONE = MultipartState.HEADER_VALUE_ALMOST_DONE
140 HEADERS_ALMOST_DONE = MultipartState.HEADERS_ALMOST_DONE
141 PART_DATA_START = MultipartState.PART_DATA_START
142 PART_DATA = MultipartState.PART_DATA
143 PART_DATA_END = MultipartState.PART_DATA_END
144 END_BOUNDARY = MultipartState.END_BOUNDARY
145 END = MultipartState.END
146 BOUNDARY_PADDING = MultipartState.BOUNDARY_PADDING
148# Flags for the multipart parser.
149FLAG_PART_BOUNDARY = 1
150FLAG_LAST_BOUNDARY = 2
152# Get constants. Since iterating over a str on Python 2 gives you a 1-length
153# string, but iterating over a bytes object on Python 3 gives you an integer,
154# we need to save these constants.
155CR = b"\r"[0]
156LF = b"\n"[0]
157COLON = b":"[0]
158SPACE = b" "[0]
159HYPHEN = b"-"[0]
160AMPERSAND = b"&"[0]
161LOWER_A = b"a"[0]
162LOWER_Z = b"z"[0]
163NULL = b"\x00"[0]
165# fmt: off
166# Mask for ASCII characters that can be http tokens.
167# Per RFC7230 - 3.2.6, this is all alpha-numeric characters
168# and these: !#$%&'*+-.^_`|~
169TOKEN_CHARS = (
170 b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
171 b"abcdefghijklmnopqrstuvwxyz"
172 b"0123456789"
173 b"!#$%&'*+-.^_`|~")
174TOKEN_CHARS_SET = frozenset(TOKEN_CHARS)
175# fmt: on
177DEFAULT_MAX_HEADER_COUNT = 8
178"""Default maximum number of headers allowed per multipart part."""
180DEFAULT_MAX_HEADER_SIZE = 4096 + 128
181"""Default maximum size of a single multipart header line, including syntax overhead."""
183MAX_BOUNDARY_LENGTH = 256
184"""Maximum allowed length of a multipart boundary.
186[RFC 2046 §5.1.1](https://datatracker.ietf.org/doc/html/rfc2046#section-5.1.1)
187recommends boundaries be at most 70 bytes. 256 bytes is generous headroom over
188every HTTP client.
189"""
192def _parseparam(s: str) -> list[str]:
193 # Vendored from the standard library's
194 # [`email.message._parseparam`](https://github.com/python/cpython/blob/v3.14.2/Lib/email/message.py#L73-L96)
195 # to split a header into its `;`-separated parts without treating a `;` inside a double-quoted string as a
196 # separator - and without the RFC 2231 decoding that `email.message.Message.get_params` would apply on top.
197 s = ";" + s
198 plist: list[str] = []
199 start = 0
200 while s.find(";", start) == start:
201 start += 1
202 end = s.find(";", start)
203 ind, diff = start, 0
204 while end > 0:
205 diff += s.count('"', ind, end) - s.count('\\"', ind, end)
206 if diff % 2 == 0:
207 break
208 end, ind = ind, s.find(";", end + 1)
209 if end < 0:
210 end = len(s)
211 i = s.find("=", start, end)
212 if i == -1:
213 f = s[start:end]
214 else:
215 f = s[start:i].rstrip(" \t").lower() + "=" + s[i + 1 : end].lstrip(" \t")
216 plist.append(f.strip(" \t"))
217 start = end
218 return plist
221def parse_options_header(value: str | bytes | None) -> tuple[bytes, dict[bytes, bytes]]:
222 """Parses a Content-Type header into a value in the following format: (content_type, {parameters})."""
223 if not value:
224 return (b"", {})
226 # If we are passed bytes, we assume that it conforms to WSGI, encoding in latin-1.
227 if isinstance(value, bytes): # pragma: no cover
228 value = value.decode("latin-1")
230 # For types
231 assert isinstance(value, str), "Value should be a string by now"
233 # If we have no options, return the string as-is.
234 if ";" not in value:
235 return (value.lower().strip(" \t").encode("latin-1"), {})
237 ctype, *segments = _parseparam(value)
238 options: dict[bytes, bytes] = {}
239 for segment in segments:
240 key, _, val = segment.partition("=")
241 # [RFC 7578 §4.2](https://datatracker.ietf.org/doc/html/rfc7578#section-4.2)
242 # forbids the RFC 5987/2231 extended syntax (`key*=`, `key*0`, ...) in
243 # multipart/form-data, so we ignore those parameters and keep the plain
244 # `key` authoritative.
245 if "*" in key:
246 continue
247 if len(val) >= 2 and val[0] == '"' and val[-1] == '"':
248 val = val[1:-1].replace("\\\\", "\\").replace('\\"', '"')
249 # Work around an IE6 bug where the full file path is sent instead of
250 # just the filename.
251 if key == "filename" and (val[1:3] == ":\\" or val[:2] == "\\\\"):
252 val = val.split("\\")[-1]
253 options[key.encode("latin-1")] = val.encode("latin-1")
254 return ctype.lower().encode("latin-1"), options
257class Field:
258 """A Field object represents a (parsed) form field. It represents a single
259 field with a corresponding name and value.
261 The name that a :class:`Field` will be instantiated with is the same name
262 that would be found in the following HTML::
264 <input name="name_goes_here" type="text"/>
266 This class defines two methods, :meth:`on_data` and :meth:`on_end`, that
267 will be called when data is written to the Field, and when the Field is
268 finalized, respectively.
270 Args:
271 name: The name of the form field.
272 content_type: The value of the Content-Type header for this field.
273 """
275 def __init__(self, name: bytes | None, *, content_type: str | None = None) -> None:
276 self._name = name
277 self._value: list[bytes] = []
278 self._content_type = content_type
280 # We cache the joined version of _value for speed.
281 self._cache = _missing
283 @classmethod
284 def from_value(cls, name: bytes, value: bytes | None) -> Field:
285 """Create an instance of a :class:`Field`, and set the corresponding
286 value - either None or an actual value. This method will also
287 finalize the Field itself.
289 Args:
290 name: the name of the form field.
291 value: the value of the form field - either a bytestring or None.
293 Returns:
294 A new instance of a [`Field`][python_multipart.Field].
295 """
297 f = cls(name)
298 if value is None:
299 f.set_none()
300 else:
301 f.write(value)
302 f.finalize()
303 return f
305 def write(self, data: bytes) -> int:
306 """Write some data into the form field.
308 Args:
309 data: The data to write to the field.
311 Returns:
312 The number of bytes written.
313 """
314 return self.on_data(data)
316 def on_data(self, data: bytes) -> int:
317 """This method is a callback that will be called whenever data is
318 written to the Field.
320 Args:
321 data: The data to write to the field.
323 Returns:
324 The number of bytes written.
325 """
326 self._value.append(data)
327 self._cache = _missing
328 return len(data)
330 def on_end(self) -> None:
331 """This method is called whenever the Field is finalized."""
332 if self._cache is _missing:
333 self._cache = b"".join(self._value)
335 def finalize(self) -> None:
336 """Finalize the form field."""
337 self.on_end()
339 def close(self) -> None:
340 """Close the Field object. This will free any underlying cache."""
341 # Free our value array.
342 if self._cache is _missing:
343 self._cache = b"".join(self._value)
345 del self._value
347 def set_none(self) -> None:
348 """Some fields in a querystring can possibly have a value of None - for
349 example, the string "foo&bar=&baz=asdf" will have a field with the
350 name "foo" and value None, one with name "bar" and value "", and one
351 with name "baz" and value "asdf". Since the write() interface doesn't
352 support writing None, this function will set the field value to None.
353 """
354 self._cache = None
356 @property
357 def field_name(self) -> bytes | None:
358 """This property returns the name of the field."""
359 return self._name
361 @property
362 def value(self) -> bytes | None:
363 """This property returns the value of the form field."""
364 if self._cache is _missing:
365 self._cache = b"".join(self._value)
367 assert isinstance(self._cache, bytes) or self._cache is None
368 return self._cache
370 @property
371 def content_type(self) -> str | None:
372 """This property returns the content_type value of the field."""
373 return self._content_type
375 def __eq__(self, other: object) -> bool:
376 if isinstance(other, Field):
377 return self.field_name == other.field_name and self.value == other.value
378 else:
379 return NotImplemented
381 def __repr__(self) -> str:
382 if self.value is not None and len(self.value) > 97:
383 # We get the repr, and then insert three dots before the final
384 # quote.
385 v = repr(self.value[:97])[:-1] + "...'"
386 else:
387 v = repr(self.value)
389 return f"{self.__class__.__name__}(field_name={self.field_name!r}, value={v})"
392class File:
393 """This class represents an uploaded file. It handles writing file data to
394 either an in-memory file or a temporary file on-disk, if the optional
395 threshold is passed.
397 There are some options that can be passed to the File to change behavior
398 of the class. Valid options are as follows:
400 | Name | Type | Default | Description |
401 |-----------------------|-------|---------|-------------|
402 | UPLOAD_DIR | `str` | None | The directory to store uploaded files in. If this is None, a temporary file will be created in the system's standard location. |
403 | UPLOAD_DELETE_TMP | `bool`| True | Delete automatically created TMP file |
404 | UPLOAD_KEEP_FILENAME | `bool`| False | Whether or not to keep the filename of the uploaded file. If True, then the filename is reduced to its basename (directory components are stripped) and the file is saved with that name. Otherwise, a temporary name will be used. |
405 | UPLOAD_KEEP_EXTENSIONS| `bool`| False | Whether or not to keep the uploaded file's extension. If False, the file will be saved with the default temporary extension (usually ".tmp"). Otherwise, the file's extension will be maintained. Note that this will properly combine with the UPLOAD_KEEP_FILENAME setting. |
406 | MAX_MEMORY_FILE_SIZE | `int` | 1 MiB | The maximum number of bytes of a File to keep in memory. By default, the contents of a File are kept into memory until a certain limit is reached, after which the contents of the File are written to a temporary file. This behavior can be disabled by setting this value to an appropriately large value (or, for example, infinity, such as `float('inf')`. |
408 Args:
409 file_name: The name of the file that this [`File`][python_multipart.File] represents.
410 field_name: The name of the form field that this file was uploaded with. This can be None, if, for example,
411 the file was uploaded with Content-Type application/octet-stream.
412 config: The configuration for this File. See above for valid configuration keys and their corresponding values.
413 content_type: The value of the Content-Type header.
414 """ # noqa: E501
416 def __init__(
417 self,
418 file_name: bytes | None,
419 field_name: bytes | None = None,
420 config: FileConfig = {},
421 *,
422 content_type: str | None = None,
423 ) -> None:
424 # Save configuration, set other variables default.
425 self.logger = logging.getLogger(__name__)
426 self._config = config
427 self._in_memory = True
428 self._bytes_written = 0
429 self._fileobj: BytesIO | BufferedRandom = BytesIO()
431 # Save the provided field/file name and content type.
432 self._field_name = field_name
433 self._file_name = file_name
434 self._content_type = content_type
436 # Our actual file name is None by default, since, depending on our
437 # config, we may not actually use the provided name.
438 self._actual_file_name: bytes | None = None
440 # Nameless uploads (e.g. `application/octet-stream`) have no base or extension.
441 self._file_base: bytes | None = None
442 self._ext = b""
443 if file_name is not None:
444 # Extract just the basename to avoid directory traversal
445 basename = os.path.basename(file_name)
446 base, ext = os.path.splitext(basename)
447 self._file_base = base
448 self._ext = ext
450 @property
451 def field_name(self) -> bytes | None:
452 """The form field associated with this file. May be None if there isn't
453 one, for example when we have an application/octet-stream upload.
454 """
455 return self._field_name
457 @property
458 def file_name(self) -> bytes | None:
459 """The file name given in the upload request."""
460 return self._file_name
462 @property
463 def actual_file_name(self) -> bytes | None:
464 """The file name that this file is saved as. Will be None if it's not
465 currently saved on disk.
466 """
467 return self._actual_file_name
469 @property
470 def file_object(self) -> BytesIO | BufferedRandom:
471 """The file object that we're currently writing to. Note that this
472 will either be an instance of a :class:`io.BytesIO`, or a regular file
473 object.
474 """
475 return self._fileobj
477 @property
478 def size(self) -> int:
479 """The total size of this file, counted as the number of bytes that
480 currently have been written to the file.
481 """
482 return self._bytes_written
484 @property
485 def in_memory(self) -> bool:
486 """A boolean representing whether or not this file object is currently
487 stored in-memory or on-disk.
488 """
489 return self._in_memory
491 @property
492 def content_type(self) -> str | None:
493 """The Content-Type value for this part, if it was set."""
494 return self._content_type
496 def flush_to_disk(self) -> None:
497 """If the file is already on-disk, do nothing. Otherwise, copy from
498 the in-memory buffer to a disk file, and then reassign our internal
499 file object to this new disk file.
501 Note that if you attempt to flush a file that is already on-disk, a
502 warning will be logged to this module's logger.
503 """
504 if not self._in_memory:
505 self.logger.warning("Trying to flush to disk when we're not in memory")
506 return
508 # Go back to the start of our file.
509 self._fileobj.seek(0)
511 # Open a new file.
512 new_file = self._get_disk_file()
514 # Copy the file objects.
515 shutil.copyfileobj(self._fileobj, new_file)
517 # Seek to the new position in our new file.
518 new_file.seek(self._bytes_written)
520 # Reassign the fileobject.
521 old_fileobj = self._fileobj
522 self._fileobj = new_file
524 # We're no longer in memory.
525 self._in_memory = False
527 # Close the old file object.
528 old_fileobj.close()
530 def _get_disk_file(self) -> BufferedRandom:
531 """This function is responsible for getting a file object on-disk for us."""
532 self.logger.info("Opening a file on disk")
534 file_dir = self._config.get("UPLOAD_DIR")
535 keep_filename = self._config.get("UPLOAD_KEEP_FILENAME", False)
536 keep_extensions = self._config.get("UPLOAD_KEEP_EXTENSIONS", False)
537 delete_tmp = self._config.get("UPLOAD_DELETE_TMP", True)
538 tmp_file: None | BufferedRandom = None
540 # If we have a directory and a filename to keep...
541 if file_dir is not None and keep_filename and self._file_base is not None:
542 self.logger.info("Saving with filename in: %r", file_dir)
544 # Build our filename.
545 fname = self._file_base + self._ext if keep_extensions else self._file_base
547 path = os.path.join(os.fsencode(file_dir), fname)
548 try:
549 self.logger.info("Opening file: %r", path)
550 tmp_file = open(path, "w+b")
551 except OSError:
552 tmp_file = None
554 self.logger.exception("Error opening temporary file")
555 raise FileError("Error opening temporary file: %r" % path)
556 else:
557 # Build options array.
558 # Note that on Python 3, tempfile doesn't support byte names. We
559 # encode our paths using the default filesystem encoding.
560 suffix = self._ext.decode(sys.getfilesystemencoding()) if keep_extensions else None
562 if file_dir is None:
563 dir = None
564 elif isinstance(file_dir, bytes):
565 dir = file_dir.decode(sys.getfilesystemencoding())
566 else:
567 dir = file_dir # pragma: no cover
569 # Create a temporary (named) file with the appropriate settings.
570 self.logger.info(
571 "Creating a temporary file with options: %r", {"suffix": suffix, "delete": delete_tmp, "dir": dir}
572 )
573 try:
574 tmp_file = cast(BufferedRandom, tempfile.NamedTemporaryFile(suffix=suffix, delete=delete_tmp, dir=dir))
575 except OSError:
576 self.logger.exception("Error creating named temporary file")
577 raise FileError("Error creating named temporary file")
579 assert tmp_file is not None
580 # Encode filename as bytes.
581 if isinstance(tmp_file.name, str):
582 fname = tmp_file.name.encode(sys.getfilesystemencoding())
583 else:
584 fname = cast(bytes, tmp_file.name) # pragma: no cover
586 self._actual_file_name = fname
587 return tmp_file
589 def write(self, data: bytes) -> int:
590 """Write some data to the File.
592 :param data: a bytestring
593 """
594 return self.on_data(data)
596 def on_data(self, data: bytes) -> int:
597 """This method is a callback that will be called whenever data is
598 written to the File.
600 Args:
601 data: The data to write to the file.
603 Returns:
604 The number of bytes written.
605 """
606 bwritten = self._fileobj.write(data)
608 # If the bytes written isn't the same as the length, just return.
609 if bwritten != len(data):
610 self.logger.warning("bwritten != len(data) (%d != %d)", bwritten, len(data))
611 return bwritten
613 # Keep track of how many bytes we've written.
614 self._bytes_written += bwritten
616 # If we're in-memory and are over our limit, we create a file.
617 max_memory_file_size = self._config.get("MAX_MEMORY_FILE_SIZE")
618 if self._in_memory and max_memory_file_size is not None and (self._bytes_written > max_memory_file_size):
619 self.logger.info("Flushing to disk")
620 self.flush_to_disk()
622 # Return the number of bytes written.
623 return bwritten
625 def on_end(self) -> None:
626 """This method is called whenever the Field is finalized."""
627 # Flush the underlying file object
628 self._fileobj.flush()
630 def finalize(self) -> None:
631 """Finalize the form file. This will not close the underlying file,
632 but simply signal that we are finished writing to the File.
633 """
634 self.on_end()
636 def close(self) -> None:
637 """Close the File object. This will actually close the underlying
638 file object (whether it's a :class:`io.BytesIO` or an actual file
639 object).
640 """
641 self._fileobj.close()
643 def __repr__(self) -> str:
644 return f"{self.__class__.__name__}(file_name={self.file_name!r}, field_name={self.field_name!r})"
647class BaseParser:
648 """This class is the base class for all parsers. It contains the logic for
649 calling and adding callbacks.
651 A callback can be one of two different forms. "Notification callbacks" are
652 callbacks that are called when something happens - for example, when a new
653 part of a multipart message is encountered by the parser. "Data callbacks"
654 are called when we get some sort of data - for example, part of the body of
655 a multipart chunk. Notification callbacks are called with no parameters,
656 whereas data callbacks are called with three, as follows::
658 data_callback(data, start, end)
660 The "data" parameter is a bytestring (i.e. "foo" on Python 2, or b"foo" on
661 Python 3). "start" and "end" are integer indexes into the "data" string
662 that represent the data of interest. Thus, in a data callback, the slice
663 `data[start:end]` represents the data that the callback is "interested in".
664 The callback is not passed a copy of the data, since copying severely hurts
665 performance.
666 """
668 def __init__(self) -> None:
669 self.logger = logging.getLogger(__name__)
670 self.callbacks: QuerystringCallbacks | OctetStreamCallbacks | MultipartCallbacks = {}
672 def callback(
673 self, name: CallbackName, data: bytes | None = None, start: int | None = None, end: int | None = None
674 ) -> None:
675 """This function calls a provided callback with some data. If the
676 callback is not set, will do nothing.
678 Args:
679 name: The name of the callback to call (as a string).
680 data: Data to pass to the callback. If None, then it is assumed that the callback is a notification
681 callback, and no parameters are given.
682 end: An integer that is passed to the data callback.
683 start: An integer that is passed to the data callback.
684 """
685 func = self.callbacks.get("on_" + name)
686 if func is None:
687 return
688 func = cast("Callable[..., Any]", func)
689 # Depending on whether we're given a buffer...
690 if data is not None:
691 # Don't do anything if we have start == end.
692 if start is not None and start == end:
693 return
694 func(data, start, end)
695 else:
696 func()
698 def set_callback(self, name: CallbackName, new_func: Callable[..., Any] | None) -> None:
699 """Update the function for a callback. Removes from the callbacks dict
700 if new_func is None.
702 :param name: The name of the callback to call (as a string).
704 :param new_func: The new function for the callback. If None, then the
705 callback will be removed (with no error if it does not
706 exist).
707 """
708 if new_func is None:
709 self.callbacks.pop("on_" + name, None) # type: ignore[misc]
710 else:
711 self.callbacks["on_" + name] = new_func # type: ignore[literal-required]
713 def close(self) -> None:
714 pass # pragma: no cover
716 def finalize(self) -> None:
717 pass # pragma: no cover
719 def __repr__(self) -> str:
720 return "%s()" % self.__class__.__name__
723class OctetStreamParser(BaseParser):
724 """This parser parses an octet-stream request body and calls callbacks when
725 incoming data is received. Callbacks are as follows:
727 | Callback Name | Parameters | Description |
728 |----------------|-----------------|-----------------------------------------------------|
729 | on_start | None | Called when the first data is parsed. |
730 | on_data | data, start, end| Called for each data chunk that is parsed. |
731 | on_end | None | Called when the parser is finished parsing all data.|
733 Args:
734 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
735 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
736 """
738 def __init__(self, callbacks: OctetStreamCallbacks | None = None, max_size: float = float("inf")):
739 super().__init__()
740 self.callbacks = {} if callbacks is None else callbacks
741 self._started = False
743 if not isinstance(max_size, Number) or max_size < 1:
744 raise ValueError("max_size must be a positive number, not %r" % max_size)
745 self.max_size: int | float = max_size
746 self._current_size = 0
748 def write(self, data: bytes) -> int:
749 """Write some data to the parser, which will perform size verification,
750 and then pass the data to the underlying callback.
752 Args:
753 data: The data to write to the parser.
755 Returns:
756 The number of bytes written.
757 """
758 if not self._started:
759 self.callback("start")
760 self._started = True
762 # Truncate data length.
763 data_len = len(data)
764 if (self._current_size + data_len) > self.max_size:
765 # We truncate the length of data that we are to process.
766 new_size = int(self.max_size - self._current_size)
767 self.logger.warning(
768 "Current size is %d (max %d), so truncating data length from %d to %d",
769 self._current_size,
770 self.max_size,
771 data_len,
772 new_size,
773 )
774 data_len = new_size
776 # Increment size, then callback, in case there's an exception.
777 self._current_size += data_len
778 self.callback("data", data, 0, data_len)
779 return data_len
781 def finalize(self) -> None:
782 """Finalize this parser, which signals to that we are finished parsing,
783 and sends the on_end callback.
784 """
785 self.callback("end")
787 def __repr__(self) -> str:
788 return "%s()" % self.__class__.__name__
791class QuerystringParser(BaseParser):
792 """This is a streaming querystring parser. It will consume data, and call
793 the callbacks given when it has data.
795 | Callback Name | Parameters | Description |
796 |----------------|-----------------|-----------------------------------------------------|
797 | on_field_start | None | Called when a new field is encountered. |
798 | on_field_name | data, start, end| Called when a portion of a field's name is encountered. |
799 | on_field_data | data, start, end| Called when a portion of a field's data is encountered. |
800 | on_field_end | None | Called when the end of a field is encountered. |
801 | on_end | None | Called when the parser is finished parsing all data.|
803 Args:
804 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
805 strict_parsing: Whether or not to parse the body strictly. Defaults to False. If this is set to True, then the
806 behavior of the parser changes as the following: if a field has a value with an equal sign
807 (e.g. "foo=bar", or "foo="), it is always included. If a field has no equals sign (e.g. "...&name&..."),
808 it will be treated as an error if 'strict_parsing' is True, otherwise included. If an error is encountered,
809 then a [`QuerystringParseError`][python_multipart.exceptions.QuerystringParseError] will be raised.
810 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
811 """ # noqa: E501
813 state: QuerystringState
815 def __init__(
816 self,
817 callbacks: QuerystringCallbacks | None = None,
818 strict_parsing: bool = False,
819 max_size: float = float("inf"),
820 ) -> None:
821 super().__init__()
822 self.state = BEFORE_FIELD
823 self._found_sep = False
825 self.callbacks = {} if callbacks is None else callbacks
827 # Max-size stuff
828 if not isinstance(max_size, Number) or max_size < 1:
829 raise ValueError("max_size must be a positive number, not %r" % max_size)
830 self.max_size: int | float = max_size
831 self._current_size = 0
833 # Should parsing be strict?
834 self.strict_parsing = strict_parsing
836 def write(self, data: bytes) -> int:
837 """Write some data to the parser, which will perform size verification,
838 parse into either a field name or value, and then pass the
839 corresponding data to the underlying callback. If an error is
840 encountered while parsing, a QuerystringParseError will be raised. The
841 "offset" attribute of the raised exception will be set to the offset in
842 the input data chunk (NOT the overall stream) that caused the error.
844 Args:
845 data: The data to write to the parser.
847 Returns:
848 The number of bytes written.
849 """
850 # Handle sizing.
851 data_len = len(data)
852 if (self._current_size + data_len) > self.max_size:
853 # We truncate the length of data that we are to process.
854 new_size = int(self.max_size - self._current_size)
855 self.logger.warning(
856 "Current size is %d (max %d), so truncating data length from %d to %d",
857 self._current_size,
858 self.max_size,
859 data_len,
860 new_size,
861 )
862 data_len = new_size
864 l = 0
865 try:
866 l = self._internal_write(data, data_len)
867 finally:
868 self._current_size += l
870 return l
872 def _internal_write(self, data: bytes, length: int) -> int:
873 state = self.state
874 strict_parsing = self.strict_parsing
875 found_sep = self._found_sep
876 callbacks = cast("QuerystringCallbacks", self.callbacks)
877 on_field_start = callbacks.get("on_field_start")
878 on_field_name = callbacks.get("on_field_name")
879 on_field_data = callbacks.get("on_field_data")
880 on_field_end = callbacks.get("on_field_end")
881 if on_field_start is None:
882 on_field_start = _noop_event
883 if on_field_name is None:
884 on_field_name = _noop_data
885 if on_field_data is None:
886 on_field_data = _noop_data
887 if on_field_end is None:
888 on_field_end = _noop_event
890 i = 0
891 while i < length:
892 ch = data[i]
894 # Depending on our state...
895 if state == BEFORE_FIELD:
896 # If the 'found_sep' flag is set, we've already encountered
897 # and skipped a single separator. If so, we check our strict
898 # parsing flag and decide what to do. Otherwise, we haven't
899 # yet reached a separator, and thus, if we do, we need to skip
900 # it as it will be the boundary between fields that's supposed
901 # to be there.
902 if ch == AMPERSAND:
903 if found_sep:
904 # If we're parsing strictly, we disallow blank chunks.
905 if strict_parsing:
906 raise QuerystringParseError("Skipping duplicate ampersand at %d" % i, offset=i)
907 else:
908 self.logger.debug("Skipping duplicate ampersand at %d", i)
909 else:
910 # This case is when we're skipping the (first)
911 # separator between fields, so we just set our flag
912 # and continue on.
913 found_sep = True
914 else:
915 # Emit a field-start event, and go to that state. Also,
916 # reset the "found_sep" flag, for the next time we get to
917 # this state.
918 on_field_start()
919 i -= 1
920 state = FIELD_NAME
921 found_sep = False
923 elif state == FIELD_NAME:
924 # Try and find a separator - we ensure that, if we do, we only
925 # look for the equal sign before it.
926 sep_pos = data.find(b"&", i, length)
928 # See if we can find an equals sign in the remaining data. If
929 # so, we can immediately emit the field name and jump to the
930 # data state.
931 if sep_pos != -1:
932 equals_pos = data.find(b"=", i, sep_pos)
933 else:
934 equals_pos = data.find(b"=", i, length)
936 if equals_pos != -1:
937 # Emit this name.
938 if i != equals_pos:
939 on_field_name(data, i, equals_pos)
941 # Jump i to this position. Note that it will then have 1
942 # added to it below, which means the next iteration of this
943 # loop will inspect the character after the equals sign.
944 i = equals_pos
945 state = FIELD_DATA
946 else:
947 # No equals sign found.
948 if not strict_parsing:
949 # See also comments in the QuerystringState.FIELD_DATA case below.
950 # If we found the separator, we emit the name and just
951 # end - there's no data callback at all (not even with
952 # a blank value).
953 if sep_pos != -1:
954 if i != sep_pos:
955 on_field_name(data, i, sep_pos)
956 on_field_end()
958 i = sep_pos - 1
959 state = BEFORE_FIELD
960 else:
961 # Otherwise, no separator in this block, so the
962 # rest of this chunk must be a name.
963 if i != length:
964 on_field_name(data, i, length)
965 i = length
967 else:
968 # We're parsing strictly. If we find a separator,
969 # this is an error - we require an equals sign.
970 if sep_pos != -1:
971 raise QuerystringParseError(
972 "When strict_parsing is True, we require an "
973 "equals sign in all field chunks. Did not "
974 "find one in the chunk that starts at %d" % (i,),
975 offset=i,
976 )
978 # No separator in the rest of this chunk, so it's just
979 # a field name.
980 if i != length:
981 on_field_name(data, i, length)
982 i = length
984 elif state == FIELD_DATA:
985 # Try finding an ampersand after this position.
986 sep_pos = data.find(b"&", i, length)
988 # If we found it, callback this bit as data and then go back
989 # to expecting to find a field.
990 if sep_pos != -1:
991 if i != sep_pos:
992 on_field_data(data, i, sep_pos)
993 on_field_end()
995 # Note that we go to the separator, which brings us to the
996 # "before field" state. This allows us to properly emit
997 # "field_start" events only when we actually have data for
998 # a field of some sort.
999 i = sep_pos - 1
1000 state = BEFORE_FIELD
1002 # Otherwise, emit the rest as data and finish.
1003 else:
1004 if i != length:
1005 on_field_data(data, i, length)
1006 i = length
1008 else: # pragma: no cover (error case)
1009 msg = "Reached an unknown state %d at %d" % (state, i)
1010 self.logger.warning(msg)
1011 raise QuerystringParseError(msg, offset=i)
1013 i += 1
1015 self.state = state
1016 self._found_sep = found_sep
1017 return length
1019 def finalize(self) -> None:
1020 """Finalize this parser, which signals to that we are finished parsing,
1021 if we're still in the middle of a field, an on_field_end callback, and
1022 then the on_end callback.
1023 """
1024 callbacks = cast("QuerystringCallbacks", self.callbacks)
1025 # If we're currently in the middle of a field, we finish it.
1026 if self.state in (FIELD_DATA, FIELD_NAME):
1027 on_field_end = callbacks.get("on_field_end")
1028 if on_field_end is None:
1029 on_field_end = _noop_event
1030 on_field_end()
1031 on_end = callbacks.get("on_end")
1032 if on_end is None:
1033 on_end = _noop_event
1034 on_end()
1036 def __repr__(self) -> str:
1037 return "{}(strict_parsing={!r}, max_size={!r})".format(
1038 self.__class__.__name__, self.strict_parsing, self.max_size
1039 )
1042class MultipartParser(BaseParser):
1043 """This class is a streaming multipart/form-data parser.
1045 | Callback Name | Parameters | Description |
1046 |--------------------|-----------------|-------------|
1047 | on_part_begin | None | Called when a new part of the multipart message is encountered. |
1048 | on_part_data | data, start, end| Called when a portion of a part's data is encountered. |
1049 | on_part_end | None | Called when the end of a part is reached. |
1050 | on_header_begin | None | Called when we've found a new header in a part of a multipart message |
1051 | on_header_field | data, start, end| Called each time an additional portion of a header is read (i.e. the part of the header that is before the colon; the "Foo" in "Foo: Bar"). |
1052 | on_header_value | data, start, end| Called when we get data for a header. |
1053 | on_header_end | None | Called when the current header is finished - i.e. we've reached the newline at the end of the header. |
1054 | on_headers_finished| None | Called when all headers are finished, and before the part data starts. |
1055 | on_end | None | Called when the parser is finished parsing all data. |
1057 Args:
1058 boundary: The multipart boundary. This is required, and must match what is given in the HTTP request - usually in the Content-Type header.
1059 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
1060 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
1061 max_header_count: The maximum number of headers allowed per part.
1062 max_header_size: The maximum size of a single header line (excluding the trailing CRLF).
1063 """ # noqa: E501
1065 def __init__(
1066 self,
1067 boundary: bytes | str,
1068 callbacks: MultipartCallbacks | None = None,
1069 max_size: float = float("inf"),
1070 *,
1071 max_header_count: int = DEFAULT_MAX_HEADER_COUNT,
1072 max_header_size: int = DEFAULT_MAX_HEADER_SIZE,
1073 ) -> None:
1074 # Initialize parser state.
1075 super().__init__()
1076 self.state = START
1077 self.index = self.flags = 0
1079 self.callbacks = {} if callbacks is None else callbacks
1081 if not isinstance(max_size, Number) or max_size < 1:
1082 raise ValueError("max_size must be a positive number, not %r" % max_size)
1083 self.max_size = max_size
1084 self._current_size = 0
1086 self.max_header_count = max_header_count
1087 self._current_header_count = 0
1089 self.max_header_size = max_header_size
1090 self._current_header_size = 0
1092 # Setup marks. These are used to track the state of data received.
1093 self.marks: dict[str, int] = {}
1095 # Save our boundary.
1096 if isinstance(boundary, str): # pragma: no cover
1097 boundary = boundary.encode("latin-1")
1098 if len(boundary) > MAX_BOUNDARY_LENGTH:
1099 raise FormParserError(f"Boundary length {len(boundary)} exceeds maximum of {MAX_BOUNDARY_LENGTH}")
1100 self.boundary = b"\r\n--" + boundary
1102 def write(self, data: bytes) -> int:
1103 """Write some data to the parser, which will perform size verification,
1104 and then parse the data into the appropriate location (e.g. header,
1105 data, etc.), and pass this on to the underlying callback. If an error
1106 is encountered, a MultipartParseError will be raised. The "offset"
1107 attribute on the raised exception will be set to the offset of the byte
1108 in the input chunk that caused the error.
1110 Args:
1111 data: The data to write to the parser.
1113 Returns:
1114 The number of bytes written.
1115 """
1116 # Handle sizing.
1117 data_len = len(data)
1118 if (self._current_size + data_len) > self.max_size:
1119 # We truncate the length of data that we are to process.
1120 new_size = int(self.max_size - self._current_size)
1121 self.logger.warning(
1122 "Current size is %d (max %d), so truncating data length from %d to %d",
1123 self._current_size,
1124 self.max_size,
1125 data_len,
1126 new_size,
1127 )
1128 data_len = new_size
1130 l = 0
1131 try:
1132 l = self._internal_write(data, data_len)
1133 finally:
1134 self._current_size += l
1136 return l
1138 def _internal_write(self, data: bytes, length: int) -> int:
1139 # Get values from locals.
1140 boundary = self.boundary
1141 boundary_length = len(boundary)
1143 # Get our state, flags and index. These are persisted between calls to
1144 # this function.
1145 state = self.state
1146 index = self.index
1147 flags = self.flags
1148 current_header_count = self._current_header_count
1149 current_header_size = self._current_header_size
1151 # Our index defaults to 0.
1152 i = 0
1154 def advance_header_size(amount: int = 1) -> None:
1155 nonlocal current_header_size
1156 current_header_size += amount
1157 if current_header_size > self.max_header_size:
1158 raise MultipartParseError("Maximum header size exceeded", offset=i)
1160 # Set a mark.
1161 def set_mark(name: str) -> None:
1162 self.marks[name] = i
1164 # Remove a mark.
1165 def delete_mark(name: str, reset: bool = False) -> None:
1166 self.marks.pop(name, None)
1168 # Helper function that makes calling a callback with data easier. The
1169 # 'remaining' parameter will callback from the marked value until the
1170 # end of the buffer, and reset the mark, instead of deleting it. This
1171 # is used at the end of the function to call our callbacks with any
1172 # remaining data in this chunk.
1173 def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None:
1174 marked_index = self.marks.get(name)
1175 if marked_index is None:
1176 return
1178 # Otherwise, we call it from the mark to the current byte we're
1179 # processing.
1180 if end_i <= marked_index:
1181 # There is no additional data to send.
1182 pass
1183 elif marked_index >= 0:
1184 # We are emitting data from the local buffer.
1185 self.callback(name, data, marked_index, end_i)
1186 else:
1187 # Some of the data comes from a partial boundary match.
1188 # and requires look-behind.
1189 # We need to use self.flags (and not flags) because we care about
1190 # the state when we entered the loop.
1191 lookbehind_len = -marked_index
1192 if lookbehind_len <= boundary_length:
1193 self.callback(name, boundary, 0, lookbehind_len)
1194 elif self.flags & FLAG_PART_BOUNDARY:
1195 lookback = boundary + b"\r\n"
1196 self.callback(name, lookback, 0, lookbehind_len)
1197 elif self.flags & FLAG_LAST_BOUNDARY:
1198 lookback = boundary + b"--\r\n"
1199 self.callback(name, lookback, 0, lookbehind_len)
1200 else: # pragma: no cover (error case)
1201 self.logger.warning("Look-back buffer error")
1203 if end_i > 0:
1204 self.callback(name, data, 0, end_i)
1205 # If we're getting remaining data, we have got all the data we
1206 # can be certain is not a boundary, leaving only a partial boundary match.
1207 if remaining:
1208 self.marks[name] = end_i - length
1209 else:
1210 self.marks.pop(name, None)
1212 # For each byte...
1213 while i < length:
1214 c = data[i]
1216 if state == START:
1217 # Skip leading newlines
1218 if c == CR or c == LF:
1219 i = data.find(b"-", i)
1220 if i == -1:
1221 # No boundary candidate in this chunk, so ignore the content after the leading CR/LF.
1222 i = length
1223 break
1224 continue
1226 # index is used as in index into our boundary. Set to 0.
1227 index = 0
1229 # Move to the next state, but decrement i so that we re-process
1230 # this character.
1231 state = START_BOUNDARY
1232 i -= 1
1234 elif state == START_BOUNDARY:
1235 if index == 0 and data.startswith(boundary[2:], i, length):
1236 index = boundary_length - 2
1237 i += index
1238 continue
1240 # Check to ensure that the last 2 characters in our boundary
1241 # are CRLF.
1242 if index == boundary_length - 2:
1243 if c == HYPHEN:
1244 # Potential empty message.
1245 state = END_BOUNDARY
1246 elif c != CR:
1247 # Error!
1248 msg = "Did not find CR at end of boundary (%d)" % (i,)
1249 self.logger.warning(msg)
1250 raise MultipartParseError(msg, offset=i)
1252 index += 1
1254 elif index == boundary_length - 1:
1255 if c != LF:
1256 msg = "Did not find LF at end of boundary (%d)" % (i,)
1257 self.logger.warning(msg)
1258 raise MultipartParseError(msg, offset=i)
1260 # The index is now used for indexing into our boundary.
1261 index = 0
1263 # Callback for the start of a part.
1264 self.callback("part_begin")
1265 current_header_count = 0
1266 current_header_size = 0
1268 # Move to the next character and state.
1269 state = HEADER_FIELD_START
1271 else:
1272 # Check to ensure our boundary matches
1273 if c != boundary[index + 2]:
1274 msg = "Expected boundary character %r, got %r at index %d" % (boundary[index + 2], c, index + 2)
1275 self.logger.warning(msg)
1276 raise MultipartParseError(msg, offset=i)
1278 # Increment index into boundary and continue.
1279 index += 1
1281 elif state == HEADER_FIELD_START:
1282 # Mark the start of a header field here, reset the index, and
1283 # continue parsing our header field.
1284 index = 0
1286 if c != CR:
1287 current_header_count += 1
1288 if current_header_count > self.max_header_count:
1289 raise MultipartParseError("Maximum header count exceeded", offset=i)
1290 current_header_size = 0
1292 # Set a mark of our header field.
1293 set_mark("header_field")
1295 # Notify that we're starting a header if the next character is
1296 # not a CR; a CR at the beginning of the header will cause us
1297 # to stop parsing headers in the MultipartState.HEADER_FIELD state,
1298 # below.
1299 if c != CR:
1300 self.callback("header_begin")
1302 # Move to parsing header fields.
1303 state = HEADER_FIELD
1304 i -= 1
1306 elif state == HEADER_FIELD:
1307 # If we've reached a CR at the beginning of a header, it means
1308 # that we've reached the second of 2 newlines, and so there are
1309 # no more headers to parse.
1310 if c == CR and index == 0:
1311 delete_mark("header_field")
1312 state = HEADERS_ALMOST_DONE
1313 i += 1
1314 continue
1316 # The field name runs until the colon; jump straight to it and
1317 # validate the whole span at once instead of byte by byte.
1318 colon = data.find(b":", i, length)
1319 end = colon if colon != -1 else length
1321 # Enforce the size limit before slicing and validating, so an oversized header
1322 # name fails fast instead of copying and scanning a potentially huge span.
1323 advance_header_size(end - i if colon == -1 else end - i + 1)
1325 field = data[i:end]
1326 if field.translate(None, TOKEN_CHARS):
1327 bad = next(b for b in field if b not in TOKEN_CHARS_SET)
1328 bad_i = i + field.index(bad)
1329 msg = "Found invalid character %r in header at %d" % (bad, bad_i)
1330 self.logger.warning(msg)
1331 raise MultipartParseError(msg, offset=bad_i)
1333 index += end - i
1334 if colon == -1:
1335 # Field name continues into the next chunk.
1336 i = length
1337 else:
1338 # A 0-length header is an error.
1339 if index == 0:
1340 msg = "Found 0-length header at %d" % (i,)
1341 self.logger.warning(msg)
1342 raise MultipartParseError(msg, offset=i)
1344 # Call our callback with the header field.
1345 i = colon
1346 data_callback("header_field", i)
1348 # Move to parsing the header value.
1349 state = HEADER_VALUE_START
1351 elif state == HEADER_VALUE_START:
1352 # Skip leading spaces.
1353 if c == SPACE:
1354 advance_header_size()
1355 i += 1
1356 continue
1358 # Mark the start of the header value.
1359 set_mark("header_value")
1361 # Move to the header-value state, reprocessing this character.
1362 state = HEADER_VALUE
1363 i -= 1
1365 elif state == HEADER_VALUE:
1366 # The value runs until the terminating CR; jump straight to it
1367 # instead of inspecting every byte.
1368 cr = data.find(b"\r", i, length)
1369 end = cr if cr != -1 else length
1370 advance_header_size(end - i)
1371 if cr != -1:
1372 i = cr
1373 data_callback("header_value", i)
1374 self.callback("header_end")
1375 current_header_size = 0
1376 state = HEADER_VALUE_ALMOST_DONE
1377 else:
1378 i = length
1380 elif state == HEADER_VALUE_ALMOST_DONE:
1381 # The last character should be a LF. If not, it's an error.
1382 if c != LF:
1383 msg = f"Did not find LF character at end of header (found {c!r})"
1384 self.logger.warning(msg)
1385 raise MultipartParseError(msg, offset=i)
1387 # Move back to the start of another header. Note that if that
1388 # state detects ANOTHER newline, it'll trigger the end of our
1389 # headers.
1390 state = HEADER_FIELD_START
1392 elif state == HEADERS_ALMOST_DONE:
1393 # We're almost done our headers. This is reached when we parse
1394 # a CR at the beginning of a header, so our next character
1395 # should be a LF, or it's an error.
1396 if c != LF:
1397 msg = f"Did not find LF at end of headers (found {c!r})"
1398 self.logger.warning(msg)
1399 raise MultipartParseError(msg, offset=i)
1401 self.callback("headers_finished")
1402 state = PART_DATA_START
1404 elif state == PART_DATA_START:
1405 # Mark the start of our part data.
1406 set_mark("part_data")
1408 # Start processing part data, including this character.
1409 state = PART_DATA
1410 i -= 1
1412 elif state == PART_DATA:
1413 # We're processing our part data right now. During this, we
1414 # need to efficiently search for our boundary, since any data
1415 # on any number of lines can be a part of the current data.
1417 # Save the current value of our index. We use this in case we
1418 # find part of a boundary, but it doesn't match fully.
1419 prev_index = index
1421 # If our index is 0, we're starting a new part, so start our
1422 # search.
1423 if index == 0:
1424 # The most common case is likely to be that the whole
1425 # boundary is present in the buffer.
1426 # Calling `find` is much faster than iterating here.
1427 i0 = data.find(boundary, i, length)
1428 if i0 >= 0:
1429 # We matched the whole boundary string.
1430 index = boundary_length - 1
1431 i = i0 + boundary_length - 1
1432 c = data[i]
1433 else:
1434 # No whole boundary, but the tail may hold a partial one
1435 # that completes in the next chunk. Boundary starts with
1436 # CR, which an RFC boundary contains nowhere else, so the
1437 # last CR in the tail is the only candidate prefix start.
1438 k = data.rfind(boundary[:1], max(i, length - boundary_length + 1), length)
1439 if k != -1 and boundary.startswith(data[k:length]):
1440 index = length - k
1441 # Carry the partial via index; the end-of-chunk flush
1442 # emits the data before it and re-marks the lookbehind.
1443 i = length
1444 continue
1446 # Now, we have a couple of cases here. If our index is before
1447 # the end of the boundary...
1448 if index < boundary_length:
1449 # If the character matches...
1450 if boundary[index] == c:
1451 # The current character matches, so continue!
1452 index += 1
1453 else:
1454 index = 0
1456 # Our index is equal to the length of our boundary!
1457 elif index == boundary_length:
1458 # First we increment it.
1459 index += 1
1461 # Now, if we've reached a newline, we need to set this as
1462 # the potential end of our boundary.
1463 if c == CR:
1464 flags |= FLAG_PART_BOUNDARY
1466 # Otherwise, if this is a hyphen, we might be at the last
1467 # of all boundaries.
1468 elif c == HYPHEN:
1469 flags |= FLAG_LAST_BOUNDARY
1471 # RFC 2046 forbids lines starting with the boundary in a part, so this can't be data.
1472 elif c in b" \t":
1473 data_callback("part_data", i - boundary_length)
1474 delete_mark("part_data")
1475 index = 0
1476 state = BOUNDARY_PADDING
1478 # Otherwise, we reset our index, since this isn't either a
1479 # newline or a hyphen.
1480 else:
1481 index = 0
1483 # Our index is right after the part boundary, which should be
1484 # a LF.
1485 elif index == boundary_length + 1:
1486 # If we're at a part boundary (i.e. we've seen a CR
1487 # character already)...
1488 if flags & FLAG_PART_BOUNDARY:
1489 # We need a LF character next.
1490 if c == LF:
1491 # Unset the part boundary flag.
1492 flags &= ~FLAG_PART_BOUNDARY
1494 # We have identified a boundary, callback for any data before it.
1495 data_callback("part_data", i - index)
1496 # Callback indicating that we've reached the end of
1497 # a part, and are starting a new one.
1498 self.callback("part_end")
1499 self.callback("part_begin")
1500 current_header_count = 0
1501 current_header_size = 0
1503 # Move to parsing new headers.
1504 index = 0
1505 state = HEADER_FIELD_START
1506 i += 1
1507 continue
1509 # We didn't find an LF character, so no match. Reset
1510 # our index and clear our flag.
1511 index = 0
1512 flags &= ~FLAG_PART_BOUNDARY
1514 # Otherwise, if we're at the last boundary (i.e. we've
1515 # seen a hyphen already)...
1516 elif flags & FLAG_LAST_BOUNDARY:
1517 # We need a second hyphen here.
1518 if c == HYPHEN:
1519 # We have identified a boundary, callback for any data before it.
1520 data_callback("part_data", i - index)
1521 # Callback to end the current part, and then the
1522 # message.
1523 self.callback("part_end")
1524 self.callback("end")
1525 state = END
1526 else:
1527 # No match, so reset index.
1528 index = 0
1530 # Otherwise, our index is 0. If the previous index is not, it
1531 # means we reset something, and we need to take the data we
1532 # thought was part of our boundary and send it along as actual
1533 # data.
1534 if index == 0 and prev_index > 0:
1535 # Overwrite our previous index.
1536 prev_index = 0
1538 # Re-consider the current character, since this could be
1539 # the start of the boundary itself.
1540 i -= 1
1542 elif state == BOUNDARY_PADDING:
1543 if index == 0 and c == CR:
1544 index = 1
1545 elif index == 1 and c == LF:
1546 self.callback("part_end")
1547 self.callback("part_begin")
1548 current_header_count = 0
1549 current_header_size = 0
1550 index = 0
1551 state = HEADER_FIELD_START
1552 elif index == 1 or c not in b" \t":
1553 msg = "Did not find CRLF after boundary padding at %d" % (i,)
1554 self.logger.warning(msg)
1555 raise MultipartParseError(msg, offset=i)
1557 elif state == END_BOUNDARY:
1558 if index == boundary_length - 1:
1559 if c != HYPHEN:
1560 msg = "Did not find - at end of boundary (%d)" % (i,)
1561 self.logger.warning(msg)
1562 raise MultipartParseError(msg, offset=i)
1563 index += 1
1564 self.callback("end")
1565 state = END
1567 elif state == END:
1568 # Silently discard any epilogue data (RFC 2046 section 5.1.1 allows a CRLF and optional
1569 # epilogue after the closing boundary). Django and Werkzeug do the same.
1570 i = length
1571 break
1573 else: # pragma: no cover (error case)
1574 # We got into a strange state somehow! Just stop processing.
1575 msg = "Reached an unknown state %d at %d" % (state, i)
1576 self.logger.warning(msg)
1577 raise MultipartParseError(msg, offset=i)
1579 # Move to the next byte.
1580 i += 1
1582 # We call our callbacks with any remaining data. Note that we pass
1583 # the 'remaining' flag, which sets the mark back to 0 instead of
1584 # deleting it, if it's found. This is because, if the mark is found
1585 # at this point, we assume that there's data for one of these things
1586 # that has been parsed, but not yet emitted. And, as such, it implies
1587 # that we haven't yet reached the end of this 'thing'. So, by setting
1588 # the mark to 0, we cause any data callbacks that take place in future
1589 # calls to this function to start from the beginning of that buffer.
1590 data_callback("header_field", length, True)
1591 data_callback("header_value", length, True)
1592 data_callback("part_data", length - index, True)
1594 # Save values to locals.
1595 self.state = state
1596 self.index = index
1597 self.flags = flags
1598 self._current_header_count = current_header_count
1599 self._current_header_size = current_header_size
1601 # Return our data length to indicate no errors, and that we processed
1602 # all of it.
1603 return length
1605 def finalize(self) -> None:
1606 """Finalize this parser, which signals to that we are finished parsing.
1608 Note: It does not currently, but in the future, it will verify that we
1609 are in the final state of the parser (i.e. the end of the multipart
1610 message is well-formed), and, if not, throw an error.
1611 """
1612 # TODO: verify that we're in the state MultipartState.END, otherwise throw an
1613 # error or otherwise state that we're not finished parsing.
1614 pass
1616 def __repr__(self) -> str:
1617 return f"{self.__class__.__name__}(boundary={self.boundary!r})"
1620class FormParser:
1621 """This class is the all-in-one form parser. Given all the information
1622 necessary to parse a form, it will instantiate the correct parser, create
1623 the proper :class:`Field` and :class:`File` classes to store the data that
1624 is parsed, and call the two given callbacks with each field and file as
1625 they become available.
1627 Args:
1628 content_type: The Content-Type of the incoming request. This is used to select the appropriate parser.
1629 on_field: The callback to call when a field has been parsed and is ready for usage. See above for parameters.
1630 on_file: The callback to call when a file has been parsed and is ready for usage. See above for parameters.
1631 on_end: An optional callback to call when all fields and files in a request has been parsed. Can be None.
1632 boundary: If the request is a multipart/form-data request, this should be the boundary of the request, as given
1633 in the Content-Type header, as a bytestring.
1634 file_name: If the request is of type application/octet-stream, then the body of the request will not contain any
1635 information about the uploaded file. In such cases, you can provide the file name of the uploaded file
1636 manually.
1637 config: Configuration to use for this FormParser. The default values are taken from the DEFAULT_CONFIG value,
1638 and then any keys present in this dictionary will overwrite the default values.
1639 """
1641 #: This is the default configuration for our form parser.
1642 #: Note: all file sizes should be in bytes.
1643 DEFAULT_CONFIG: FormParserConfig = {
1644 "MAX_BODY_SIZE": float("inf"),
1645 "MAX_FIELDS": float("inf"),
1646 "MAX_FIELD_SIZE": float("inf"),
1647 "MAX_HEADER_COUNT": DEFAULT_MAX_HEADER_COUNT,
1648 "MAX_HEADER_SIZE": DEFAULT_MAX_HEADER_SIZE,
1649 "MAX_MEMORY_FILE_SIZE": 1 * 1024 * 1024,
1650 "UPLOAD_DIR": None,
1651 "UPLOAD_DELETE_TMP": True,
1652 "UPLOAD_KEEP_FILENAME": False,
1653 "UPLOAD_KEEP_EXTENSIONS": False,
1654 # Error on invalid Content-Transfer-Encoding?
1655 "UPLOAD_ERROR_ON_BAD_CTE": False,
1656 }
1658 def __init__(
1659 self,
1660 content_type: str,
1661 on_field: Callable[[Field], None] | None,
1662 on_file: Callable[[File], None] | None,
1663 on_end: Callable[[], None] | None = None,
1664 boundary: bytes | str | None = None,
1665 file_name: bytes | None = None,
1666 config: dict[Any, Any] = {},
1667 ) -> None:
1668 self.logger = logging.getLogger(__name__)
1670 # Save variables.
1671 self.content_type = content_type
1672 self.boundary = boundary
1673 self.bytes_received = 0
1674 self.parser = None
1676 # Save callbacks.
1677 self.on_field = on_field
1678 self.on_file = on_file
1679 self.on_end = on_end
1681 # Set configuration options.
1682 self.config: FormParserConfig = self.DEFAULT_CONFIG.copy()
1683 self.config.update(config) # type: ignore[typeddict-item]
1685 parser: OctetStreamParser | MultipartParser | QuerystringParser | None = None
1687 # Depending on the Content-Type, we instantiate the correct parser.
1688 if content_type == "application/octet-stream":
1689 file: File | None = None
1691 def on_start() -> None:
1692 nonlocal file
1693 file = File(file_name, None, config=self.config)
1695 def on_data(data: bytes, start: int, end: int) -> None:
1696 nonlocal file
1697 assert file is not None
1698 file.write(data[start:end])
1700 def _on_end() -> None:
1701 nonlocal file
1702 assert file is not None
1703 # Finalize the file itself.
1704 file.finalize()
1706 # Call our callback.
1707 if on_file:
1708 on_file(file)
1710 # Call the on-end callback.
1711 if self.on_end is not None:
1712 self.on_end()
1714 # Instantiate an octet-stream parser
1715 parser = OctetStreamParser(
1716 callbacks={"on_start": on_start, "on_data": on_data, "on_end": _on_end},
1717 max_size=self.config["MAX_BODY_SIZE"],
1718 )
1720 elif content_type == "application/x-www-form-urlencoded" or content_type == "application/x-url-encoded":
1721 max_fields = self.config["MAX_FIELDS"]
1722 max_field_size = self.config["MAX_FIELD_SIZE"]
1723 for key, limit in (("MAX_FIELDS", max_fields), ("MAX_FIELD_SIZE", max_field_size)):
1724 if not isinstance(limit, (int, float)) or not limit >= 0:
1725 raise ValueError(f"{key} must be a non-negative number, not {limit!r}")
1727 name_buffer: list[bytes] = []
1728 field_count = 0
1729 field_size = 0
1730 f: Field | None = None
1732 def on_field_start() -> None:
1733 nonlocal field_count, field_size
1734 field_count += 1
1735 if field_count > max_fields:
1736 raise FormParserError(f"Too many fields. Maximum number of fields is {max_fields}.")
1737 field_size = 0
1739 def on_field_name(data: bytes, start: int, end: int) -> None:
1740 nonlocal field_size
1741 field_size += end - start
1742 if field_size > max_field_size:
1743 raise FormParserError(f"Field exceeded maximum size of {max_field_size} bytes.")
1744 name_buffer.append(data[start:end])
1746 def on_field_data(data: bytes, start: int, end: int) -> None:
1747 nonlocal f, field_size
1748 field_size += end - start
1749 if field_size > max_field_size:
1750 raise FormParserError(f"Field exceeded maximum size of {max_field_size} bytes.")
1751 if f is None:
1752 f = Field(b"".join(name_buffer))
1753 del name_buffer[:]
1754 f.write(data[start:end])
1756 def on_field_end() -> None:
1757 nonlocal f
1758 # Finalize and call callback.
1759 if f is None:
1760 # If we get here, it's because there was no field data.
1761 # We create a field, set it to None, and then continue.
1762 f = Field(b"".join(name_buffer))
1763 del name_buffer[:]
1764 f.set_none()
1766 f.finalize()
1767 if on_field:
1768 on_field(f)
1769 f = None
1771 def _on_end() -> None:
1772 if self.on_end is not None:
1773 self.on_end()
1775 # Instantiate parser.
1776 parser = QuerystringParser(
1777 callbacks={
1778 "on_field_start": on_field_start,
1779 "on_field_name": on_field_name,
1780 "on_field_data": on_field_data,
1781 "on_field_end": on_field_end,
1782 "on_end": _on_end,
1783 },
1784 max_size=self.config["MAX_BODY_SIZE"],
1785 )
1787 elif content_type == "multipart/form-data":
1788 if boundary is None:
1789 self.logger.error("No boundary given")
1790 raise FormParserError("No boundary given")
1792 header_name: list[bytes] = []
1793 header_value: list[bytes] = []
1794 headers: dict[bytes, bytes] = {}
1796 f_multi: File | Field | None = None
1797 writer: File | Field | Base64Decoder | QuotedPrintableDecoder | None = None
1798 is_file = False
1800 def on_part_begin() -> None:
1801 # Reset headers in case this isn't the first part.
1802 nonlocal headers
1803 headers = {}
1805 def on_part_data(data: bytes, start: int, end: int) -> None:
1806 nonlocal writer
1807 assert writer is not None
1808 writer.write(data[start:end])
1809 # TODO: check for error here.
1811 def on_part_end() -> None:
1812 assert f_multi is not None
1813 assert writer is not None
1814 writer.finalize()
1815 if is_file:
1816 if on_file:
1817 assert isinstance(f_multi, File)
1818 on_file(f_multi)
1819 else:
1820 if on_field:
1821 assert isinstance(f_multi, Field)
1822 on_field(f_multi)
1824 def on_header_field(data: bytes, start: int, end: int) -> None:
1825 header_name.append(data[start:end])
1827 def on_header_value(data: bytes, start: int, end: int) -> None:
1828 header_value.append(data[start:end])
1830 def on_header_end() -> None:
1831 headers[b"".join(header_name).lower()] = b"".join(header_value)
1832 del header_name[:]
1833 del header_value[:]
1835 def on_headers_finished() -> None:
1836 nonlocal is_file, f_multi, writer
1837 # Reset the 'is file' flag.
1838 is_file = False
1840 # Parse the content-disposition header.
1841 content_disp = headers.get(b"content-disposition")
1842 disp, options = parse_options_header(content_disp)
1844 # Get the field and filename.
1845 field_name = options.get(b"name")
1846 file_name = options.get(b"filename")
1847 # RFC 7578 §4.2: each part MUST have a Content-Disposition header with a "name" parameter.
1848 if field_name is None:
1849 raise FormParserError(f'Field name not found in Content-Disposition: "{content_disp!r}"')
1851 # Create the proper class.
1852 content_type_b = headers.get(b"content-type")
1853 content_type = content_type_b.decode("latin-1") if content_type_b is not None else None
1854 if file_name is None:
1855 f_multi = Field(field_name, content_type=content_type)
1856 else:
1857 f_multi = File(file_name, field_name, config=self.config, content_type=content_type)
1858 is_file = True
1860 # Parse the given Content-Transfer-Encoding to determine what
1861 # we need to do with the incoming data.
1862 # TODO: check that we properly handle 8bit / 7bit encoding.
1863 # RFC 2045 section 6.1: Content-Transfer-Encoding values are case-insensitive.
1864 # https://www.rfc-editor.org/rfc/rfc2045#section-6.1
1865 transfer_encoding = headers.get(b"content-transfer-encoding", b"7bit").lower()
1867 if transfer_encoding in (b"binary", b"8bit", b"7bit"):
1868 writer = f_multi
1870 elif transfer_encoding == b"base64":
1871 writer = Base64Decoder(f_multi)
1873 elif transfer_encoding == b"quoted-printable":
1874 writer = QuotedPrintableDecoder(f_multi)
1876 else:
1877 self.logger.warning("Unknown Content-Transfer-Encoding: %r", transfer_encoding)
1878 if self.config["UPLOAD_ERROR_ON_BAD_CTE"]:
1879 raise FormParserError(f'Unknown Content-Transfer-Encoding "{transfer_encoding!r}"')
1880 else:
1881 # If we aren't erroring, then we just treat this as an
1882 # unencoded Content-Transfer-Encoding.
1883 writer = f_multi
1885 def _on_end() -> None:
1886 if self.on_end is not None:
1887 self.on_end()
1889 # Instantiate a multipart parser.
1890 parser = MultipartParser(
1891 boundary,
1892 callbacks={
1893 "on_part_begin": on_part_begin,
1894 "on_part_data": on_part_data,
1895 "on_part_end": on_part_end,
1896 "on_header_field": on_header_field,
1897 "on_header_value": on_header_value,
1898 "on_header_end": on_header_end,
1899 "on_headers_finished": on_headers_finished,
1900 "on_end": _on_end,
1901 },
1902 max_size=self.config["MAX_BODY_SIZE"],
1903 max_header_count=self.config["MAX_HEADER_COUNT"],
1904 max_header_size=self.config["MAX_HEADER_SIZE"],
1905 )
1907 else:
1908 self.logger.warning("Unknown Content-Type: %r", content_type)
1909 raise FormParserError(f"Unknown Content-Type: {content_type}")
1911 self.parser = parser
1913 def write(self, data: bytes) -> int:
1914 """Write some data. The parser will forward this to the appropriate
1915 underlying parser.
1917 Args:
1918 data: The data to write.
1920 Returns:
1921 The number of bytes processed.
1922 """
1923 self.bytes_received += len(data)
1924 # TODO: check the parser's return value for errors?
1925 assert self.parser is not None
1926 return self.parser.write(data)
1928 def finalize(self) -> None:
1929 """Finalize the parser."""
1930 if self.parser is not None and hasattr(self.parser, "finalize"):
1931 self.parser.finalize()
1933 def close(self) -> None:
1934 """Close the parser."""
1935 if self.parser is not None and hasattr(self.parser, "close"):
1936 self.parser.close()
1938 def __repr__(self) -> str:
1939 return f"{self.__class__.__name__}(content_type={self.content_type!r}, parser={self.parser!r})"
1942def create_form_parser(
1943 headers: dict[str, bytes],
1944 on_field: Callable[[Field], None] | None,
1945 on_file: Callable[[File], None] | None,
1946 config: dict[Any, Any] = {},
1947) -> FormParser:
1948 """This function is a helper function to aid in creating a FormParser
1949 instances. Given a dictionary-like headers object, it will determine
1950 the correct information needed, instantiate a FormParser with the
1951 appropriate values and given callbacks, and then return the corresponding
1952 parser.
1954 Args:
1955 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1956 on_field: Callback to call with each parsed field.
1957 on_file: Callback to call with each parsed file.
1958 config: Configuration variables to pass to the FormParser.
1959 """
1960 content_type: str | bytes | None = headers.get("Content-Type")
1961 if content_type is None:
1962 logging.getLogger(__name__).warning("No Content-Type header given")
1963 raise ValueError("No Content-Type header given!")
1965 # Boundaries are optional (the FormParser will raise if one is needed
1966 # but not given).
1967 content_type, params = parse_options_header(content_type)
1968 boundary = params.get(b"boundary")
1970 # We need content_type to be a string, not a bytes object.
1971 content_type = content_type.decode("latin-1")
1973 # Instantiate a form parser.
1974 form_parser = FormParser(content_type, on_field, on_file, boundary=boundary, config=config)
1976 # Return our parser.
1977 return form_parser
1980def parse_form(
1981 headers: dict[str, bytes],
1982 input_stream: SupportsRead,
1983 on_field: Callable[[Field], None] | None,
1984 on_file: Callable[[File], None] | None,
1985 chunk_size: int = 1048576,
1986) -> None:
1987 """This function is useful if you just want to parse a request body,
1988 without too much work. Pass it a dictionary-like object of the request's
1989 headers, and a file-like object for the input stream, along with two
1990 callbacks that will get called whenever a field or file is parsed.
1992 Args:
1993 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1994 input_stream: A file-like object that represents the request body. The read() method must return bytestrings.
1995 on_field: Callback to call with each parsed field.
1996 on_file: Callback to call with each parsed file.
1997 chunk_size: The maximum size to read from the input stream and write to the parser at one time.
1998 Defaults to 1 MiB.
1999 """
2000 if chunk_size < 1:
2001 raise ValueError(f"chunk_size must be a positive number, not {chunk_size!r}")
2003 # Create our form parser.
2004 parser = create_form_parser(headers, on_field, on_file)
2006 # Read chunks of 1MiB and write to the parser, but never read more than
2007 # the given Content-Length, if any.
2008 content_length: int | float | bytes | None = headers.get("Content-Length")
2009 if content_length is not None:
2010 content_length = int(content_length)
2011 if content_length < 0:
2012 raise ValueError("Content-Length must be non-negative")
2013 else:
2014 content_length = float("inf")
2015 bytes_read = 0
2017 while True:
2018 # Read only up to the Content-Length given.
2019 max_readable = int(min(content_length - bytes_read, chunk_size))
2020 buff = input_stream.read(max_readable)
2022 # Write to the parser and update our length.
2023 parser.write(buff)
2024 bytes_read += len(buff)
2026 # If we get a buffer that's smaller than the size requested, or if we
2027 # have read up to our content length, we're done.
2028 if len(buff) != max_readable or bytes_read == content_length:
2029 break
2031 # Tell our parser that we're done writing data.
2032 parser.finalize()