Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import logging
4import os
5import shutil
6import sys
7import tempfile
8from enum import IntEnum, global_enum
9from io import BufferedRandom, BytesIO
10from numbers import Number
11from typing import TYPE_CHECKING, cast
13from .decoders import Base64Decoder, QuotedPrintableDecoder
14from .exceptions import FileError, FormParserError, MultipartParseError, QuerystringParseError
16if TYPE_CHECKING:
17 from collections.abc import Callable
18 from typing import Any, Literal, Protocol, TypeAlias, TypedDict
20 class SupportsRead(Protocol):
21 def read(self, __n: int) -> bytes: ...
23 class QuerystringCallbacks(TypedDict, total=False):
24 on_field_start: Callable[[], None]
25 on_field_name: Callable[[bytes, int, int], None]
26 on_field_data: Callable[[bytes, int, int], None]
27 on_field_end: Callable[[], None]
28 on_end: Callable[[], None]
30 class OctetStreamCallbacks(TypedDict, total=False):
31 on_start: Callable[[], None]
32 on_data: Callable[[bytes, int, int], None]
33 on_end: Callable[[], None]
35 class MultipartCallbacks(TypedDict, total=False):
36 on_part_begin: Callable[[], None]
37 on_part_data: Callable[[bytes, int, int], None]
38 on_part_end: Callable[[], None]
39 on_header_begin: Callable[[], None]
40 on_header_field: Callable[[bytes, int, int], None]
41 on_header_value: Callable[[bytes, int, int], None]
42 on_header_end: Callable[[], None]
43 on_headers_finished: Callable[[], None]
44 on_end: Callable[[], None]
46 class FileConfig(TypedDict, total=False):
47 UPLOAD_DIR: str | bytes | None
48 UPLOAD_DELETE_TMP: bool
49 UPLOAD_KEEP_FILENAME: bool
50 UPLOAD_KEEP_EXTENSIONS: bool
51 MAX_MEMORY_FILE_SIZE: int
53 class FormParserConfig(FileConfig):
54 UPLOAD_ERROR_ON_BAD_CTE: bool
55 MAX_BODY_SIZE: float
56 MAX_FIELDS: int | float
57 MAX_FIELD_SIZE: int | float
58 MAX_HEADER_COUNT: int
59 MAX_HEADER_SIZE: int
61 CallbackName: TypeAlias = Literal[
62 "start",
63 "data",
64 "end",
65 "field_start",
66 "field_name",
67 "field_data",
68 "field_end",
69 "part_begin",
70 "part_data",
71 "part_end",
72 "header_begin",
73 "header_field",
74 "header_value",
75 "header_end",
76 "headers_finished",
77 ]
79# Unique missing object.
80_missing = object()
83def _noop_event() -> None:
84 pass
87def _noop_data(_data: bytes, _start: int, _end: int) -> None:
88 pass
91@global_enum
92class QuerystringState(IntEnum):
93 """Querystring parser states.
95 These are used to keep track of the state of the parser, and are used to determine
96 what to do when new data is encountered.
97 """
99 BEFORE_FIELD = 0
100 FIELD_NAME = 1
101 FIELD_DATA = 2
104@global_enum
105class MultipartState(IntEnum):
106 """Multipart parser states.
108 These are used to keep track of the state of the parser, and are used to determine
109 what to do when new data is encountered.
110 """
112 START = 0
113 START_BOUNDARY = 1
114 HEADER_FIELD_START = 2
115 HEADER_FIELD = 3
116 HEADER_VALUE_START = 4
117 HEADER_VALUE = 5
118 HEADER_VALUE_ALMOST_DONE = 6
119 HEADERS_ALMOST_DONE = 7
120 PART_DATA_START = 8
121 PART_DATA = 9
122 PART_DATA_END = 10
123 END_BOUNDARY = 11
124 END = 12
127if TYPE_CHECKING:
128 BEFORE_FIELD = QuerystringState.BEFORE_FIELD
129 FIELD_NAME = QuerystringState.FIELD_NAME
130 FIELD_DATA = QuerystringState.FIELD_DATA
132 START = MultipartState.START
133 START_BOUNDARY = MultipartState.START_BOUNDARY
134 HEADER_FIELD_START = MultipartState.HEADER_FIELD_START
135 HEADER_FIELD = MultipartState.HEADER_FIELD
136 HEADER_VALUE_START = MultipartState.HEADER_VALUE_START
137 HEADER_VALUE = MultipartState.HEADER_VALUE
138 HEADER_VALUE_ALMOST_DONE = MultipartState.HEADER_VALUE_ALMOST_DONE
139 HEADERS_ALMOST_DONE = MultipartState.HEADERS_ALMOST_DONE
140 PART_DATA_START = MultipartState.PART_DATA_START
141 PART_DATA = MultipartState.PART_DATA
142 PART_DATA_END = MultipartState.PART_DATA_END
143 END_BOUNDARY = MultipartState.END_BOUNDARY
144 END = MultipartState.END
146# Flags for the multipart parser.
147FLAG_PART_BOUNDARY = 1
148FLAG_LAST_BOUNDARY = 2
150# Get constants. Since iterating over a str on Python 2 gives you a 1-length
151# string, but iterating over a bytes object on Python 3 gives you an integer,
152# we need to save these constants.
153CR = b"\r"[0]
154LF = b"\n"[0]
155COLON = b":"[0]
156SPACE = b" "[0]
157HYPHEN = b"-"[0]
158AMPERSAND = b"&"[0]
159LOWER_A = b"a"[0]
160LOWER_Z = b"z"[0]
161NULL = b"\x00"[0]
163# fmt: off
164# Mask for ASCII characters that can be http tokens.
165# Per RFC7230 - 3.2.6, this is all alpha-numeric characters
166# and these: !#$%&'*+-.^_`|~
167TOKEN_CHARS = (
168 b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
169 b"abcdefghijklmnopqrstuvwxyz"
170 b"0123456789"
171 b"!#$%&'*+-.^_`|~")
172TOKEN_CHARS_SET = frozenset(TOKEN_CHARS)
173# fmt: on
175DEFAULT_MAX_HEADER_COUNT = 8
176"""Default maximum number of headers allowed per multipart part."""
178DEFAULT_MAX_HEADER_SIZE = 4096 + 128
179"""Default maximum size of a single multipart header line, including syntax overhead."""
181MAX_BOUNDARY_LENGTH = 256
182"""Maximum allowed length of a multipart boundary.
184[RFC 2046 §5.1.1](https://datatracker.ietf.org/doc/html/rfc2046#section-5.1.1)
185recommends boundaries be at most 70 bytes. 256 bytes is generous headroom over
186every HTTP client.
187"""
190def _parseparam(s: str) -> list[str]:
191 # Vendored from the standard library's
192 # [`email.message._parseparam`](https://github.com/python/cpython/blob/v3.14.2/Lib/email/message.py#L73-L96)
193 # to split a header into its `;`-separated parts without treating a `;` inside a double-quoted string as a
194 # separator - and without the RFC 2231 decoding that `email.message.Message.get_params` would apply on top.
195 s = ";" + s
196 plist: list[str] = []
197 start = 0
198 while s.find(";", start) == start:
199 start += 1
200 end = s.find(";", start)
201 ind, diff = start, 0
202 while end > 0:
203 diff += s.count('"', ind, end) - s.count('\\"', ind, end)
204 if diff % 2 == 0:
205 break
206 end, ind = ind, s.find(";", end + 1)
207 if end < 0:
208 end = len(s)
209 i = s.find("=", start, end)
210 if i == -1:
211 f = s[start:end]
212 else:
213 f = s[start:i].rstrip().lower() + "=" + s[i + 1 : end].lstrip()
214 plist.append(f.strip())
215 start = end
216 return plist
219def parse_options_header(value: str | bytes | None) -> tuple[bytes, dict[bytes, bytes]]:
220 """Parses a Content-Type header into a value in the following format: (content_type, {parameters})."""
221 if not value:
222 return (b"", {})
224 # If we are passed bytes, we assume that it conforms to WSGI, encoding in latin-1.
225 if isinstance(value, bytes): # pragma: no cover
226 value = value.decode("latin-1")
228 # For types
229 assert isinstance(value, str), "Value should be a string by now"
231 # If we have no options, return the string as-is.
232 if ";" not in value:
233 return (value.lower().strip().encode("latin-1"), {})
235 ctype, *segments = _parseparam(value)
236 options: dict[bytes, bytes] = {}
237 for segment in segments:
238 key, _, val = segment.partition("=")
239 # [RFC 7578 §4.2](https://datatracker.ietf.org/doc/html/rfc7578#section-4.2)
240 # forbids the RFC 5987/2231 extended syntax (`key*=`, `key*0`, ...) in
241 # multipart/form-data, so we ignore those parameters and keep the plain
242 # `key` authoritative.
243 if "*" in key:
244 continue
245 if len(val) >= 2 and val[0] == '"' and val[-1] == '"':
246 val = val[1:-1].replace("\\\\", "\\").replace('\\"', '"')
247 # Work around an IE6 bug where the full file path is sent instead of
248 # just the filename.
249 if key == "filename" and (val[1:3] == ":\\" or val[:2] == "\\\\"):
250 val = val.split("\\")[-1]
251 options[key.encode("latin-1")] = val.encode("latin-1")
252 return ctype.lower().encode("latin-1"), options
255class Field:
256 """A Field object represents a (parsed) form field. It represents a single
257 field with a corresponding name and value.
259 The name that a :class:`Field` will be instantiated with is the same name
260 that would be found in the following HTML::
262 <input name="name_goes_here" type="text"/>
264 This class defines two methods, :meth:`on_data` and :meth:`on_end`, that
265 will be called when data is written to the Field, and when the Field is
266 finalized, respectively.
268 Args:
269 name: The name of the form field.
270 content_type: The value of the Content-Type header for this field.
271 """
273 def __init__(self, name: bytes | None, *, content_type: str | None = None) -> None:
274 self._name = name
275 self._value: list[bytes] = []
276 self._content_type = content_type
278 # We cache the joined version of _value for speed.
279 self._cache = _missing
281 @classmethod
282 def from_value(cls, name: bytes, value: bytes | None) -> Field:
283 """Create an instance of a :class:`Field`, and set the corresponding
284 value - either None or an actual value. This method will also
285 finalize the Field itself.
287 Args:
288 name: the name of the form field.
289 value: the value of the form field - either a bytestring or None.
291 Returns:
292 A new instance of a [`Field`][python_multipart.Field].
293 """
295 f = cls(name)
296 if value is None:
297 f.set_none()
298 else:
299 f.write(value)
300 f.finalize()
301 return f
303 def write(self, data: bytes) -> int:
304 """Write some data into the form field.
306 Args:
307 data: The data to write to the field.
309 Returns:
310 The number of bytes written.
311 """
312 return self.on_data(data)
314 def on_data(self, data: bytes) -> int:
315 """This method is a callback that will be called whenever data is
316 written to the Field.
318 Args:
319 data: The data to write to the field.
321 Returns:
322 The number of bytes written.
323 """
324 self._value.append(data)
325 self._cache = _missing
326 return len(data)
328 def on_end(self) -> None:
329 """This method is called whenever the Field is finalized."""
330 if self._cache is _missing:
331 self._cache = b"".join(self._value)
333 def finalize(self) -> None:
334 """Finalize the form field."""
335 self.on_end()
337 def close(self) -> None:
338 """Close the Field object. This will free any underlying cache."""
339 # Free our value array.
340 if self._cache is _missing:
341 self._cache = b"".join(self._value)
343 del self._value
345 def set_none(self) -> None:
346 """Some fields in a querystring can possibly have a value of None - for
347 example, the string "foo&bar=&baz=asdf" will have a field with the
348 name "foo" and value None, one with name "bar" and value "", and one
349 with name "baz" and value "asdf". Since the write() interface doesn't
350 support writing None, this function will set the field value to None.
351 """
352 self._cache = None
354 @property
355 def field_name(self) -> bytes | None:
356 """This property returns the name of the field."""
357 return self._name
359 @property
360 def value(self) -> bytes | None:
361 """This property returns the value of the form field."""
362 if self._cache is _missing:
363 self._cache = b"".join(self._value)
365 assert isinstance(self._cache, bytes) or self._cache is None
366 return self._cache
368 @property
369 def content_type(self) -> str | None:
370 """This property returns the content_type value of the field."""
371 return self._content_type
373 def __eq__(self, other: object) -> bool:
374 if isinstance(other, Field):
375 return self.field_name == other.field_name and self.value == other.value
376 else:
377 return NotImplemented
379 def __repr__(self) -> str:
380 if self.value is not None and len(self.value) > 97:
381 # We get the repr, and then insert three dots before the final
382 # quote.
383 v = repr(self.value[:97])[:-1] + "...'"
384 else:
385 v = repr(self.value)
387 return f"{self.__class__.__name__}(field_name={self.field_name!r}, value={v})"
390class File:
391 """This class represents an uploaded file. It handles writing file data to
392 either an in-memory file or a temporary file on-disk, if the optional
393 threshold is passed.
395 There are some options that can be passed to the File to change behavior
396 of the class. Valid options are as follows:
398 | Name | Type | Default | Description |
399 |-----------------------|-------|---------|-------------|
400 | UPLOAD_DIR | `str` | None | The directory to store uploaded files in. If this is None, a temporary file will be created in the system's standard location. |
401 | UPLOAD_DELETE_TMP | `bool`| True | Delete automatically created TMP file |
402 | UPLOAD_KEEP_FILENAME | `bool`| False | Whether or not to keep the filename of the uploaded file. If True, then the filename is reduced to its basename (directory components are stripped) and the file is saved with that name. Otherwise, a temporary name will be used. |
403 | UPLOAD_KEEP_EXTENSIONS| `bool`| False | Whether or not to keep the uploaded file's extension. If False, the file will be saved with the default temporary extension (usually ".tmp"). Otherwise, the file's extension will be maintained. Note that this will properly combine with the UPLOAD_KEEP_FILENAME setting. |
404 | MAX_MEMORY_FILE_SIZE | `int` | 1 MiB | The maximum number of bytes of a File to keep in memory. By default, the contents of a File are kept into memory until a certain limit is reached, after which the contents of the File are written to a temporary file. This behavior can be disabled by setting this value to an appropriately large value (or, for example, infinity, such as `float('inf')`. |
406 Args:
407 file_name: The name of the file that this [`File`][python_multipart.File] represents.
408 field_name: The name of the form field that this file was uploaded with. This can be None, if, for example,
409 the file was uploaded with Content-Type application/octet-stream.
410 config: The configuration for this File. See above for valid configuration keys and their corresponding values.
411 content_type: The value of the Content-Type header.
412 """ # noqa: E501
414 def __init__(
415 self,
416 file_name: bytes | None,
417 field_name: bytes | None = None,
418 config: FileConfig = {},
419 *,
420 content_type: str | None = None,
421 ) -> None:
422 # Save configuration, set other variables default.
423 self.logger = logging.getLogger(__name__)
424 self._config = config
425 self._in_memory = True
426 self._bytes_written = 0
427 self._fileobj: BytesIO | BufferedRandom = BytesIO()
429 # Save the provided field/file name and content type.
430 self._field_name = field_name
431 self._file_name = file_name
432 self._content_type = content_type
434 # Our actual file name is None by default, since, depending on our
435 # config, we may not actually use the provided name.
436 self._actual_file_name: bytes | None = None
438 # Nameless uploads (e.g. `application/octet-stream`) have no base or extension.
439 self._file_base: bytes | None = None
440 self._ext = b""
441 if file_name is not None:
442 # Extract just the basename to avoid directory traversal
443 basename = os.path.basename(file_name)
444 base, ext = os.path.splitext(basename)
445 self._file_base = base
446 self._ext = ext
448 @property
449 def field_name(self) -> bytes | None:
450 """The form field associated with this file. May be None if there isn't
451 one, for example when we have an application/octet-stream upload.
452 """
453 return self._field_name
455 @property
456 def file_name(self) -> bytes | None:
457 """The file name given in the upload request."""
458 return self._file_name
460 @property
461 def actual_file_name(self) -> bytes | None:
462 """The file name that this file is saved as. Will be None if it's not
463 currently saved on disk.
464 """
465 return self._actual_file_name
467 @property
468 def file_object(self) -> BytesIO | BufferedRandom:
469 """The file object that we're currently writing to. Note that this
470 will either be an instance of a :class:`io.BytesIO`, or a regular file
471 object.
472 """
473 return self._fileobj
475 @property
476 def size(self) -> int:
477 """The total size of this file, counted as the number of bytes that
478 currently have been written to the file.
479 """
480 return self._bytes_written
482 @property
483 def in_memory(self) -> bool:
484 """A boolean representing whether or not this file object is currently
485 stored in-memory or on-disk.
486 """
487 return self._in_memory
489 @property
490 def content_type(self) -> str | None:
491 """The Content-Type value for this part, if it was set."""
492 return self._content_type
494 def flush_to_disk(self) -> None:
495 """If the file is already on-disk, do nothing. Otherwise, copy from
496 the in-memory buffer to a disk file, and then reassign our internal
497 file object to this new disk file.
499 Note that if you attempt to flush a file that is already on-disk, a
500 warning will be logged to this module's logger.
501 """
502 if not self._in_memory:
503 self.logger.warning("Trying to flush to disk when we're not in memory")
504 return
506 # Go back to the start of our file.
507 self._fileobj.seek(0)
509 # Open a new file.
510 new_file = self._get_disk_file()
512 # Copy the file objects.
513 shutil.copyfileobj(self._fileobj, new_file)
515 # Seek to the new position in our new file.
516 new_file.seek(self._bytes_written)
518 # Reassign the fileobject.
519 old_fileobj = self._fileobj
520 self._fileobj = new_file
522 # We're no longer in memory.
523 self._in_memory = False
525 # Close the old file object.
526 old_fileobj.close()
528 def _get_disk_file(self) -> BufferedRandom:
529 """This function is responsible for getting a file object on-disk for us."""
530 self.logger.info("Opening a file on disk")
532 file_dir = self._config.get("UPLOAD_DIR")
533 keep_filename = self._config.get("UPLOAD_KEEP_FILENAME", False)
534 keep_extensions = self._config.get("UPLOAD_KEEP_EXTENSIONS", False)
535 delete_tmp = self._config.get("UPLOAD_DELETE_TMP", True)
536 tmp_file: None | BufferedRandom = None
538 # If we have a directory and a filename to keep...
539 if file_dir is not None and keep_filename and self._file_base is not None:
540 self.logger.info("Saving with filename in: %r", file_dir)
542 # Build our filename.
543 fname = self._file_base + self._ext if keep_extensions else self._file_base
545 path = os.path.join(os.fsencode(file_dir), fname)
546 try:
547 self.logger.info("Opening file: %r", path)
548 tmp_file = open(path, "w+b")
549 except OSError:
550 tmp_file = None
552 self.logger.exception("Error opening temporary file")
553 raise FileError("Error opening temporary file: %r" % path)
554 else:
555 # Build options array.
556 # Note that on Python 3, tempfile doesn't support byte names. We
557 # encode our paths using the default filesystem encoding.
558 suffix = self._ext.decode(sys.getfilesystemencoding()) if keep_extensions else None
560 if file_dir is None:
561 dir = None
562 elif isinstance(file_dir, bytes):
563 dir = file_dir.decode(sys.getfilesystemencoding())
564 else:
565 dir = file_dir # pragma: no cover
567 # Create a temporary (named) file with the appropriate settings.
568 self.logger.info(
569 "Creating a temporary file with options: %r", {"suffix": suffix, "delete": delete_tmp, "dir": dir}
570 )
571 try:
572 tmp_file = cast(BufferedRandom, tempfile.NamedTemporaryFile(suffix=suffix, delete=delete_tmp, dir=dir))
573 except OSError:
574 self.logger.exception("Error creating named temporary file")
575 raise FileError("Error creating named temporary file")
577 assert tmp_file is not None
578 # Encode filename as bytes.
579 if isinstance(tmp_file.name, str):
580 fname = tmp_file.name.encode(sys.getfilesystemencoding())
581 else:
582 fname = cast(bytes, tmp_file.name) # pragma: no cover
584 self._actual_file_name = fname
585 return tmp_file
587 def write(self, data: bytes) -> int:
588 """Write some data to the File.
590 :param data: a bytestring
591 """
592 return self.on_data(data)
594 def on_data(self, data: bytes) -> int:
595 """This method is a callback that will be called whenever data is
596 written to the File.
598 Args:
599 data: The data to write to the file.
601 Returns:
602 The number of bytes written.
603 """
604 bwritten = self._fileobj.write(data)
606 # If the bytes written isn't the same as the length, just return.
607 if bwritten != len(data):
608 self.logger.warning("bwritten != len(data) (%d != %d)", bwritten, len(data))
609 return bwritten
611 # Keep track of how many bytes we've written.
612 self._bytes_written += bwritten
614 # If we're in-memory and are over our limit, we create a file.
615 max_memory_file_size = self._config.get("MAX_MEMORY_FILE_SIZE")
616 if self._in_memory and max_memory_file_size is not None and (self._bytes_written > max_memory_file_size):
617 self.logger.info("Flushing to disk")
618 self.flush_to_disk()
620 # Return the number of bytes written.
621 return bwritten
623 def on_end(self) -> None:
624 """This method is called whenever the Field is finalized."""
625 # Flush the underlying file object
626 self._fileobj.flush()
628 def finalize(self) -> None:
629 """Finalize the form file. This will not close the underlying file,
630 but simply signal that we are finished writing to the File.
631 """
632 self.on_end()
634 def close(self) -> None:
635 """Close the File object. This will actually close the underlying
636 file object (whether it's a :class:`io.BytesIO` or an actual file
637 object).
638 """
639 self._fileobj.close()
641 def __repr__(self) -> str:
642 return f"{self.__class__.__name__}(file_name={self.file_name!r}, field_name={self.field_name!r})"
645class BaseParser:
646 """This class is the base class for all parsers. It contains the logic for
647 calling and adding callbacks.
649 A callback can be one of two different forms. "Notification callbacks" are
650 callbacks that are called when something happens - for example, when a new
651 part of a multipart message is encountered by the parser. "Data callbacks"
652 are called when we get some sort of data - for example, part of the body of
653 a multipart chunk. Notification callbacks are called with no parameters,
654 whereas data callbacks are called with three, as follows::
656 data_callback(data, start, end)
658 The "data" parameter is a bytestring (i.e. "foo" on Python 2, or b"foo" on
659 Python 3). "start" and "end" are integer indexes into the "data" string
660 that represent the data of interest. Thus, in a data callback, the slice
661 `data[start:end]` represents the data that the callback is "interested in".
662 The callback is not passed a copy of the data, since copying severely hurts
663 performance.
664 """
666 def __init__(self) -> None:
667 self.logger = logging.getLogger(__name__)
668 self.callbacks: QuerystringCallbacks | OctetStreamCallbacks | MultipartCallbacks = {}
670 def callback(
671 self, name: CallbackName, data: bytes | None = None, start: int | None = None, end: int | None = None
672 ) -> None:
673 """This function calls a provided callback with some data. If the
674 callback is not set, will do nothing.
676 Args:
677 name: The name of the callback to call (as a string).
678 data: Data to pass to the callback. If None, then it is assumed that the callback is a notification
679 callback, and no parameters are given.
680 end: An integer that is passed to the data callback.
681 start: An integer that is passed to the data callback.
682 """
683 func = self.callbacks.get("on_" + name)
684 if func is None:
685 return
686 func = cast("Callable[..., Any]", func)
687 # Depending on whether we're given a buffer...
688 if data is not None:
689 # Don't do anything if we have start == end.
690 if start is not None and start == end:
691 return
692 func(data, start, end)
693 else:
694 func()
696 def set_callback(self, name: CallbackName, new_func: Callable[..., Any] | None) -> None:
697 """Update the function for a callback. Removes from the callbacks dict
698 if new_func is None.
700 :param name: The name of the callback to call (as a string).
702 :param new_func: The new function for the callback. If None, then the
703 callback will be removed (with no error if it does not
704 exist).
705 """
706 if new_func is None:
707 self.callbacks.pop("on_" + name, None) # type: ignore[misc]
708 else:
709 self.callbacks["on_" + name] = new_func # type: ignore[literal-required]
711 def close(self) -> None:
712 pass # pragma: no cover
714 def finalize(self) -> None:
715 pass # pragma: no cover
717 def __repr__(self) -> str:
718 return "%s()" % self.__class__.__name__
721class OctetStreamParser(BaseParser):
722 """This parser parses an octet-stream request body and calls callbacks when
723 incoming data is received. Callbacks are as follows:
725 | Callback Name | Parameters | Description |
726 |----------------|-----------------|-----------------------------------------------------|
727 | on_start | None | Called when the first data is parsed. |
728 | on_data | data, start, end| Called for each data chunk that is parsed. |
729 | on_end | None | Called when the parser is finished parsing all data.|
731 Args:
732 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
733 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
734 """
736 def __init__(self, callbacks: OctetStreamCallbacks = {}, max_size: float = float("inf")):
737 super().__init__()
738 self.callbacks = callbacks
739 self._started = False
741 if not isinstance(max_size, Number) or max_size < 1:
742 raise ValueError("max_size must be a positive number, not %r" % max_size)
743 self.max_size: int | float = max_size
744 self._current_size = 0
746 def write(self, data: bytes) -> int:
747 """Write some data to the parser, which will perform size verification,
748 and then pass the data to the underlying callback.
750 Args:
751 data: The data to write to the parser.
753 Returns:
754 The number of bytes written.
755 """
756 if not self._started:
757 self.callback("start")
758 self._started = True
760 # Truncate data length.
761 data_len = len(data)
762 if (self._current_size + data_len) > self.max_size:
763 # We truncate the length of data that we are to process.
764 new_size = int(self.max_size - self._current_size)
765 self.logger.warning(
766 "Current size is %d (max %d), so truncating data length from %d to %d",
767 self._current_size,
768 self.max_size,
769 data_len,
770 new_size,
771 )
772 data_len = new_size
774 # Increment size, then callback, in case there's an exception.
775 self._current_size += data_len
776 self.callback("data", data, 0, data_len)
777 return data_len
779 def finalize(self) -> None:
780 """Finalize this parser, which signals to that we are finished parsing,
781 and sends the on_end callback.
782 """
783 self.callback("end")
785 def __repr__(self) -> str:
786 return "%s()" % self.__class__.__name__
789class QuerystringParser(BaseParser):
790 """This is a streaming querystring parser. It will consume data, and call
791 the callbacks given when it has data.
793 | Callback Name | Parameters | Description |
794 |----------------|-----------------|-----------------------------------------------------|
795 | on_field_start | None | Called when a new field is encountered. |
796 | on_field_name | data, start, end| Called when a portion of a field's name is encountered. |
797 | on_field_data | data, start, end| Called when a portion of a field's data is encountered. |
798 | on_field_end | None | Called when the end of a field is encountered. |
799 | on_end | None | Called when the parser is finished parsing all data.|
801 Args:
802 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
803 strict_parsing: Whether or not to parse the body strictly. Defaults to False. If this is set to True, then the
804 behavior of the parser changes as the following: if a field has a value with an equal sign
805 (e.g. "foo=bar", or "foo="), it is always included. If a field has no equals sign (e.g. "...&name&..."),
806 it will be treated as an error if 'strict_parsing' is True, otherwise included. If an error is encountered,
807 then a [`QuerystringParseError`][python_multipart.exceptions.QuerystringParseError] will be raised.
808 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
809 """ # noqa: E501
811 state: QuerystringState
813 def __init__(
814 self, callbacks: QuerystringCallbacks = {}, strict_parsing: bool = False, max_size: float = float("inf")
815 ) -> None:
816 super().__init__()
817 self.state = BEFORE_FIELD
818 self._found_sep = False
820 self.callbacks = callbacks
822 # Max-size stuff
823 if not isinstance(max_size, Number) or max_size < 1:
824 raise ValueError("max_size must be a positive number, not %r" % max_size)
825 self.max_size: int | float = max_size
826 self._current_size = 0
828 # Should parsing be strict?
829 self.strict_parsing = strict_parsing
831 def write(self, data: bytes) -> int:
832 """Write some data to the parser, which will perform size verification,
833 parse into either a field name or value, and then pass the
834 corresponding data to the underlying callback. If an error is
835 encountered while parsing, a QuerystringParseError will be raised. The
836 "offset" attribute of the raised exception will be set to the offset in
837 the input data chunk (NOT the overall stream) that caused the error.
839 Args:
840 data: The data to write to the parser.
842 Returns:
843 The number of bytes written.
844 """
845 # Handle sizing.
846 data_len = len(data)
847 if (self._current_size + data_len) > self.max_size:
848 # We truncate the length of data that we are to process.
849 new_size = int(self.max_size - self._current_size)
850 self.logger.warning(
851 "Current size is %d (max %d), so truncating data length from %d to %d",
852 self._current_size,
853 self.max_size,
854 data_len,
855 new_size,
856 )
857 data_len = new_size
859 l = 0
860 try:
861 l = self._internal_write(data, data_len)
862 finally:
863 self._current_size += l
865 return l
867 def _internal_write(self, data: bytes, length: int) -> int:
868 state = self.state
869 strict_parsing = self.strict_parsing
870 found_sep = self._found_sep
871 callbacks = cast("QuerystringCallbacks", self.callbacks)
872 on_field_start = callbacks.get("on_field_start")
873 on_field_name = callbacks.get("on_field_name")
874 on_field_data = callbacks.get("on_field_data")
875 on_field_end = callbacks.get("on_field_end")
876 if on_field_start is None:
877 on_field_start = _noop_event
878 if on_field_name is None:
879 on_field_name = _noop_data
880 if on_field_data is None:
881 on_field_data = _noop_data
882 if on_field_end is None:
883 on_field_end = _noop_event
885 i = 0
886 while i < length:
887 ch = data[i]
889 # Depending on our state...
890 if state == BEFORE_FIELD:
891 # If the 'found_sep' flag is set, we've already encountered
892 # and skipped a single separator. If so, we check our strict
893 # parsing flag and decide what to do. Otherwise, we haven't
894 # yet reached a separator, and thus, if we do, we need to skip
895 # it as it will be the boundary between fields that's supposed
896 # to be there.
897 if ch == AMPERSAND:
898 if found_sep:
899 # If we're parsing strictly, we disallow blank chunks.
900 if strict_parsing:
901 raise QuerystringParseError("Skipping duplicate ampersand at %d" % i, offset=i)
902 else:
903 self.logger.debug("Skipping duplicate ampersand at %d", i)
904 else:
905 # This case is when we're skipping the (first)
906 # separator between fields, so we just set our flag
907 # and continue on.
908 found_sep = True
909 else:
910 # Emit a field-start event, and go to that state. Also,
911 # reset the "found_sep" flag, for the next time we get to
912 # this state.
913 on_field_start()
914 i -= 1
915 state = FIELD_NAME
916 found_sep = False
918 elif state == FIELD_NAME:
919 # Try and find a separator - we ensure that, if we do, we only
920 # look for the equal sign before it.
921 sep_pos = data.find(b"&", i, length)
923 # See if we can find an equals sign in the remaining data. If
924 # so, we can immediately emit the field name and jump to the
925 # data state.
926 if sep_pos != -1:
927 equals_pos = data.find(b"=", i, sep_pos)
928 else:
929 equals_pos = data.find(b"=", i, length)
931 if equals_pos != -1:
932 # Emit this name.
933 if i != equals_pos:
934 on_field_name(data, i, equals_pos)
936 # Jump i to this position. Note that it will then have 1
937 # added to it below, which means the next iteration of this
938 # loop will inspect the character after the equals sign.
939 i = equals_pos
940 state = FIELD_DATA
941 else:
942 # No equals sign found.
943 if not strict_parsing:
944 # See also comments in the QuerystringState.FIELD_DATA case below.
945 # If we found the separator, we emit the name and just
946 # end - there's no data callback at all (not even with
947 # a blank value).
948 if sep_pos != -1:
949 if i != sep_pos:
950 on_field_name(data, i, sep_pos)
951 on_field_end()
953 i = sep_pos - 1
954 state = BEFORE_FIELD
955 else:
956 # Otherwise, no separator in this block, so the
957 # rest of this chunk must be a name.
958 if i != length:
959 on_field_name(data, i, length)
960 i = length
962 else:
963 # We're parsing strictly. If we find a separator,
964 # this is an error - we require an equals sign.
965 if sep_pos != -1:
966 raise QuerystringParseError(
967 "When strict_parsing is True, we require an "
968 "equals sign in all field chunks. Did not "
969 "find one in the chunk that starts at %d" % (i,),
970 offset=i,
971 )
973 # No separator in the rest of this chunk, so it's just
974 # a field name.
975 if i != length:
976 on_field_name(data, i, length)
977 i = length
979 elif state == FIELD_DATA:
980 # Try finding an ampersand after this position.
981 sep_pos = data.find(b"&", i, length)
983 # If we found it, callback this bit as data and then go back
984 # to expecting to find a field.
985 if sep_pos != -1:
986 if i != sep_pos:
987 on_field_data(data, i, sep_pos)
988 on_field_end()
990 # Note that we go to the separator, which brings us to the
991 # "before field" state. This allows us to properly emit
992 # "field_start" events only when we actually have data for
993 # a field of some sort.
994 i = sep_pos - 1
995 state = BEFORE_FIELD
997 # Otherwise, emit the rest as data and finish.
998 else:
999 if i != length:
1000 on_field_data(data, i, length)
1001 i = length
1003 else: # pragma: no cover (error case)
1004 msg = "Reached an unknown state %d at %d" % (state, i)
1005 self.logger.warning(msg)
1006 raise QuerystringParseError(msg, offset=i)
1008 i += 1
1010 self.state = state
1011 self._found_sep = found_sep
1012 return length
1014 def finalize(self) -> None:
1015 """Finalize this parser, which signals to that we are finished parsing,
1016 if we're still in the middle of a field, an on_field_end callback, and
1017 then the on_end callback.
1018 """
1019 callbacks = cast("QuerystringCallbacks", self.callbacks)
1020 # If we're currently in the middle of a field, we finish it.
1021 if self.state in (FIELD_DATA, FIELD_NAME):
1022 on_field_end = callbacks.get("on_field_end")
1023 if on_field_end is None:
1024 on_field_end = _noop_event
1025 on_field_end()
1026 on_end = callbacks.get("on_end")
1027 if on_end is None:
1028 on_end = _noop_event
1029 on_end()
1031 def __repr__(self) -> str:
1032 return "{}(strict_parsing={!r}, max_size={!r})".format(
1033 self.__class__.__name__, self.strict_parsing, self.max_size
1034 )
1037class MultipartParser(BaseParser):
1038 """This class is a streaming multipart/form-data parser.
1040 | Callback Name | Parameters | Description |
1041 |--------------------|-----------------|-------------|
1042 | on_part_begin | None | Called when a new part of the multipart message is encountered. |
1043 | on_part_data | data, start, end| Called when a portion of a part's data is encountered. |
1044 | on_part_end | None | Called when the end of a part is reached. |
1045 | on_header_begin | None | Called when we've found a new header in a part of a multipart message |
1046 | on_header_field | data, start, end| Called each time an additional portion of a header is read (i.e. the part of the header that is before the colon; the "Foo" in "Foo: Bar"). |
1047 | on_header_value | data, start, end| Called when we get data for a header. |
1048 | on_header_end | None | Called when the current header is finished - i.e. we've reached the newline at the end of the header. |
1049 | on_headers_finished| None | Called when all headers are finished, and before the part data starts. |
1050 | on_end | None | Called when the parser is finished parsing all data. |
1052 Args:
1053 boundary: The multipart boundary. This is required, and must match what is given in the HTTP request - usually in the Content-Type header.
1054 callbacks: A dictionary of callbacks. See the documentation for [`BaseParser`][python_multipart.BaseParser].
1055 max_size: The maximum size of body to parse. Defaults to infinity - i.e. unbounded.
1056 max_header_count: The maximum number of headers allowed per part.
1057 max_header_size: The maximum size of a single header line (excluding the trailing CRLF).
1058 """ # noqa: E501
1060 def __init__(
1061 self,
1062 boundary: bytes | str,
1063 callbacks: MultipartCallbacks = {},
1064 max_size: float = float("inf"),
1065 *,
1066 max_header_count: int = DEFAULT_MAX_HEADER_COUNT,
1067 max_header_size: int = DEFAULT_MAX_HEADER_SIZE,
1068 ) -> None:
1069 # Initialize parser state.
1070 super().__init__()
1071 self.state = START
1072 self.index = self.flags = 0
1074 self.callbacks = callbacks
1076 if not isinstance(max_size, Number) or max_size < 1:
1077 raise ValueError("max_size must be a positive number, not %r" % max_size)
1078 self.max_size = max_size
1079 self._current_size = 0
1081 self.max_header_count = max_header_count
1082 self._current_header_count = 0
1084 self.max_header_size = max_header_size
1085 self._current_header_size = 0
1087 # Setup marks. These are used to track the state of data received.
1088 self.marks: dict[str, int] = {}
1090 # Save our boundary.
1091 if isinstance(boundary, str): # pragma: no cover
1092 boundary = boundary.encode("latin-1")
1093 if len(boundary) > MAX_BOUNDARY_LENGTH:
1094 raise FormParserError(f"Boundary length {len(boundary)} exceeds maximum of {MAX_BOUNDARY_LENGTH}")
1095 self.boundary = b"\r\n--" + boundary
1097 def write(self, data: bytes) -> int:
1098 """Write some data to the parser, which will perform size verification,
1099 and then parse the data into the appropriate location (e.g. header,
1100 data, etc.), and pass this on to the underlying callback. If an error
1101 is encountered, a MultipartParseError will be raised. The "offset"
1102 attribute on the raised exception will be set to the offset of the byte
1103 in the input chunk that caused the error.
1105 Args:
1106 data: The data to write to the parser.
1108 Returns:
1109 The number of bytes written.
1110 """
1111 # Handle sizing.
1112 data_len = len(data)
1113 if (self._current_size + data_len) > self.max_size:
1114 # We truncate the length of data that we are to process.
1115 new_size = int(self.max_size - self._current_size)
1116 self.logger.warning(
1117 "Current size is %d (max %d), so truncating data length from %d to %d",
1118 self._current_size,
1119 self.max_size,
1120 data_len,
1121 new_size,
1122 )
1123 data_len = new_size
1125 l = 0
1126 try:
1127 l = self._internal_write(data, data_len)
1128 finally:
1129 self._current_size += l
1131 return l
1133 def _internal_write(self, data: bytes, length: int) -> int:
1134 # Get values from locals.
1135 boundary = self.boundary
1136 boundary_length = len(boundary)
1138 # Get our state, flags and index. These are persisted between calls to
1139 # this function.
1140 state = self.state
1141 index = self.index
1142 flags = self.flags
1143 current_header_count = self._current_header_count
1144 current_header_size = self._current_header_size
1146 # Our index defaults to 0.
1147 i = 0
1149 def advance_header_size(amount: int = 1) -> None:
1150 nonlocal current_header_size
1151 current_header_size += amount
1152 if current_header_size > self.max_header_size:
1153 raise MultipartParseError("Maximum header size exceeded", offset=i)
1155 # Set a mark.
1156 def set_mark(name: str) -> None:
1157 self.marks[name] = i
1159 # Remove a mark.
1160 def delete_mark(name: str, reset: bool = False) -> None:
1161 self.marks.pop(name, None)
1163 # Helper function that makes calling a callback with data easier. The
1164 # 'remaining' parameter will callback from the marked value until the
1165 # end of the buffer, and reset the mark, instead of deleting it. This
1166 # is used at the end of the function to call our callbacks with any
1167 # remaining data in this chunk.
1168 def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None:
1169 marked_index = self.marks.get(name)
1170 if marked_index is None:
1171 return
1173 # Otherwise, we call it from the mark to the current byte we're
1174 # processing.
1175 if end_i <= marked_index:
1176 # There is no additional data to send.
1177 pass
1178 elif marked_index >= 0:
1179 # We are emitting data from the local buffer.
1180 self.callback(name, data, marked_index, end_i)
1181 else:
1182 # Some of the data comes from a partial boundary match.
1183 # and requires look-behind.
1184 # We need to use self.flags (and not flags) because we care about
1185 # the state when we entered the loop.
1186 lookbehind_len = -marked_index
1187 if lookbehind_len <= boundary_length:
1188 self.callback(name, boundary, 0, lookbehind_len)
1189 elif self.flags & FLAG_PART_BOUNDARY:
1190 lookback = boundary + b"\r\n"
1191 self.callback(name, lookback, 0, lookbehind_len)
1192 elif self.flags & FLAG_LAST_BOUNDARY:
1193 lookback = boundary + b"--\r\n"
1194 self.callback(name, lookback, 0, lookbehind_len)
1195 else: # pragma: no cover (error case)
1196 self.logger.warning("Look-back buffer error")
1198 if end_i > 0:
1199 self.callback(name, data, 0, end_i)
1200 # If we're getting remaining data, we have got all the data we
1201 # can be certain is not a boundary, leaving only a partial boundary match.
1202 if remaining:
1203 self.marks[name] = end_i - length
1204 else:
1205 self.marks.pop(name, None)
1207 # For each byte...
1208 while i < length:
1209 c = data[i]
1211 if state == START:
1212 # Skip leading newlines
1213 if c == CR or c == LF:
1214 i = data.find(b"-", i)
1215 if i == -1:
1216 # No boundary candidate in this chunk, so ignore the content after the leading CR/LF.
1217 i = length
1218 break
1219 continue
1221 # index is used as in index into our boundary. Set to 0.
1222 index = 0
1224 # Move to the next state, but decrement i so that we re-process
1225 # this character.
1226 state = START_BOUNDARY
1227 i -= 1
1229 elif state == START_BOUNDARY:
1230 if index == 0 and data.startswith(boundary[2:], i, length):
1231 index = boundary_length - 2
1232 i += index
1233 continue
1235 # Check to ensure that the last 2 characters in our boundary
1236 # are CRLF.
1237 if index == boundary_length - 2:
1238 if c == HYPHEN:
1239 # Potential empty message.
1240 state = END_BOUNDARY
1241 elif c != CR:
1242 # Error!
1243 msg = "Did not find CR at end of boundary (%d)" % (i,)
1244 self.logger.warning(msg)
1245 raise MultipartParseError(msg, offset=i)
1247 index += 1
1249 elif index == boundary_length - 1:
1250 if c != LF:
1251 msg = "Did not find LF at end of boundary (%d)" % (i,)
1252 self.logger.warning(msg)
1253 raise MultipartParseError(msg, offset=i)
1255 # The index is now used for indexing into our boundary.
1256 index = 0
1258 # Callback for the start of a part.
1259 self.callback("part_begin")
1260 current_header_count = 0
1261 current_header_size = 0
1263 # Move to the next character and state.
1264 state = HEADER_FIELD_START
1266 else:
1267 # Check to ensure our boundary matches
1268 if c != boundary[index + 2]:
1269 msg = "Expected boundary character %r, got %r at index %d" % (boundary[index + 2], c, index + 2)
1270 self.logger.warning(msg)
1271 raise MultipartParseError(msg, offset=i)
1273 # Increment index into boundary and continue.
1274 index += 1
1276 elif state == HEADER_FIELD_START:
1277 # Mark the start of a header field here, reset the index, and
1278 # continue parsing our header field.
1279 index = 0
1281 if c != CR:
1282 current_header_count += 1
1283 if current_header_count > self.max_header_count:
1284 raise MultipartParseError("Maximum header count exceeded", offset=i)
1285 current_header_size = 0
1287 # Set a mark of our header field.
1288 set_mark("header_field")
1290 # Notify that we're starting a header if the next character is
1291 # not a CR; a CR at the beginning of the header will cause us
1292 # to stop parsing headers in the MultipartState.HEADER_FIELD state,
1293 # below.
1294 if c != CR:
1295 self.callback("header_begin")
1297 # Move to parsing header fields.
1298 state = HEADER_FIELD
1299 i -= 1
1301 elif state == HEADER_FIELD:
1302 # If we've reached a CR at the beginning of a header, it means
1303 # that we've reached the second of 2 newlines, and so there are
1304 # no more headers to parse.
1305 if c == CR and index == 0:
1306 delete_mark("header_field")
1307 state = HEADERS_ALMOST_DONE
1308 i += 1
1309 continue
1311 # The field name runs until the colon; jump straight to it and
1312 # validate the whole span at once instead of byte by byte.
1313 colon = data.find(b":", i, length)
1314 end = colon if colon != -1 else length
1316 # Enforce the size limit before slicing and validating, so an oversized header
1317 # name fails fast instead of copying and scanning a potentially huge span.
1318 advance_header_size(end - i if colon == -1 else end - i + 1)
1320 field = data[i:end]
1321 if field.translate(None, TOKEN_CHARS):
1322 bad = next(b for b in field if b not in TOKEN_CHARS_SET)
1323 bad_i = i + field.index(bad)
1324 msg = "Found invalid character %r in header at %d" % (bad, bad_i)
1325 self.logger.warning(msg)
1326 raise MultipartParseError(msg, offset=bad_i)
1328 index += end - i
1329 if colon == -1:
1330 # Field name continues into the next chunk.
1331 i = length
1332 else:
1333 # A 0-length header is an error.
1334 if index == 0:
1335 msg = "Found 0-length header at %d" % (i,)
1336 self.logger.warning(msg)
1337 raise MultipartParseError(msg, offset=i)
1339 # Call our callback with the header field.
1340 i = colon
1341 data_callback("header_field", i)
1343 # Move to parsing the header value.
1344 state = HEADER_VALUE_START
1346 elif state == HEADER_VALUE_START:
1347 # Skip leading spaces.
1348 if c == SPACE:
1349 advance_header_size()
1350 i += 1
1351 continue
1353 # Mark the start of the header value.
1354 set_mark("header_value")
1356 # Move to the header-value state, reprocessing this character.
1357 state = HEADER_VALUE
1358 i -= 1
1360 elif state == HEADER_VALUE:
1361 # The value runs until the terminating CR; jump straight to it
1362 # instead of inspecting every byte.
1363 cr = data.find(b"\r", i, length)
1364 end = cr if cr != -1 else length
1365 advance_header_size(end - i)
1366 if cr != -1:
1367 i = cr
1368 data_callback("header_value", i)
1369 self.callback("header_end")
1370 current_header_size = 0
1371 state = HEADER_VALUE_ALMOST_DONE
1372 else:
1373 i = length
1375 elif state == HEADER_VALUE_ALMOST_DONE:
1376 # The last character should be a LF. If not, it's an error.
1377 if c != LF:
1378 msg = f"Did not find LF character at end of header (found {c!r})"
1379 self.logger.warning(msg)
1380 raise MultipartParseError(msg, offset=i)
1382 # Move back to the start of another header. Note that if that
1383 # state detects ANOTHER newline, it'll trigger the end of our
1384 # headers.
1385 state = HEADER_FIELD_START
1387 elif state == HEADERS_ALMOST_DONE:
1388 # We're almost done our headers. This is reached when we parse
1389 # a CR at the beginning of a header, so our next character
1390 # should be a LF, or it's an error.
1391 if c != LF:
1392 msg = f"Did not find LF at end of headers (found {c!r})"
1393 self.logger.warning(msg)
1394 raise MultipartParseError(msg, offset=i)
1396 self.callback("headers_finished")
1397 state = PART_DATA_START
1399 elif state == PART_DATA_START:
1400 # Mark the start of our part data.
1401 set_mark("part_data")
1403 # Start processing part data, including this character.
1404 state = PART_DATA
1405 i -= 1
1407 elif state == PART_DATA:
1408 # We're processing our part data right now. During this, we
1409 # need to efficiently search for our boundary, since any data
1410 # on any number of lines can be a part of the current data.
1412 # Save the current value of our index. We use this in case we
1413 # find part of a boundary, but it doesn't match fully.
1414 prev_index = index
1416 # If our index is 0, we're starting a new part, so start our
1417 # search.
1418 if index == 0:
1419 # The most common case is likely to be that the whole
1420 # boundary is present in the buffer.
1421 # Calling `find` is much faster than iterating here.
1422 i0 = data.find(boundary, i, length)
1423 if i0 >= 0:
1424 # We matched the whole boundary string.
1425 index = boundary_length - 1
1426 i = i0 + boundary_length - 1
1427 c = data[i]
1428 else:
1429 # No whole boundary, but the tail may hold a partial one
1430 # that completes in the next chunk. Boundary starts with
1431 # CR, which an RFC boundary contains nowhere else, so the
1432 # last CR in the tail is the only candidate prefix start.
1433 k = data.rfind(boundary[:1], max(i, length - boundary_length + 1), length)
1434 if k != -1 and boundary.startswith(data[k:length]):
1435 index = length - k
1436 # Carry the partial via index; the end-of-chunk flush
1437 # emits the data before it and re-marks the lookbehind.
1438 i = length
1439 continue
1441 # Now, we have a couple of cases here. If our index is before
1442 # the end of the boundary...
1443 if index < boundary_length:
1444 # If the character matches...
1445 if boundary[index] == c:
1446 # The current character matches, so continue!
1447 index += 1
1448 else:
1449 index = 0
1451 # Our index is equal to the length of our boundary!
1452 elif index == boundary_length:
1453 # First we increment it.
1454 index += 1
1456 # Now, if we've reached a newline, we need to set this as
1457 # the potential end of our boundary.
1458 if c == CR:
1459 flags |= FLAG_PART_BOUNDARY
1461 # Otherwise, if this is a hyphen, we might be at the last
1462 # of all boundaries.
1463 elif c == HYPHEN:
1464 flags |= FLAG_LAST_BOUNDARY
1466 # Otherwise, we reset our index, since this isn't either a
1467 # newline or a hyphen.
1468 else:
1469 index = 0
1471 # Our index is right after the part boundary, which should be
1472 # a LF.
1473 elif index == boundary_length + 1:
1474 # If we're at a part boundary (i.e. we've seen a CR
1475 # character already)...
1476 if flags & FLAG_PART_BOUNDARY:
1477 # We need a LF character next.
1478 if c == LF:
1479 # Unset the part boundary flag.
1480 flags &= ~FLAG_PART_BOUNDARY
1482 # We have identified a boundary, callback for any data before it.
1483 data_callback("part_data", i - index)
1484 # Callback indicating that we've reached the end of
1485 # a part, and are starting a new one.
1486 self.callback("part_end")
1487 self.callback("part_begin")
1488 current_header_count = 0
1489 current_header_size = 0
1491 # Move to parsing new headers.
1492 index = 0
1493 state = HEADER_FIELD_START
1494 i += 1
1495 continue
1497 # We didn't find an LF character, so no match. Reset
1498 # our index and clear our flag.
1499 index = 0
1500 flags &= ~FLAG_PART_BOUNDARY
1502 # Otherwise, if we're at the last boundary (i.e. we've
1503 # seen a hyphen already)...
1504 elif flags & FLAG_LAST_BOUNDARY:
1505 # We need a second hyphen here.
1506 if c == HYPHEN:
1507 # We have identified a boundary, callback for any data before it.
1508 data_callback("part_data", i - index)
1509 # Callback to end the current part, and then the
1510 # message.
1511 self.callback("part_end")
1512 self.callback("end")
1513 state = END
1514 else:
1515 # No match, so reset index.
1516 index = 0
1518 # Otherwise, our index is 0. If the previous index is not, it
1519 # means we reset something, and we need to take the data we
1520 # thought was part of our boundary and send it along as actual
1521 # data.
1522 if index == 0 and prev_index > 0:
1523 # Overwrite our previous index.
1524 prev_index = 0
1526 # Re-consider the current character, since this could be
1527 # the start of the boundary itself.
1528 i -= 1
1530 elif state == END_BOUNDARY:
1531 if index == boundary_length - 1:
1532 if c != HYPHEN:
1533 msg = "Did not find - at end of boundary (%d)" % (i,)
1534 self.logger.warning(msg)
1535 raise MultipartParseError(msg, offset=i)
1536 index += 1
1537 self.callback("end")
1538 state = END
1540 elif state == END:
1541 # Silently discard any epilogue data (RFC 2046 section 5.1.1 allows a CRLF and optional
1542 # epilogue after the closing boundary). Django and Werkzeug do the same.
1543 i = length
1544 break
1546 else: # pragma: no cover (error case)
1547 # We got into a strange state somehow! Just stop processing.
1548 msg = "Reached an unknown state %d at %d" % (state, i)
1549 self.logger.warning(msg)
1550 raise MultipartParseError(msg, offset=i)
1552 # Move to the next byte.
1553 i += 1
1555 # We call our callbacks with any remaining data. Note that we pass
1556 # the 'remaining' flag, which sets the mark back to 0 instead of
1557 # deleting it, if it's found. This is because, if the mark is found
1558 # at this point, we assume that there's data for one of these things
1559 # that has been parsed, but not yet emitted. And, as such, it implies
1560 # that we haven't yet reached the end of this 'thing'. So, by setting
1561 # the mark to 0, we cause any data callbacks that take place in future
1562 # calls to this function to start from the beginning of that buffer.
1563 data_callback("header_field", length, True)
1564 data_callback("header_value", length, True)
1565 data_callback("part_data", length - index, True)
1567 # Save values to locals.
1568 self.state = state
1569 self.index = index
1570 self.flags = flags
1571 self._current_header_count = current_header_count
1572 self._current_header_size = current_header_size
1574 # Return our data length to indicate no errors, and that we processed
1575 # all of it.
1576 return length
1578 def finalize(self) -> None:
1579 """Finalize this parser, which signals to that we are finished parsing.
1581 Note: It does not currently, but in the future, it will verify that we
1582 are in the final state of the parser (i.e. the end of the multipart
1583 message is well-formed), and, if not, throw an error.
1584 """
1585 # TODO: verify that we're in the state MultipartState.END, otherwise throw an
1586 # error or otherwise state that we're not finished parsing.
1587 pass
1589 def __repr__(self) -> str:
1590 return f"{self.__class__.__name__}(boundary={self.boundary!r})"
1593class FormParser:
1594 """This class is the all-in-one form parser. Given all the information
1595 necessary to parse a form, it will instantiate the correct parser, create
1596 the proper :class:`Field` and :class:`File` classes to store the data that
1597 is parsed, and call the two given callbacks with each field and file as
1598 they become available.
1600 Args:
1601 content_type: The Content-Type of the incoming request. This is used to select the appropriate parser.
1602 on_field: The callback to call when a field has been parsed and is ready for usage. See above for parameters.
1603 on_file: The callback to call when a file has been parsed and is ready for usage. See above for parameters.
1604 on_end: An optional callback to call when all fields and files in a request has been parsed. Can be None.
1605 boundary: If the request is a multipart/form-data request, this should be the boundary of the request, as given
1606 in the Content-Type header, as a bytestring.
1607 file_name: If the request is of type application/octet-stream, then the body of the request will not contain any
1608 information about the uploaded file. In such cases, you can provide the file name of the uploaded file
1609 manually.
1610 config: Configuration to use for this FormParser. The default values are taken from the DEFAULT_CONFIG value,
1611 and then any keys present in this dictionary will overwrite the default values.
1612 """
1614 #: This is the default configuration for our form parser.
1615 #: Note: all file sizes should be in bytes.
1616 DEFAULT_CONFIG: FormParserConfig = {
1617 "MAX_BODY_SIZE": float("inf"),
1618 "MAX_FIELDS": float("inf"),
1619 "MAX_FIELD_SIZE": float("inf"),
1620 "MAX_HEADER_COUNT": DEFAULT_MAX_HEADER_COUNT,
1621 "MAX_HEADER_SIZE": DEFAULT_MAX_HEADER_SIZE,
1622 "MAX_MEMORY_FILE_SIZE": 1 * 1024 * 1024,
1623 "UPLOAD_DIR": None,
1624 "UPLOAD_DELETE_TMP": True,
1625 "UPLOAD_KEEP_FILENAME": False,
1626 "UPLOAD_KEEP_EXTENSIONS": False,
1627 # Error on invalid Content-Transfer-Encoding?
1628 "UPLOAD_ERROR_ON_BAD_CTE": False,
1629 }
1631 def __init__(
1632 self,
1633 content_type: str,
1634 on_field: Callable[[Field], None] | None,
1635 on_file: Callable[[File], None] | None,
1636 on_end: Callable[[], None] | None = None,
1637 boundary: bytes | str | None = None,
1638 file_name: bytes | None = None,
1639 config: dict[Any, Any] = {},
1640 ) -> None:
1641 self.logger = logging.getLogger(__name__)
1643 # Save variables.
1644 self.content_type = content_type
1645 self.boundary = boundary
1646 self.bytes_received = 0
1647 self.parser = None
1649 # Save callbacks.
1650 self.on_field = on_field
1651 self.on_file = on_file
1652 self.on_end = on_end
1654 # Set configuration options.
1655 self.config: FormParserConfig = self.DEFAULT_CONFIG.copy()
1656 self.config.update(config) # type: ignore[typeddict-item]
1658 parser: OctetStreamParser | MultipartParser | QuerystringParser | None = None
1660 # Depending on the Content-Type, we instantiate the correct parser.
1661 if content_type == "application/octet-stream":
1662 file: File | None = None
1664 def on_start() -> None:
1665 nonlocal file
1666 file = File(file_name, None, config=self.config)
1668 def on_data(data: bytes, start: int, end: int) -> None:
1669 nonlocal file
1670 assert file is not None
1671 file.write(data[start:end])
1673 def _on_end() -> None:
1674 nonlocal file
1675 assert file is not None
1676 # Finalize the file itself.
1677 file.finalize()
1679 # Call our callback.
1680 if on_file:
1681 on_file(file)
1683 # Call the on-end callback.
1684 if self.on_end is not None:
1685 self.on_end()
1687 # Instantiate an octet-stream parser
1688 parser = OctetStreamParser(
1689 callbacks={"on_start": on_start, "on_data": on_data, "on_end": _on_end},
1690 max_size=self.config["MAX_BODY_SIZE"],
1691 )
1693 elif content_type == "application/x-www-form-urlencoded" or content_type == "application/x-url-encoded":
1694 max_fields = self.config["MAX_FIELDS"]
1695 max_field_size = self.config["MAX_FIELD_SIZE"]
1696 for key, limit in (("MAX_FIELDS", max_fields), ("MAX_FIELD_SIZE", max_field_size)):
1697 if not isinstance(limit, (int, float)) or not limit >= 0:
1698 raise ValueError(f"{key} must be a non-negative number, not {limit!r}")
1700 name_buffer: list[bytes] = []
1701 field_count = 0
1702 field_size = 0
1703 f: Field | None = None
1705 def on_field_start() -> None:
1706 nonlocal field_count, field_size
1707 field_count += 1
1708 if field_count > max_fields:
1709 raise FormParserError(f"Too many fields. Maximum number of fields is {max_fields}.")
1710 field_size = 0
1712 def on_field_name(data: bytes, start: int, end: int) -> None:
1713 nonlocal field_size
1714 field_size += end - start
1715 if field_size > max_field_size:
1716 raise FormParserError(f"Field exceeded maximum size of {max_field_size} bytes.")
1717 name_buffer.append(data[start:end])
1719 def on_field_data(data: bytes, start: int, end: int) -> None:
1720 nonlocal f, field_size
1721 field_size += end - start
1722 if field_size > max_field_size:
1723 raise FormParserError(f"Field exceeded maximum size of {max_field_size} bytes.")
1724 if f is None:
1725 f = Field(b"".join(name_buffer))
1726 del name_buffer[:]
1727 f.write(data[start:end])
1729 def on_field_end() -> None:
1730 nonlocal f
1731 # Finalize and call callback.
1732 if f is None:
1733 # If we get here, it's because there was no field data.
1734 # We create a field, set it to None, and then continue.
1735 f = Field(b"".join(name_buffer))
1736 del name_buffer[:]
1737 f.set_none()
1739 f.finalize()
1740 if on_field:
1741 on_field(f)
1742 f = None
1744 def _on_end() -> None:
1745 if self.on_end is not None:
1746 self.on_end()
1748 # Instantiate parser.
1749 parser = QuerystringParser(
1750 callbacks={
1751 "on_field_start": on_field_start,
1752 "on_field_name": on_field_name,
1753 "on_field_data": on_field_data,
1754 "on_field_end": on_field_end,
1755 "on_end": _on_end,
1756 },
1757 max_size=self.config["MAX_BODY_SIZE"],
1758 )
1760 elif content_type == "multipart/form-data":
1761 if boundary is None:
1762 self.logger.error("No boundary given")
1763 raise FormParserError("No boundary given")
1765 header_name: list[bytes] = []
1766 header_value: list[bytes] = []
1767 headers: dict[bytes, bytes] = {}
1769 f_multi: File | Field | None = None
1770 writer: File | Field | Base64Decoder | QuotedPrintableDecoder | None = None
1771 is_file = False
1773 def on_part_begin() -> None:
1774 # Reset headers in case this isn't the first part.
1775 nonlocal headers
1776 headers = {}
1778 def on_part_data(data: bytes, start: int, end: int) -> None:
1779 nonlocal writer
1780 assert writer is not None
1781 writer.write(data[start:end])
1782 # TODO: check for error here.
1784 def on_part_end() -> None:
1785 assert f_multi is not None
1786 assert writer is not None
1787 writer.finalize()
1788 if is_file:
1789 if on_file:
1790 assert isinstance(f_multi, File)
1791 on_file(f_multi)
1792 else:
1793 if on_field:
1794 assert isinstance(f_multi, Field)
1795 on_field(f_multi)
1797 def on_header_field(data: bytes, start: int, end: int) -> None:
1798 header_name.append(data[start:end])
1800 def on_header_value(data: bytes, start: int, end: int) -> None:
1801 header_value.append(data[start:end])
1803 def on_header_end() -> None:
1804 headers[b"".join(header_name).lower()] = b"".join(header_value)
1805 del header_name[:]
1806 del header_value[:]
1808 def on_headers_finished() -> None:
1809 nonlocal is_file, f_multi, writer
1810 # Reset the 'is file' flag.
1811 is_file = False
1813 # Parse the content-disposition header.
1814 content_disp = headers.get(b"content-disposition")
1815 disp, options = parse_options_header(content_disp)
1817 # Get the field and filename.
1818 field_name = options.get(b"name")
1819 file_name = options.get(b"filename")
1820 # RFC 7578 §4.2: each part MUST have a Content-Disposition header with a "name" parameter.
1821 if field_name is None:
1822 raise FormParserError(f'Field name not found in Content-Disposition: "{content_disp!r}"')
1824 # Create the proper class.
1825 content_type_b = headers.get(b"content-type")
1826 content_type = content_type_b.decode("latin-1") if content_type_b is not None else None
1827 if file_name is None:
1828 f_multi = Field(field_name, content_type=content_type)
1829 else:
1830 f_multi = File(file_name, field_name, config=self.config, content_type=content_type)
1831 is_file = True
1833 # Parse the given Content-Transfer-Encoding to determine what
1834 # we need to do with the incoming data.
1835 # TODO: check that we properly handle 8bit / 7bit encoding.
1836 # RFC 2045 section 6.1: Content-Transfer-Encoding values are case-insensitive.
1837 # https://www.rfc-editor.org/rfc/rfc2045#section-6.1
1838 transfer_encoding = headers.get(b"content-transfer-encoding", b"7bit").lower()
1840 if transfer_encoding in (b"binary", b"8bit", b"7bit"):
1841 writer = f_multi
1843 elif transfer_encoding == b"base64":
1844 writer = Base64Decoder(f_multi)
1846 elif transfer_encoding == b"quoted-printable":
1847 writer = QuotedPrintableDecoder(f_multi)
1849 else:
1850 self.logger.warning("Unknown Content-Transfer-Encoding: %r", transfer_encoding)
1851 if self.config["UPLOAD_ERROR_ON_BAD_CTE"]:
1852 raise FormParserError(f'Unknown Content-Transfer-Encoding "{transfer_encoding!r}"')
1853 else:
1854 # If we aren't erroring, then we just treat this as an
1855 # unencoded Content-Transfer-Encoding.
1856 writer = f_multi
1858 def _on_end() -> None:
1859 if self.on_end is not None:
1860 self.on_end()
1862 # Instantiate a multipart parser.
1863 parser = MultipartParser(
1864 boundary,
1865 callbacks={
1866 "on_part_begin": on_part_begin,
1867 "on_part_data": on_part_data,
1868 "on_part_end": on_part_end,
1869 "on_header_field": on_header_field,
1870 "on_header_value": on_header_value,
1871 "on_header_end": on_header_end,
1872 "on_headers_finished": on_headers_finished,
1873 "on_end": _on_end,
1874 },
1875 max_size=self.config["MAX_BODY_SIZE"],
1876 max_header_count=self.config["MAX_HEADER_COUNT"],
1877 max_header_size=self.config["MAX_HEADER_SIZE"],
1878 )
1880 else:
1881 self.logger.warning("Unknown Content-Type: %r", content_type)
1882 raise FormParserError(f"Unknown Content-Type: {content_type}")
1884 self.parser = parser
1886 def write(self, data: bytes) -> int:
1887 """Write some data. The parser will forward this to the appropriate
1888 underlying parser.
1890 Args:
1891 data: The data to write.
1893 Returns:
1894 The number of bytes processed.
1895 """
1896 self.bytes_received += len(data)
1897 # TODO: check the parser's return value for errors?
1898 assert self.parser is not None
1899 return self.parser.write(data)
1901 def finalize(self) -> None:
1902 """Finalize the parser."""
1903 if self.parser is not None and hasattr(self.parser, "finalize"):
1904 self.parser.finalize()
1906 def close(self) -> None:
1907 """Close the parser."""
1908 if self.parser is not None and hasattr(self.parser, "close"):
1909 self.parser.close()
1911 def __repr__(self) -> str:
1912 return f"{self.__class__.__name__}(content_type={self.content_type!r}, parser={self.parser!r})"
1915def create_form_parser(
1916 headers: dict[str, bytes],
1917 on_field: Callable[[Field], None] | None,
1918 on_file: Callable[[File], None] | None,
1919 config: dict[Any, Any] = {},
1920) -> FormParser:
1921 """This function is a helper function to aid in creating a FormParser
1922 instances. Given a dictionary-like headers object, it will determine
1923 the correct information needed, instantiate a FormParser with the
1924 appropriate values and given callbacks, and then return the corresponding
1925 parser.
1927 Args:
1928 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1929 on_field: Callback to call with each parsed field.
1930 on_file: Callback to call with each parsed file.
1931 config: Configuration variables to pass to the FormParser.
1932 """
1933 content_type: str | bytes | None = headers.get("Content-Type")
1934 if content_type is None:
1935 logging.getLogger(__name__).warning("No Content-Type header given")
1936 raise ValueError("No Content-Type header given!")
1938 # Boundaries are optional (the FormParser will raise if one is needed
1939 # but not given).
1940 content_type, params = parse_options_header(content_type)
1941 boundary = params.get(b"boundary")
1943 # We need content_type to be a string, not a bytes object.
1944 content_type = content_type.decode("latin-1")
1946 # Instantiate a form parser.
1947 form_parser = FormParser(content_type, on_field, on_file, boundary=boundary, config=config)
1949 # Return our parser.
1950 return form_parser
1953def parse_form(
1954 headers: dict[str, bytes],
1955 input_stream: SupportsRead,
1956 on_field: Callable[[Field], None] | None,
1957 on_file: Callable[[File], None] | None,
1958 chunk_size: int = 1048576,
1959) -> None:
1960 """This function is useful if you just want to parse a request body,
1961 without too much work. Pass it a dictionary-like object of the request's
1962 headers, and a file-like object for the input stream, along with two
1963 callbacks that will get called whenever a field or file is parsed.
1965 Args:
1966 headers: A dictionary-like object of HTTP headers. The only required header is Content-Type.
1967 input_stream: A file-like object that represents the request body. The read() method must return bytestrings.
1968 on_field: Callback to call with each parsed field.
1969 on_file: Callback to call with each parsed file.
1970 chunk_size: The maximum size to read from the input stream and write to the parser at one time.
1971 Defaults to 1 MiB.
1972 """
1973 if chunk_size < 1:
1974 raise ValueError(f"chunk_size must be a positive number, not {chunk_size!r}")
1976 # Create our form parser.
1977 parser = create_form_parser(headers, on_field, on_file)
1979 # Read chunks of 1MiB and write to the parser, but never read more than
1980 # the given Content-Length, if any.
1981 content_length: int | float | bytes | None = headers.get("Content-Length")
1982 if content_length is not None:
1983 content_length = int(content_length)
1984 if content_length < 0:
1985 raise ValueError("Content-Length must be non-negative")
1986 else:
1987 content_length = float("inf")
1988 bytes_read = 0
1990 while True:
1991 # Read only up to the Content-Length given.
1992 max_readable = int(min(content_length - bytes_read, chunk_size))
1993 buff = input_stream.read(max_readable)
1995 # Write to the parser and update our length.
1996 parser.write(buff)
1997 bytes_read += len(buff)
1999 # If we get a buffer that's smaller than the size requested, or if we
2000 # have read up to our content length, we're done.
2001 if len(buff) != max_readable or bytes_read == content_length:
2002 break
2004 # Tell our parser that we're done writing data.
2005 parser.finalize()