Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/response.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import collections
4import io
5import json as _json
6import logging
7import socket
8import sys
9import typing
10import warnings
11import zlib
12from contextlib import contextmanager
13from http.client import HTTPMessage as _HttplibHTTPMessage
14from http.client import HTTPResponse as _HttplibHTTPResponse
15from socket import timeout as SocketTimeout
17if typing.TYPE_CHECKING:
18 from ._base_connection import BaseHTTPConnection
20try:
21 try:
22 import brotlicffi as brotli # type: ignore[import-not-found]
23 except ImportError:
24 import brotli # type: ignore[import-not-found]
25except ImportError:
26 brotli = None
28from . import util
29from ._base_connection import _TYPE_BODY
30from ._collections import HTTPHeaderDict
31from .connection import BaseSSLError, HTTPConnection, HTTPException
32from .exceptions import (
33 BodyNotHttplibCompatible,
34 DecodeError,
35 DependencyWarning,
36 HTTPError,
37 IncompleteRead,
38 InvalidChunkLength,
39 InvalidHeader,
40 ProtocolError,
41 ReadTimeoutError,
42 ResponseNotChunked,
43 SSLError,
44)
45from .util.response import is_fp_closed, is_response_to_head
46from .util.retry import Retry
48if typing.TYPE_CHECKING:
49 from .connectionpool import HTTPConnectionPool
51log = logging.getLogger(__name__)
53# Read in 64 KiB chunks
54_READ_CHUNK_SIZE = 2**16
57class ContentDecoder:
58 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
59 raise NotImplementedError()
61 @property
62 def has_unconsumed_tail(self) -> bool:
63 raise NotImplementedError()
65 def flush(self) -> bytes:
66 raise NotImplementedError()
69class DeflateDecoder(ContentDecoder):
70 def __init__(self) -> None:
71 self._first_try = True
72 self._first_try_data = b""
73 self._unfed_data = b""
74 self._obj = zlib.decompressobj()
76 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
77 data = self._unfed_data + data
78 self._unfed_data = b""
79 if not data and not self._obj.unconsumed_tail:
80 return data
81 original_max_length = max_length
82 if original_max_length < 0:
83 max_length = 0
84 elif original_max_length == 0:
85 # We should not pass 0 to the zlib decompressor because 0 is
86 # the default value that will make zlib decompress without a
87 # length limit.
88 # Data should be stored for subsequent calls.
89 self._unfed_data = data
90 return b""
92 # Subsequent calls always reuse `self._obj`. zlib requires
93 # passing the unconsumed tail if decompression is to continue.
94 if not self._first_try:
95 return self._obj.decompress(
96 self._obj.unconsumed_tail + data, max_length=max_length
97 )
99 # First call tries with RFC 1950 ZLIB format.
100 self._first_try_data += data
101 try:
102 decompressed = self._obj.decompress(data, max_length=max_length)
103 if decompressed:
104 self._first_try = False
105 self._first_try_data = b""
106 return decompressed
107 # On failure, it falls back to RFC 1951 DEFLATE format.
108 except zlib.error:
109 self._first_try = False
110 self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
111 try:
112 return self.decompress(
113 self._first_try_data, max_length=original_max_length
114 )
115 finally:
116 self._first_try_data = b""
118 @property
119 def has_unconsumed_tail(self) -> bool:
120 return bool(self._unfed_data) or (
121 bool(self._obj.unconsumed_tail) and not self._first_try
122 )
124 def flush(self) -> bytes:
125 return self._obj.flush()
128class GzipDecoderState:
129 FIRST_MEMBER = 0
130 OTHER_MEMBERS = 1
131 SWALLOW_DATA = 2
134class GzipDecoder(ContentDecoder):
135 def __init__(self) -> None:
136 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
137 self._state = GzipDecoderState.FIRST_MEMBER
138 self._unconsumed_tail = b""
140 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
141 ret = bytearray()
142 if self._state == GzipDecoderState.SWALLOW_DATA:
143 return bytes(ret)
145 if max_length == 0:
146 # We should not pass 0 to the zlib decompressor because 0 is
147 # the default value that will make zlib decompress without a
148 # length limit.
149 # Data should be stored for subsequent calls.
150 self._unconsumed_tail += data
151 return b""
153 # zlib requires passing the unconsumed tail to the subsequent
154 # call if decompression is to continue.
155 data = self._unconsumed_tail + data
156 if not data and self._obj.eof:
157 return bytes(ret)
159 while True:
160 try:
161 ret += self._obj.decompress(
162 data, max_length=max(max_length - len(ret), 0)
163 )
164 except zlib.error:
165 previous_state = self._state
166 # Ignore data after the first error
167 self._state = GzipDecoderState.SWALLOW_DATA
168 self._unconsumed_tail = b""
169 if previous_state == GzipDecoderState.OTHER_MEMBERS:
170 # Allow trailing garbage acceptable in other gzip clients
171 return bytes(ret)
172 raise
174 self._unconsumed_tail = data = (
175 self._obj.unconsumed_tail or self._obj.unused_data
176 )
177 if max_length > 0 and len(ret) >= max_length:
178 break
180 if not data:
181 return bytes(ret)
182 # When the end of a gzip member is reached, a new decompressor
183 # must be created for unused (possibly future) data.
184 if self._obj.eof:
185 self._state = GzipDecoderState.OTHER_MEMBERS
186 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
188 return bytes(ret)
190 @property
191 def has_unconsumed_tail(self) -> bool:
192 return bool(self._unconsumed_tail)
194 def flush(self) -> bytes:
195 return self._obj.flush()
198if brotli is not None:
200 class BrotliDecoder(ContentDecoder):
201 # Supports both 'brotlipy' and 'Brotli' packages
202 # since they share an import name. The top branches
203 # are for 'brotlipy' and bottom branches for 'Brotli'
204 def __init__(self) -> None:
205 self._obj = brotli.Decompressor()
206 if hasattr(self._obj, "decompress"):
207 setattr(self, "_decompress", self._obj.decompress)
208 else:
209 setattr(self, "_decompress", self._obj.process)
211 # Requires Brotli >= 1.2.0 for `output_buffer_limit`.
212 def _decompress(self, data: bytes, output_buffer_limit: int = -1) -> bytes:
213 raise NotImplementedError()
215 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
216 try:
217 if max_length > 0:
218 return self._decompress(data, output_buffer_limit=max_length)
219 else:
220 return self._decompress(data)
221 except TypeError:
222 # Fallback for Brotli/brotlicffi/brotlipy versions without
223 # the `output_buffer_limit` parameter.
224 warnings.warn(
225 "Brotli >= 1.2.0 is required to prevent decompression bombs.",
226 DependencyWarning,
227 )
228 return self._decompress(data)
230 @property
231 def has_unconsumed_tail(self) -> bool:
232 try:
233 return not self._obj.can_accept_more_data()
234 except AttributeError:
235 return False
237 def flush(self) -> bytes:
238 if hasattr(self._obj, "flush"):
239 return self._obj.flush() # type: ignore[no-any-return]
240 return b""
243try:
244 if sys.version_info >= (3, 14):
245 from compression import zstd
246 else:
247 from backports import zstd
248except ImportError:
249 HAS_ZSTD = False
250else:
251 HAS_ZSTD = True
253 class ZstdDecoder(ContentDecoder):
254 def __init__(self) -> None:
255 self._obj = zstd.ZstdDecompressor()
257 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
258 if not data and not self.has_unconsumed_tail:
259 return b""
260 if self._obj.eof:
261 data = self._obj.unused_data + data
262 self._obj = zstd.ZstdDecompressor()
263 part = self._obj.decompress(data, max_length=max_length)
264 length = len(part)
265 data_parts = [part]
266 # Every loop iteration is supposed to read data from a separate frame.
267 # The loop breaks when:
268 # - enough data is read;
269 # - no more unused data is available;
270 # - end of the last read frame has not been reached (i.e.,
271 # more data has to be fed).
272 while (
273 self._obj.eof
274 and self._obj.unused_data
275 and (max_length < 0 or length < max_length)
276 ):
277 unused_data = self._obj.unused_data
278 if not self._obj.needs_input:
279 self._obj = zstd.ZstdDecompressor()
280 part = self._obj.decompress(
281 unused_data,
282 max_length=(max_length - length) if max_length > 0 else -1,
283 )
284 if part_length := len(part):
285 data_parts.append(part)
286 length += part_length
287 elif self._obj.needs_input:
288 break
289 return b"".join(data_parts)
291 @property
292 def has_unconsumed_tail(self) -> bool:
293 return not (self._obj.needs_input or self._obj.eof) or bool(
294 self._obj.unused_data
295 )
297 def flush(self) -> bytes:
298 if not self._obj.eof:
299 raise DecodeError("Zstandard data is incomplete")
300 return b""
303class MultiDecoder(ContentDecoder):
304 """
305 From RFC7231:
306 If one or more encodings have been applied to a representation, the
307 sender that applied the encodings MUST generate a Content-Encoding
308 header field that lists the content codings in the order in which
309 they were applied.
310 """
312 # Maximum allowed number of chained HTTP encodings in the
313 # Content-Encoding header.
314 max_decode_links = 5
316 def __init__(self, modes: str) -> None:
317 encodings = [m.strip() for m in modes.split(",")]
318 if len(encodings) > self.max_decode_links:
319 raise DecodeError(
320 "Too many content encodings in the chain: "
321 f"{len(encodings)} > {self.max_decode_links}"
322 )
323 self._decoders = [_get_decoder(e) for e in encodings]
325 def flush(self) -> bytes:
326 return self._decoders[0].flush()
328 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
329 if max_length <= 0:
330 for d in reversed(self._decoders):
331 data = d.decompress(data)
332 return data
334 ret = bytearray()
335 # Every while loop iteration goes through all decoders once.
336 # It exits when enough data is read or no more data can be read.
337 # It is possible that the while loop iteration does not produce
338 # any data because we retrieve up to `max_length` from every
339 # decoder, and the amount of bytes may be insufficient for the
340 # next decoder to produce enough/any output.
341 while True:
342 any_data = False
343 for d in reversed(self._decoders):
344 data = d.decompress(data, max_length=max_length - len(ret))
345 if data:
346 any_data = True
347 # We should not break when no data is returned because
348 # next decoders may produce data even with empty input.
349 ret += data
350 if not any_data or len(ret) >= max_length:
351 return bytes(ret)
352 data = b""
354 @property
355 def has_unconsumed_tail(self) -> bool:
356 return any(d.has_unconsumed_tail for d in self._decoders)
359def _get_decoder(mode: str) -> ContentDecoder:
360 if "," in mode:
361 return MultiDecoder(mode)
363 # According to RFC 9110 section 8.4.1.3, recipients should
364 # consider x-gzip equivalent to gzip
365 if mode in ("gzip", "x-gzip"):
366 return GzipDecoder()
368 if brotli is not None and mode == "br":
369 return BrotliDecoder()
371 if HAS_ZSTD and mode == "zstd":
372 return ZstdDecoder()
374 return DeflateDecoder()
377class BytesQueueBuffer:
378 """Memory-efficient bytes buffer
380 To return decoded data in read() and still follow the BufferedIOBase API, we need a
381 buffer to always return the correct amount of bytes.
383 This buffer should be filled using calls to put()
385 Our maximum memory usage is determined by the sum of the size of:
387 * self.buffer, which contains the full data
388 * the largest chunk that we will copy in get()
389 """
391 def __init__(self) -> None:
392 self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque()
393 self._size: int = 0
395 def __len__(self) -> int:
396 return self._size
398 def put(self, data: bytes) -> None:
399 self.buffer.append(data)
400 self._size += len(data)
402 def get(self, n: int) -> bytes:
403 if n == 0:
404 return b""
405 elif not self.buffer:
406 raise RuntimeError("buffer is empty")
407 elif n < 0:
408 raise ValueError("n should be > 0")
410 if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes):
411 self._size -= n
412 return self.buffer.popleft()
414 fetched = 0
415 ret = io.BytesIO()
416 while fetched < n:
417 remaining = n - fetched
418 chunk = self.buffer.popleft()
419 chunk_length = len(chunk)
420 if remaining < chunk_length:
421 chunk = memoryview(chunk)
422 left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
423 ret.write(left_chunk)
424 self.buffer.appendleft(right_chunk)
425 self._size -= remaining
426 break
427 else:
428 ret.write(chunk)
429 self._size -= chunk_length
430 fetched += chunk_length
432 if not self.buffer:
433 break
435 return ret.getvalue()
437 def get_all(self) -> bytes:
438 buffer = self.buffer
439 if not buffer:
440 assert self._size == 0
441 return b""
442 if len(buffer) == 1:
443 result = buffer.pop()
444 if isinstance(result, memoryview):
445 result = result.tobytes()
446 else:
447 ret = io.BytesIO()
448 ret.writelines(buffer.popleft() for _ in range(len(buffer)))
449 result = ret.getvalue()
450 self._size = 0
451 return result
454class BaseHTTPResponse(io.IOBase):
455 CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
456 if brotli is not None:
457 CONTENT_DECODERS += ["br"]
458 if HAS_ZSTD:
459 CONTENT_DECODERS += ["zstd"]
460 REDIRECT_STATUSES = [301, 302, 303, 307, 308]
462 DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
463 if brotli is not None:
464 DECODER_ERROR_CLASSES += (brotli.error,)
466 if HAS_ZSTD:
467 DECODER_ERROR_CLASSES += (zstd.ZstdError,)
469 def __init__(
470 self,
471 *,
472 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
473 status: int,
474 version: int,
475 version_string: str,
476 reason: str | None,
477 decode_content: bool,
478 request_url: str | None,
479 retries: Retry | None = None,
480 ) -> None:
481 if isinstance(headers, HTTPHeaderDict):
482 self.headers = headers
483 else:
484 self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type]
485 self.status = status
486 self.version = version
487 self.version_string = version_string
488 self.reason = reason
489 self.decode_content = decode_content
490 self._has_decoded_content = False
491 self._request_url: str | None = request_url
492 self.retries = retries
494 self.chunked = False
495 tr_enc = self.headers.get("transfer-encoding", "").lower()
496 # Don't incur the penalty of creating a list and then discarding it
497 encodings = (enc.strip() for enc in tr_enc.split(","))
498 if "chunked" in encodings:
499 self.chunked = True
501 self._decoder: ContentDecoder | None = None
502 # Distinguish an uninitialized decoder from a response needing no decoder.
503 self._decoder_initialized = False
504 self.length_remaining: int | None
506 def get_redirect_location(self) -> str | None | typing.Literal[False]:
507 """
508 Should we redirect and where to?
510 :returns: Truthy redirect location string if we got a redirect status
511 code and valid location. ``None`` if redirect status and no
512 location. ``False`` if not a redirect status code.
513 """
514 if self.status in self.REDIRECT_STATUSES:
515 return self.headers.get("location")
516 return False
518 @property
519 def data(self) -> bytes:
520 raise NotImplementedError()
522 def json(self) -> typing.Any:
523 """
524 Deserializes the body of the HTTP response as a Python object.
526 The body of the HTTP response must be encoded using UTF-8, as per
527 `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
529 To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
530 your custom decoder instead.
532 If the body of the HTTP response is not decodable to UTF-8, a
533 `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
534 valid JSON document, a `json.JSONDecodeError` will be raised.
536 Read more :ref:`here <json_content>`.
538 :returns: The body of the HTTP response as a Python object.
539 """
540 data = self.data.decode("utf-8")
541 return _json.loads(data)
543 @property
544 def url(self) -> str | None:
545 raise NotImplementedError()
547 @url.setter
548 def url(self, url: str | None) -> None:
549 raise NotImplementedError()
551 @property
552 def connection(self) -> BaseHTTPConnection | None:
553 raise NotImplementedError()
555 @property
556 def retries(self) -> Retry | None:
557 return self._retries
559 @retries.setter
560 def retries(self, retries: Retry | None) -> None:
561 # Override the request_url if retries has a redirect location.
562 if retries is not None and retries.history:
563 self.url = retries.history[-1].redirect_location
564 self._retries = retries
566 def stream(
567 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
568 ) -> typing.Iterator[bytes]:
569 raise NotImplementedError()
571 def read(
572 self,
573 amt: int | None = None,
574 decode_content: bool | None = None,
575 cache_content: bool = False,
576 ) -> bytes:
577 raise NotImplementedError()
579 def read1(
580 self,
581 amt: int | None = None,
582 decode_content: bool | None = None,
583 ) -> bytes:
584 raise NotImplementedError()
586 def read_chunked(
587 self,
588 amt: int | None = None,
589 decode_content: bool | None = None,
590 ) -> typing.Iterator[bytes]:
591 raise NotImplementedError()
593 def release_conn(self) -> None:
594 raise NotImplementedError()
596 def drain_conn(self) -> None:
597 raise NotImplementedError()
599 def shutdown(self) -> None:
600 raise NotImplementedError()
602 def close(self) -> None:
603 raise NotImplementedError()
605 def _init_decoder(self) -> None:
606 """
607 Set-up the _decoder attribute if necessary.
608 """
609 if self._decoder_initialized:
610 return
612 if self._decoder is None:
613 # Note: content-encoding value should be case-insensitive, per RFC 7230
614 # Section 3.2
615 content_encoding = self.headers.get("content-encoding", "").lower()
616 if content_encoding in self.CONTENT_DECODERS:
617 self._decoder = _get_decoder(content_encoding)
618 elif "," in content_encoding:
619 encodings = [
620 e.strip()
621 for e in content_encoding.split(",")
622 if e.strip() in self.CONTENT_DECODERS
623 ]
624 if encodings:
625 self._decoder = _get_decoder(content_encoding)
627 self._decoder_initialized = True
629 def _decode(
630 self,
631 data: bytes,
632 decode_content: bool | None,
633 flush_decoder: bool,
634 max_length: int | None = None,
635 ) -> bytes:
636 """
637 Decode the data passed in and potentially flush the decoder.
638 """
639 if not decode_content:
640 if self._has_decoded_content:
641 raise RuntimeError(
642 "Calling read(decode_content=False) is not supported after "
643 "read(decode_content=True) was called."
644 )
645 return data
647 if max_length is None or flush_decoder:
648 max_length = -1
650 try:
651 if self._decoder:
652 data = self._decoder.decompress(data, max_length=max_length)
653 self._has_decoded_content = True
654 except self.DECODER_ERROR_CLASSES as e:
655 content_encoding = self.headers.get("content-encoding", "").lower()
656 raise DecodeError(
657 "Received response with content-encoding: %s, but "
658 "failed to decode it." % content_encoding,
659 e,
660 ) from e
661 if flush_decoder:
662 data += self._flush_decoder()
664 return data
666 def _flush_decoder(self) -> bytes:
667 """
668 Flushes the decoder. Should only be called if the decoder is actually
669 being used.
670 """
671 if self._decoder:
672 return self._decoder.decompress(b"") + self._decoder.flush()
673 return b""
675 # Compatibility methods for `io` module
676 def readinto(self, b: bytearray | memoryview[int]) -> int:
677 temp = self.read(len(b))
678 if len(temp) == 0:
679 return 0
680 else:
681 b[: len(temp)] = temp
682 return len(temp)
684 # Methods used by dependent libraries
685 def getheaders(self) -> HTTPHeaderDict:
686 return self.headers
688 def getheader(self, name: str, default: str | None = None) -> str | None:
689 return self.headers.get(name, default)
691 # Compatibility method for http.cookiejar
692 def info(self) -> HTTPHeaderDict:
693 return self.headers
695 def geturl(self) -> str | None:
696 return self.url
699class HTTPResponse(BaseHTTPResponse):
700 """
701 HTTP Response container.
703 Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
704 loaded and decoded on-demand when the ``data`` property is accessed. This
705 class is also compatible with the Python standard library's :mod:`io`
706 module, and can hence be treated as a readable object in the context of that
707 framework.
709 Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
711 :param preload_content:
712 If True, the response's body will be preloaded during construction.
714 :param decode_content:
715 If True, will attempt to decode the body based on the
716 'content-encoding' header.
718 :param original_response:
719 When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
720 object, it's convenient to include the original for debug purposes. It's
721 otherwise unused.
723 :param retries:
724 The retries contains the last :class:`~urllib3.util.retry.Retry` that
725 was used during the request.
727 :param enforce_content_length:
728 Enforce content length checking. Body returned by server must match
729 value of Content-Length header, if present. Otherwise, raise error.
730 """
732 def __init__(
733 self,
734 body: _TYPE_BODY = "",
735 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
736 status: int = 0,
737 version: int = 0,
738 version_string: str = "HTTP/?",
739 reason: str | None = None,
740 preload_content: bool = True,
741 decode_content: bool = True,
742 original_response: _HttplibHTTPResponse | None = None,
743 pool: HTTPConnectionPool | None = None,
744 connection: HTTPConnection | None = None,
745 msg: _HttplibHTTPMessage | None = None,
746 retries: Retry | None = None,
747 enforce_content_length: bool = True,
748 request_method: str | None = None,
749 request_url: str | None = None,
750 auto_close: bool = True,
751 sock_shutdown: typing.Callable[[int], None] | None = None,
752 ) -> None:
753 super().__init__(
754 headers=headers,
755 status=status,
756 version=version,
757 version_string=version_string,
758 reason=reason,
759 decode_content=decode_content,
760 request_url=request_url,
761 retries=retries,
762 )
764 self.enforce_content_length = enforce_content_length
765 self.auto_close = auto_close
767 self._body = None
768 self._uncached_read_occurred = False
769 self._fp: _HttplibHTTPResponse | None = None
770 self._original_response = original_response
771 self._fp_bytes_read = 0
772 self.msg = msg
774 if body and isinstance(body, (str, bytes)):
775 self._body = body
777 self._pool = pool
778 self._connection = connection
780 if hasattr(body, "read"):
781 self._fp = body # type: ignore[assignment]
782 self._sock_shutdown = sock_shutdown
784 # Are we using the chunked-style of transfer encoding?
785 self.chunk_left: int | None = None
787 # Determine length of response
788 self.length_remaining = self._init_length(request_method)
790 # Used to return the correct amount of bytes for partial read()s
791 self._decoded_buffer = BytesQueueBuffer()
793 # If requested, preload the body.
794 if preload_content and not self._body:
795 self._body = self.read(decode_content=decode_content)
797 def release_conn(self) -> None:
798 if not self._pool or not self._connection:
799 return None
801 self._pool._put_conn(self._connection)
802 self._connection = None
804 def drain_conn(self) -> None:
805 """
806 Read and discard any remaining HTTP response data in the response connection.
808 Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
809 """
810 try:
811 while self._raw_read(_READ_CHUNK_SIZE):
812 pass
813 except (HTTPError, OSError, BaseSSLError, HTTPException):
814 pass
815 if self._has_decoded_content:
816 # `_raw_read` skips decompression, so we should clean up the
817 # decoder to avoid keeping unnecessary data in memory.
818 self._decoded_buffer = BytesQueueBuffer()
819 self._decoder = None
821 @property
822 def data(self) -> bytes:
823 # For backwards-compat with earlier urllib3 0.4 and earlier.
824 if self._body:
825 return self._body # type: ignore[return-value]
827 if self._fp:
828 return self.read(cache_content=True)
830 return None # type: ignore[return-value]
832 @property
833 def connection(self) -> HTTPConnection | None:
834 return self._connection
836 def isclosed(self) -> bool:
837 return is_fp_closed(self._fp)
839 def tell(self) -> int:
840 """
841 Obtain the number of bytes pulled over the wire so far. May differ from
842 the amount of content returned by :meth:`HTTPResponse.read`
843 if bytes are encoded on the wire (e.g, compressed).
844 """
845 return self._fp_bytes_read
847 def _init_length(self, request_method: str | None) -> int | None:
848 """
849 Set initial length value for Response content if available.
850 """
851 length: int | None
852 content_length: str | None = self.headers.get("content-length")
854 if content_length is not None:
855 if self.chunked:
856 # This Response will fail with an IncompleteRead if it can't be
857 # received as chunked. This method falls back to attempt reading
858 # the response before raising an exception.
859 log.warning(
860 "Received response with both Content-Length and "
861 "Transfer-Encoding set. This is expressly forbidden "
862 "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
863 "attempting to process response as Transfer-Encoding: "
864 "chunked."
865 )
866 return None
868 try:
869 # RFC 7230 section 3.3.2 specifies multiple content lengths can
870 # be sent in a single Content-Length header
871 # (e.g. Content-Length: 42, 42). This line ensures the values
872 # are all valid ints and that as long as the `set` length is 1,
873 # all values are the same. Otherwise, the header is invalid.
874 lengths = {int(val) for val in content_length.split(",")}
875 if len(lengths) > 1:
876 raise InvalidHeader(
877 "Content-Length contained multiple "
878 "unmatching values (%s)" % content_length
879 )
880 length = lengths.pop()
881 except ValueError:
882 length = None
883 else:
884 if length < 0:
885 length = None
887 else: # if content_length is None
888 length = None
890 # Convert status to int for comparison
891 # In some cases, httplib returns a status of "_UNKNOWN"
892 try:
893 status = int(self.status)
894 except ValueError:
895 status = 0
897 # Check for responses that shouldn't include a body
898 if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
899 length = 0
901 return length
903 @contextmanager
904 def _error_catcher(self) -> typing.Generator[None]:
905 """
906 Catch low-level python exceptions, instead re-raising urllib3
907 variants, so that low-level exceptions are not leaked in the
908 high-level api.
910 On exit, release the connection back to the pool.
911 """
912 clean_exit = False
914 try:
915 try:
916 yield
918 except SocketTimeout as e:
919 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
920 # there is yet no clean way to get at it from this context.
921 raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
923 except BaseSSLError as e:
924 # SSL errors related to framing/MAC get wrapped and reraised here
925 raise SSLError(e) from e
927 except IncompleteRead as e:
928 if (
929 e.expected is not None
930 and e.partial is not None
931 and e.expected == -e.partial
932 ):
933 arg = "Response may not contain content."
934 else:
935 arg = f"Connection broken: {e!r}"
936 raise ProtocolError(arg, e) from e
938 except (HTTPException, OSError) as e:
939 raise ProtocolError(f"Connection broken: {e!r}", e) from e
941 # If no exception is thrown, we should avoid cleaning up
942 # unnecessarily.
943 clean_exit = True
944 finally:
945 # If we didn't terminate cleanly, we need to throw away our
946 # connection.
947 if not clean_exit:
948 # The response may not be closed but we're not going to use it
949 # anymore so close it now to ensure that the connection is
950 # released back to the pool.
951 if self._original_response:
952 self._original_response.close()
954 # Closing the response may not actually be sufficient to close
955 # everything, so if we have a hold of the connection close that
956 # too.
957 if self._connection:
958 self._connection.close()
960 # If we hold the original response but it's closed now, we should
961 # return the connection back to the pool.
962 if self._original_response and self._original_response.isclosed():
963 self.release_conn()
965 def _fp_read(
966 self,
967 amt: int | None = None,
968 *,
969 read1: bool = False,
970 ) -> bytes:
971 """
972 Read a response with the thought that reading the number of bytes
973 larger than can fit in a 32-bit int at a time via SSL in some
974 known cases leads to an overflow error that has to be prevented
975 if `amt` or `self.length_remaining` indicate that a problem may
976 happen.
978 This happens to urllib3 injected with pyOpenSSL-backed SSL-support.
979 """
980 assert self._fp
981 c_int_max = 2**31 - 1
982 if (
983 (amt and amt > c_int_max)
984 or (
985 amt is None
986 and self.length_remaining
987 and self.length_remaining > c_int_max
988 )
989 ) and util.IS_PYOPENSSL:
990 if read1:
991 return self._fp.read1(c_int_max)
992 buffer = io.BytesIO()
993 # Besides `max_chunk_amt` being a maximum chunk size, it
994 # affects memory overhead of reading a response by this
995 # method in CPython.
996 # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
997 # chunk size that does not lead to an overflow error, but
998 # 256 MiB is a compromise.
999 max_chunk_amt = 2**28
1000 while amt is None or amt != 0:
1001 if amt is not None:
1002 chunk_amt = min(amt, max_chunk_amt)
1003 amt -= chunk_amt
1004 else:
1005 chunk_amt = max_chunk_amt
1006 data = self._fp.read(chunk_amt)
1007 if not data:
1008 break
1009 buffer.write(data)
1010 del data # to reduce peak memory usage by `max_chunk_amt`.
1011 return buffer.getvalue()
1012 elif read1:
1013 return self._fp.read1(amt) if amt is not None else self._fp.read1()
1014 else:
1015 # StringIO doesn't like amt=None
1016 return self._fp.read(amt) if amt is not None else self._fp.read()
1018 def _raw_read(
1019 self,
1020 amt: int | None = None,
1021 *,
1022 read1: bool = False,
1023 ) -> bytes:
1024 """
1025 Reads `amt` of bytes from the socket.
1026 """
1027 if self._fp is None:
1028 return None # type: ignore[return-value]
1030 fp_closed = getattr(self._fp, "closed", False)
1032 with self._error_catcher():
1033 data = self._fp_read(amt, read1=read1) if not fp_closed else b""
1034 if amt is not None and amt != 0 and not data:
1035 # Platform-specific: Buggy versions of Python.
1036 # Close the connection when no data is returned
1037 #
1038 # This is redundant to what httplib/http.client _should_
1039 # already do. However, versions of python released before
1040 # December 15, 2012 (http://bugs.python.org/issue16298) do
1041 # not properly close the connection in all cases. There is
1042 # no harm in redundantly calling close.
1043 self._fp.close()
1044 if (
1045 self.enforce_content_length
1046 and self.length_remaining is not None
1047 and self.length_remaining != 0
1048 ):
1049 # This is an edge case that httplib failed to cover due
1050 # to concerns of backward compatibility. We're
1051 # addressing it here to make sure IncompleteRead is
1052 # raised during streaming, so all calls with incorrect
1053 # Content-Length are caught.
1054 raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
1055 elif read1 and (
1056 (amt != 0 and not data) or self.length_remaining == len(data)
1057 ):
1058 # All data has been read, but `self._fp.read1` in
1059 # CPython 3.12 and older doesn't always close
1060 # `http.client.HTTPResponse`, so we close it here.
1061 # See https://github.com/python/cpython/issues/113199
1062 self._fp.close()
1064 if data:
1065 self._fp_bytes_read += len(data)
1066 if self.length_remaining is not None:
1067 self.length_remaining -= len(data)
1068 return data
1070 def read(
1071 self,
1072 amt: int | None = None,
1073 decode_content: bool | None = None,
1074 cache_content: bool = False,
1075 ) -> bytes:
1076 """
1077 Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
1078 parameters: ``decode_content`` and ``cache_content``.
1080 :param amt:
1081 How much of the content to read. If specified, caching is skipped
1082 because it doesn't make sense to cache partial content as the full
1083 response.
1085 :param decode_content:
1086 If True, will attempt to decode the body based on the
1087 'content-encoding' header.
1089 :param cache_content:
1090 If True, will save the returned data such that the same result is
1091 returned despite of the state of the underlying file object. This
1092 is useful if you want the ``.data`` property to continue working
1093 after having ``.read()`` the file object. (Overridden if ``amt`` is
1094 set.)
1095 """
1096 self._init_decoder()
1097 if decode_content is None:
1098 decode_content = self.decode_content
1100 if amt and amt < 0:
1101 # Negative numbers and `None` should be treated the same.
1102 amt = None
1103 elif amt is not None:
1104 cache_content = False
1106 if (
1107 self._decoder
1108 and self._decoder.has_unconsumed_tail
1109 and len(self._decoded_buffer) < amt
1110 ):
1111 decoded_data = self._decode(
1112 b"",
1113 decode_content,
1114 flush_decoder=False,
1115 max_length=amt - len(self._decoded_buffer),
1116 )
1117 self._decoded_buffer.put(decoded_data)
1118 if len(self._decoded_buffer) >= amt:
1119 return self._decoded_buffer.get(amt)
1121 data = self._raw_read(amt)
1122 if not cache_content:
1123 self._uncached_read_occurred = True
1125 flush_decoder = amt is None or (amt != 0 and not data)
1127 if (
1128 not data
1129 and len(self._decoded_buffer) == 0
1130 and not (self._decoder and self._decoder.has_unconsumed_tail)
1131 ):
1132 return data
1134 if amt is None:
1135 data = self._decode(data, decode_content, flush_decoder)
1136 # It's possible that there is buffered decoded data after a
1137 # partial read.
1138 if decode_content and len(self._decoded_buffer) > 0:
1139 self._decoded_buffer.put(data)
1140 data = self._decoded_buffer.get_all()
1142 if cache_content and not self._uncached_read_occurred:
1143 self._body = data
1144 else:
1145 # do not waste memory on buffer when not decoding
1146 if not decode_content:
1147 if self._has_decoded_content:
1148 raise RuntimeError(
1149 "Calling read(decode_content=False) is not supported after "
1150 "read(decode_content=True) was called."
1151 )
1152 return data
1154 decoded_data = self._decode(
1155 data,
1156 decode_content,
1157 flush_decoder,
1158 max_length=amt - len(self._decoded_buffer),
1159 )
1160 self._decoded_buffer.put(decoded_data)
1162 while len(self._decoded_buffer) < amt and data:
1163 # TODO make sure to initially read enough data to get past the headers
1164 # For example, the GZ file header takes 10 bytes, we don't want to read
1165 # it one byte at a time
1166 data = self._raw_read(amt)
1167 decoded_data = self._decode(
1168 data,
1169 decode_content,
1170 flush_decoder,
1171 max_length=amt - len(self._decoded_buffer),
1172 )
1173 self._decoded_buffer.put(decoded_data)
1174 data = self._decoded_buffer.get(amt)
1176 return data
1178 def read1(
1179 self,
1180 amt: int | None = None,
1181 decode_content: bool | None = None,
1182 ) -> bytes:
1183 """
1184 Similar to ``http.client.HTTPResponse.read1`` and documented
1185 in :meth:`io.BufferedReader.read1`, but with an additional parameter:
1186 ``decode_content``.
1188 :param amt:
1189 How much of the content to read.
1191 :param decode_content:
1192 If True, will attempt to decode the body based on the
1193 'content-encoding' header.
1194 """
1195 if decode_content is None:
1196 decode_content = self.decode_content
1197 if amt and amt < 0:
1198 # Negative numbers and `None` should be treated the same.
1199 amt = None
1200 # try and respond without going to the network
1201 if self._has_decoded_content:
1202 if not decode_content:
1203 raise RuntimeError(
1204 "Calling read1(decode_content=False) is not supported after "
1205 "read1(decode_content=True) was called."
1206 )
1207 if (
1208 self._decoder
1209 and self._decoder.has_unconsumed_tail
1210 and (amt is None or len(self._decoded_buffer) < amt)
1211 ):
1212 decoded_data = self._decode(
1213 b"",
1214 decode_content,
1215 flush_decoder=False,
1216 max_length=(
1217 amt - len(self._decoded_buffer) if amt is not None else None
1218 ),
1219 )
1220 self._decoded_buffer.put(decoded_data)
1221 if len(self._decoded_buffer) > 0:
1222 if amt is None:
1223 return self._decoded_buffer.get_all()
1224 return self._decoded_buffer.get(amt)
1225 if amt == 0:
1226 return b""
1228 # FIXME, this method's type doesn't say returning None is possible
1229 data = self._raw_read(amt, read1=True)
1230 self._uncached_read_occurred = True
1231 if not decode_content or data is None:
1232 return data
1234 self._init_decoder()
1235 while True:
1236 flush_decoder = not data
1237 decoded_data = self._decode(
1238 data, decode_content, flush_decoder, max_length=amt
1239 )
1240 self._decoded_buffer.put(decoded_data)
1241 if decoded_data or flush_decoder:
1242 break
1243 data = self._raw_read(8192, read1=True)
1245 if amt is None:
1246 return self._decoded_buffer.get_all()
1247 return self._decoded_buffer.get(amt)
1249 def stream(
1250 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
1251 ) -> typing.Generator[bytes]:
1252 """
1253 A generator wrapper for the read() method. A call will block until
1254 ``amt`` bytes have been read from the connection or until the
1255 connection is closed.
1257 :param amt:
1258 How much of the content to read. The generator will return up to
1259 much data per iteration, but may return less. This is particularly
1260 likely when using compressed data. However, the empty string will
1261 never be returned.
1263 :param decode_content:
1264 If True, will attempt to decode the body based on the
1265 'content-encoding' header.
1266 """
1267 if amt == 0:
1268 return
1270 if self.chunked and self.supports_chunked_reads():
1271 yield from self.read_chunked(amt, decode_content=decode_content)
1272 else:
1273 while (
1274 not is_fp_closed(self._fp)
1275 or len(self._decoded_buffer) > 0
1276 or (self._decoder and self._decoder.has_unconsumed_tail)
1277 ):
1278 data = self.read(amt=amt, decode_content=decode_content)
1280 if data:
1281 yield data
1283 # Overrides from io.IOBase
1284 def readable(self) -> bool:
1285 return True
1287 def shutdown(self) -> None:
1288 if not self._sock_shutdown:
1289 raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set")
1290 if self._connection is None:
1291 raise RuntimeError(
1292 "Cannot shutdown as connection has already been released to the pool"
1293 )
1294 self._sock_shutdown(socket.SHUT_RD)
1296 def close(self) -> None:
1297 self._sock_shutdown = None
1299 if not self.closed and self._fp:
1300 self._fp.close()
1302 if self._connection:
1303 self._connection.close()
1305 if not self.auto_close:
1306 io.IOBase.close(self)
1308 @property
1309 def closed(self) -> bool:
1310 if not self.auto_close:
1311 return io.IOBase.closed.__get__(self) # type: ignore[no-any-return]
1312 elif self._fp is None:
1313 return True
1314 elif hasattr(self._fp, "isclosed"):
1315 return self._fp.isclosed()
1316 elif hasattr(self._fp, "closed"):
1317 return self._fp.closed
1318 else:
1319 return True
1321 def fileno(self) -> int:
1322 if self._fp is None:
1323 raise OSError("HTTPResponse has no file to get a fileno from")
1324 elif hasattr(self._fp, "fileno"):
1325 return self._fp.fileno()
1326 else:
1327 raise OSError(
1328 "The file-like object this HTTPResponse is wrapped "
1329 "around has no file descriptor"
1330 )
1332 def flush(self) -> None:
1333 if (
1334 self._fp is not None
1335 and hasattr(self._fp, "flush")
1336 and not getattr(self._fp, "closed", False)
1337 ):
1338 return self._fp.flush()
1340 def supports_chunked_reads(self) -> bool:
1341 """
1342 Checks if the underlying file-like object looks like a
1343 :class:`http.client.HTTPResponse` object. We do this by testing for
1344 the fp attribute. If it is present we assume it returns raw chunks as
1345 processed by read_chunked().
1346 """
1347 return hasattr(self._fp, "fp")
1349 def _update_chunk_length(self) -> None:
1350 # First, we'll figure out length of a chunk and then
1351 # we'll try to read it from socket.
1352 if self.chunk_left is not None:
1353 return None
1354 line = self._fp.fp.readline() # type: ignore[union-attr]
1355 line = line.split(b";", 1)[0]
1356 try:
1357 self.chunk_left = int(line, 16)
1358 except ValueError:
1359 self.close()
1360 if line:
1361 # Invalid chunked protocol response, abort.
1362 raise InvalidChunkLength(self, line) from None
1363 else:
1364 # Truncated at start of next chunk
1365 raise ProtocolError("Response ended prematurely") from None
1367 def _handle_chunk(self, amt: int | None) -> bytes:
1368 returned_chunk = None
1369 if amt is None:
1370 chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1371 returned_chunk = chunk
1372 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1373 self.chunk_left = None
1374 elif self.chunk_left is not None and amt < self.chunk_left:
1375 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1376 self.chunk_left = self.chunk_left - amt
1377 returned_chunk = value
1378 elif amt == self.chunk_left:
1379 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1380 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1381 self.chunk_left = None
1382 returned_chunk = value
1383 else: # amt > self.chunk_left
1384 returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1385 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1386 self.chunk_left = None
1387 return returned_chunk # type: ignore[no-any-return]
1389 def read_chunked(
1390 self, amt: int | None = None, decode_content: bool | None = None
1391 ) -> typing.Generator[bytes]:
1392 """
1393 Similar to :meth:`HTTPResponse.read`, but with an additional
1394 parameter: ``decode_content``.
1396 :param amt:
1397 How much of the content to read. If specified, caching is skipped
1398 because it doesn't make sense to cache partial content as the full
1399 response.
1401 :param decode_content:
1402 If True, will attempt to decode the body based on the
1403 'content-encoding' header.
1404 """
1405 self._init_decoder()
1406 # FIXME: Rewrite this method and make it a class with a better structured logic.
1407 if not self.chunked:
1408 raise ResponseNotChunked(
1409 "Response is not chunked. "
1410 "Header 'transfer-encoding: chunked' is missing."
1411 )
1412 if not self.supports_chunked_reads():
1413 raise BodyNotHttplibCompatible(
1414 "Body should be http.client.HTTPResponse like. "
1415 "It should have have an fp attribute which returns raw chunks."
1416 )
1418 with self._error_catcher():
1419 # Don't bother reading the body of a HEAD request.
1420 if self._original_response and is_response_to_head(self._original_response):
1421 self._original_response.close()
1422 return None
1424 # If a response is already read and closed
1425 # then return immediately.
1426 if self._fp.fp is None: # type: ignore[union-attr]
1427 return None
1429 if amt == 0:
1430 return
1431 elif amt and amt < 0:
1432 # Negative numbers and `None` should be treated the same,
1433 # but httplib handles only `None` correctly.
1434 amt = None
1436 while True:
1437 # First, check if any data is left in the decoder's buffer.
1438 if self._decoder and self._decoder.has_unconsumed_tail:
1439 chunk = b""
1440 else:
1441 self._update_chunk_length()
1442 self._uncached_read_occurred = True
1443 if self.chunk_left == 0:
1444 break
1445 chunk = self._handle_chunk(amt)
1446 decoded = self._decode(
1447 chunk,
1448 decode_content=decode_content,
1449 flush_decoder=False,
1450 max_length=amt,
1451 )
1452 if decoded:
1453 yield decoded
1455 if decode_content:
1456 # On CPython and PyPy, we should never need to flush the
1457 # decoder. However, on Jython we *might* need to, so
1458 # lets defensively do it anyway.
1459 decoded = self._flush_decoder()
1460 if decoded: # Platform-specific: Jython.
1461 yield decoded
1463 # Chunk content ends with \r\n: discard it.
1464 while self._fp is not None:
1465 line = self._fp.fp.readline()
1466 if not line:
1467 # Some sites may not end with '\r\n'.
1468 break
1469 if line == b"\r\n":
1470 break
1472 # We read everything; close the "file".
1473 if self._original_response:
1474 self._original_response.close()
1476 @property
1477 def url(self) -> str | None:
1478 """
1479 Returns the URL that was the source of this response.
1480 If the request that generated this response redirected, this method
1481 will return the final redirect location.
1482 """
1483 return self._request_url
1485 @url.setter
1486 def url(self, url: str | None) -> None:
1487 self._request_url = url
1489 def __iter__(self) -> typing.Iterator[bytes]:
1490 buffer: list[bytes] = []
1491 for chunk in self.stream(decode_content=True):
1492 if b"\n" in chunk:
1493 chunks = chunk.split(b"\n")
1494 yield b"".join(buffer) + chunks[0] + b"\n"
1495 for x in chunks[1:-1]:
1496 yield x + b"\n"
1497 if chunks[-1]:
1498 buffer = [chunks[-1]]
1499 else:
1500 buffer = []
1501 else:
1502 buffer.append(chunk)
1503 if buffer:
1504 yield b"".join(buffer)