Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/response.py: 19%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import collections
4import io
5import json as _json
6import logging
7import socket
8import sys
9import typing
10import warnings
11import zlib
12from contextlib import contextmanager
13from http.client import HTTPMessage as _HttplibHTTPMessage
14from http.client import HTTPResponse as _HttplibHTTPResponse
15from socket import timeout as SocketTimeout
17if typing.TYPE_CHECKING:
18 from ._base_connection import BaseHTTPConnection
20try:
21 try:
22 import brotlicffi as brotli # type: ignore[import-not-found]
23 except ImportError:
24 import brotli # type: ignore[import-not-found]
25except ImportError:
26 brotli = None
28from . import util
29from ._base_connection import _TYPE_BODY
30from ._collections import HTTPHeaderDict
31from .connection import BaseSSLError, HTTPConnection, HTTPException
32from .exceptions import (
33 BodyNotHttplibCompatible,
34 DecodeError,
35 DependencyWarning,
36 HTTPError,
37 IncompleteRead,
38 InvalidChunkLength,
39 InvalidHeader,
40 ProtocolError,
41 ReadTimeoutError,
42 ResponseNotChunked,
43 SSLError,
44)
45from .util.response import is_fp_closed, is_response_to_head
46from .util.retry import Retry
48if typing.TYPE_CHECKING:
49 from .connectionpool import HTTPConnectionPool
51log = logging.getLogger(__name__)
53# Read in 64 KiB chunks
54_READ_CHUNK_SIZE = 2**16
56# Maximum length of a chunk-size line (and the trailing CRLF line) when reading a
57# chunked-transfer-encoded response. Matches ``http.client._MAXLINE`` so urllib3's
58# streaming path bounds these reads exactly like the stdlib ``.read()`` path does,
59# preventing a malicious server from forcing unbounded buffering (memory exhaustion)
60# via an unterminated chunk-size line.
61_MAX_CHUNK_LINE_LENGTH = 2**16
64class ContentDecoder:
65 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
66 raise NotImplementedError()
68 @property
69 def has_unconsumed_tail(self) -> bool:
70 raise NotImplementedError()
72 def flush(self) -> bytes:
73 raise NotImplementedError()
76class DeflateDecoder(ContentDecoder):
77 def __init__(self) -> None:
78 self._first_try = True
79 self._first_try_data = b""
80 self._unfed_data = b""
81 self._obj = zlib.decompressobj()
83 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
84 data = self._unfed_data + data
85 self._unfed_data = b""
86 # Data received after EOF cannot produce more output from this
87 # Deflate stream.
88 if self._obj.eof:
89 return b""
90 if not data and not self._obj.unconsumed_tail:
91 return data
92 original_max_length = max_length
93 if original_max_length < 0:
94 max_length = 0
95 elif original_max_length == 0:
96 # We should not pass 0 to the zlib decompressor because 0 is
97 # the default value that will make zlib decompress without a
98 # length limit.
99 # Data should be stored for subsequent calls.
100 self._unfed_data = data
101 return b""
103 # Subsequent calls always reuse `self._obj`. zlib requires
104 # passing the unconsumed tail if decompression is to continue.
105 if not self._first_try:
106 return self._obj.decompress(
107 self._obj.unconsumed_tail + data, max_length=max_length
108 )
110 # First call tries with RFC 1950 ZLIB format.
111 self._first_try_data += data
112 try:
113 decompressed = self._obj.decompress(data, max_length=max_length)
114 if decompressed:
115 self._first_try = False
116 self._first_try_data = b""
117 return decompressed
118 # On failure, it falls back to RFC 1951 DEFLATE format.
119 except zlib.error:
120 self._first_try = False
121 self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
122 try:
123 return self.decompress(
124 self._first_try_data, max_length=original_max_length
125 )
126 finally:
127 self._first_try_data = b""
129 @property
130 def has_unconsumed_tail(self) -> bool:
131 return bool(self._unfed_data) or (
132 bool(self._obj.unconsumed_tail)
133 and not self._first_try
134 and not self._obj.eof
135 )
137 def flush(self) -> bytes:
138 return self._obj.flush()
141class GzipDecoderState:
142 FIRST_MEMBER = 0
143 OTHER_MEMBERS = 1
144 SWALLOW_DATA = 2
147class GzipDecoder(ContentDecoder):
148 def __init__(self) -> None:
149 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
150 self._state = GzipDecoderState.FIRST_MEMBER
151 self._unconsumed_tail = b""
153 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
154 if self._state == GzipDecoderState.SWALLOW_DATA:
155 return b""
157 if max_length == 0:
158 # We should not pass 0 to the zlib decompressor because 0 is
159 # the default value that will make zlib decompress without a
160 # length limit.
161 # Data should be stored for subsequent calls.
162 self._unconsumed_tail += data
163 return b""
165 # zlib requires passing the unconsumed tail to the subsequent
166 # call if decompression is to continue.
167 data = self._unconsumed_tail + data
168 if not data and self._obj.eof:
169 return b""
171 ret: bytes | bytearray = b""
172 while True:
173 try:
174 chunk = self._obj.decompress(
175 data, max_length=max(max_length - len(ret), 0)
176 )
177 if not ret:
178 ret = chunk
179 elif chunk:
180 if isinstance(ret, bytes):
181 ret = bytearray(ret)
182 ret += chunk
183 del chunk
184 except zlib.error:
185 previous_state = self._state
186 # Ignore data after the first error
187 self._state = GzipDecoderState.SWALLOW_DATA
188 self._unconsumed_tail = b""
189 if previous_state == GzipDecoderState.OTHER_MEMBERS:
190 # Allow trailing garbage acceptable in other gzip clients
191 return ret if isinstance(ret, bytes) else bytes(ret)
192 raise
194 self._unconsumed_tail = data = (
195 self._obj.unconsumed_tail or self._obj.unused_data
196 )
197 if max_length > 0 and len(ret) >= max_length:
198 break
200 if not data:
201 return ret if isinstance(ret, bytes) else bytes(ret)
202 # When the end of a gzip member is reached, a new decompressor
203 # must be created for unused (possibly future) data.
204 if self._obj.eof:
205 self._state = GzipDecoderState.OTHER_MEMBERS
206 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
208 return ret if isinstance(ret, bytes) else bytes(ret)
210 @property
211 def has_unconsumed_tail(self) -> bool:
212 return bool(self._unconsumed_tail)
214 def flush(self) -> bytes:
215 if self._state == GzipDecoderState.SWALLOW_DATA:
216 return b""
217 return self._obj.flush()
220if brotli is not None:
222 class BrotliDecoder(ContentDecoder):
223 # Supports both 'brotlipy' and 'Brotli' packages
224 # since they share an import name. The top branches
225 # are for 'brotlipy' and bottom branches for 'Brotli'
226 def __init__(self) -> None:
227 self._obj = brotli.Decompressor()
228 if hasattr(self._obj, "decompress"):
229 setattr(self, "_decompress", self._obj.decompress)
230 else:
231 setattr(self, "_decompress", self._obj.process)
233 # Requires Brotli >= 1.2.0 for `output_buffer_limit`.
234 def _decompress(self, data: bytes, output_buffer_limit: int = -1) -> bytes:
235 raise NotImplementedError()
237 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
238 try:
239 if max_length > 0:
240 return self._decompress(data, output_buffer_limit=max_length)
241 else:
242 return self._decompress(data)
243 except TypeError:
244 # Fallback for Brotli/brotlicffi/brotlipy versions without
245 # the `output_buffer_limit` parameter.
246 warnings.warn(
247 "Brotli >= 1.2.0 is required to prevent decompression bombs.",
248 DependencyWarning,
249 )
250 return self._decompress(data)
252 @property
253 def has_unconsumed_tail(self) -> bool:
254 try:
255 return not self._obj.can_accept_more_data()
256 except AttributeError:
257 return False
259 def flush(self) -> bytes:
260 if hasattr(self._obj, "flush"):
261 return self._obj.flush() # type: ignore[no-any-return]
262 return b""
265try:
266 if sys.version_info >= (3, 14):
267 from compression import zstd
268 else:
269 from backports import zstd
270except ImportError:
271 HAS_ZSTD = False
272else:
273 HAS_ZSTD = True
275 class ZstdDecoder(ContentDecoder):
276 def __init__(self) -> None:
277 self._obj = zstd.ZstdDecompressor()
279 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
280 if not data and not self.has_unconsumed_tail:
281 return b""
282 if self._obj.eof:
283 data = self._obj.unused_data + data
284 self._obj = zstd.ZstdDecompressor()
285 part = self._obj.decompress(data, max_length=max_length)
286 length = len(part)
287 data_parts = [part]
288 # Every loop iteration is supposed to read data from a separate frame.
289 # The loop breaks when:
290 # - enough data is read;
291 # - no more unused data is available;
292 # - end of the last read frame has not been reached (i.e.,
293 # more data has to be fed).
294 while (
295 self._obj.eof
296 and self._obj.unused_data
297 and (max_length < 0 or length < max_length)
298 ):
299 unused_data = self._obj.unused_data
300 if not self._obj.needs_input:
301 self._obj = zstd.ZstdDecompressor()
302 part = self._obj.decompress(
303 unused_data,
304 max_length=(max_length - length) if max_length > 0 else -1,
305 )
306 if part_length := len(part):
307 data_parts.append(part)
308 length += part_length
309 elif self._obj.needs_input:
310 break
311 return b"".join(data_parts)
313 @property
314 def has_unconsumed_tail(self) -> bool:
315 return not (self._obj.needs_input or self._obj.eof) or bool(
316 self._obj.unused_data
317 )
319 def flush(self) -> bytes:
320 if not self._obj.eof:
321 raise DecodeError("Zstandard data is incomplete")
322 return b""
325class MultiDecoder(ContentDecoder):
326 """
327 From RFC7231:
328 If one or more encodings have been applied to a representation, the
329 sender that applied the encodings MUST generate a Content-Encoding
330 header field that lists the content codings in the order in which
331 they were applied.
332 """
334 # Maximum allowed number of chained HTTP encodings in the
335 # Content-Encoding header.
336 max_decode_links = 5
338 def __init__(self, modes: str) -> None:
339 encodings = [m.strip() for m in modes.split(",")]
340 if len(encodings) > self.max_decode_links:
341 raise DecodeError(
342 "Too many content encodings in the chain: "
343 f"{len(encodings)} > {self.max_decode_links}"
344 )
345 self._decoders = [_get_decoder(e) for e in encodings]
347 def flush(self) -> bytes:
348 return self._decoders[0].flush()
350 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
351 if max_length <= 0:
352 for d in reversed(self._decoders):
353 data = d.decompress(data)
354 return data
356 ret = bytearray()
357 # Every while loop iteration goes through all decoders once.
358 # It exits when enough data is read or no more data can be read.
359 # It is possible that the while loop iteration does not produce
360 # any data because we retrieve up to `max_length` from every
361 # decoder, and the amount of bytes may be insufficient for the
362 # next decoder to produce enough/any output.
363 while True:
364 any_data = False
365 for d in reversed(self._decoders):
366 data = d.decompress(data, max_length=max_length - len(ret))
367 if data:
368 any_data = True
369 # We should not break when no data is returned because
370 # next decoders may produce data even with empty input.
371 ret += data
372 if not any_data or len(ret) >= max_length:
373 return bytes(ret)
374 data = b""
376 @property
377 def has_unconsumed_tail(self) -> bool:
378 return any(d.has_unconsumed_tail for d in self._decoders)
381def _get_decoder(mode: str) -> ContentDecoder:
382 if "," in mode:
383 return MultiDecoder(mode)
385 # According to RFC 9110 section 8.4.1.3, recipients should
386 # consider x-gzip equivalent to gzip
387 if mode in ("gzip", "x-gzip"):
388 return GzipDecoder()
390 if brotli is not None and mode == "br":
391 return BrotliDecoder()
393 if HAS_ZSTD and mode == "zstd":
394 return ZstdDecoder()
396 return DeflateDecoder()
399class BytesQueueBuffer:
400 """Memory-efficient bytes buffer
402 To return decoded data in read() and still follow the BufferedIOBase API, we need a
403 buffer to always return the correct amount of bytes.
405 This buffer should be filled using calls to put()
407 Our maximum memory usage is determined by the sum of the size of:
409 * self.buffer, which contains the full data
410 * the largest chunk that we will copy in get()
411 """
413 def __init__(self) -> None:
414 self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque()
415 self._size: int = 0
417 def __len__(self) -> int:
418 return self._size
420 def put(self, data: bytes) -> None:
421 self.buffer.append(data)
422 self._size += len(data)
424 def get(self, n: int) -> bytes:
425 if n == 0:
426 return b""
427 elif not self.buffer:
428 raise RuntimeError("buffer is empty")
429 elif n < 0:
430 raise ValueError("n should be > 0")
432 if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes):
433 self._size -= n
434 return self.buffer.popleft()
436 fetched = 0
437 ret = io.BytesIO()
438 while fetched < n:
439 remaining = n - fetched
440 chunk = self.buffer.popleft()
441 chunk_length = len(chunk)
442 if remaining < chunk_length:
443 chunk = memoryview(chunk)
444 left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
445 ret.write(left_chunk)
446 self.buffer.appendleft(right_chunk)
447 self._size -= remaining
448 break
449 else:
450 ret.write(chunk)
451 self._size -= chunk_length
452 fetched += chunk_length
454 if not self.buffer:
455 break
457 return ret.getvalue()
459 def get_all(self) -> bytes:
460 buffer = self.buffer
461 if not buffer:
462 assert self._size == 0
463 return b""
464 if len(buffer) == 1:
465 result = buffer.pop()
466 if isinstance(result, memoryview):
467 result = result.tobytes()
468 else:
469 ret = io.BytesIO()
470 ret.writelines(buffer.popleft() for _ in range(len(buffer)))
471 result = ret.getvalue()
472 self._size = 0
473 return result
476class BaseHTTPResponse(io.IOBase):
477 CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
478 if brotli is not None:
479 CONTENT_DECODERS += ["br"]
480 if HAS_ZSTD:
481 CONTENT_DECODERS += ["zstd"]
482 REDIRECT_STATUSES = [301, 302, 303, 307, 308]
484 DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
485 if brotli is not None:
486 DECODER_ERROR_CLASSES += (brotli.error,)
488 if HAS_ZSTD:
489 DECODER_ERROR_CLASSES += (zstd.ZstdError,)
491 def __init__(
492 self,
493 *,
494 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
495 status: int,
496 version: int,
497 version_string: str,
498 reason: str | None,
499 decode_content: bool,
500 request_url: str | None,
501 retries: Retry | None = None,
502 ) -> None:
503 if isinstance(headers, HTTPHeaderDict):
504 self.headers = headers
505 else:
506 self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type]
507 self.status = status
508 self.version = version
509 self.version_string = version_string
510 self.reason = reason
511 self.decode_content = decode_content
512 self._has_decoded_content = False
513 self._request_url: str | None = request_url
514 self.retries = retries
516 self.chunked = False
517 tr_enc = self.headers.get("transfer-encoding", "").lower()
518 # Don't incur the penalty of creating a list and then discarding it
519 encodings = (enc.strip() for enc in tr_enc.split(","))
520 if "chunked" in encodings:
521 self.chunked = True
523 self._decoder: ContentDecoder | None = None
524 # Distinguish an uninitialized decoder from a response needing no decoder.
525 self._decoder_initialized = False
526 self.length_remaining: int | None
528 def get_redirect_location(self) -> str | None | typing.Literal[False]:
529 """
530 Should we redirect and where to?
532 :returns: Truthy redirect location string if we got a redirect status
533 code and valid location. ``None`` if redirect status and no
534 location. ``False`` if not a redirect status code.
535 """
536 if self.status in self.REDIRECT_STATUSES:
537 return self.headers.get("location")
538 return False
540 @property
541 def data(self) -> bytes:
542 raise NotImplementedError()
544 def json(self) -> typing.Any:
545 """
546 Deserializes the body of the HTTP response as a Python object.
548 The body of the HTTP response must be encoded using UTF-8, as per
549 `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
551 To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
552 your custom decoder instead.
554 If the body of the HTTP response is not decodable to UTF-8, a
555 `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
556 valid JSON document, a `json.JSONDecodeError` will be raised.
558 Read more :ref:`here <json_content>`.
560 :returns: The body of the HTTP response as a Python object.
561 """
562 data = self.data.decode("utf-8")
563 return _json.loads(data)
565 @property
566 def url(self) -> str | None:
567 raise NotImplementedError()
569 @url.setter
570 def url(self, url: str | None) -> None:
571 raise NotImplementedError()
573 @property
574 def connection(self) -> BaseHTTPConnection | None:
575 raise NotImplementedError()
577 @property
578 def retries(self) -> Retry | None:
579 return self._retries
581 @retries.setter
582 def retries(self, retries: Retry | None) -> None:
583 # Override the request_url if retries has a redirect location.
584 if retries is not None and retries.history:
585 self.url = retries.history[-1].redirect_location
586 self._retries = retries
588 def stream(
589 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
590 ) -> typing.Iterator[bytes]:
591 raise NotImplementedError()
593 def read(
594 self,
595 amt: int | None = None,
596 decode_content: bool | None = None,
597 cache_content: bool = False,
598 ) -> bytes:
599 raise NotImplementedError()
601 def read1(
602 self,
603 amt: int | None = None,
604 decode_content: bool | None = None,
605 ) -> bytes:
606 raise NotImplementedError()
608 def read_chunked(
609 self,
610 amt: int | None = None,
611 decode_content: bool | None = None,
612 ) -> typing.Iterator[bytes]:
613 raise NotImplementedError()
615 def release_conn(self) -> None:
616 raise NotImplementedError()
618 def drain_conn(self) -> None:
619 raise NotImplementedError()
621 def shutdown(self) -> None:
622 raise NotImplementedError()
624 def close(self) -> None:
625 raise NotImplementedError()
627 def _init_decoder(self) -> None:
628 """
629 Set-up the _decoder attribute if necessary.
630 """
631 if self._decoder_initialized:
632 return
634 if self._decoder is None:
635 # Note: content-encoding value should be case-insensitive, per RFC 7230
636 # Section 3.2
637 content_encoding = self.headers.get("content-encoding", "").lower()
638 if content_encoding in self.CONTENT_DECODERS:
639 self._decoder = _get_decoder(content_encoding)
640 elif "," in content_encoding:
641 encodings = [
642 e.strip()
643 for e in content_encoding.split(",")
644 if e.strip() in self.CONTENT_DECODERS
645 ]
646 if encodings:
647 self._decoder = _get_decoder(content_encoding)
649 self._decoder_initialized = True
651 def _decode(
652 self,
653 data: bytes,
654 decode_content: bool | None,
655 flush_decoder: bool,
656 max_length: int | None = None,
657 ) -> bytes:
658 """
659 Decode the data passed in and potentially flush the decoder.
660 """
661 if not decode_content:
662 if self._has_decoded_content:
663 raise RuntimeError(
664 "Calling read(decode_content=False) is not supported after "
665 "read(decode_content=True) was called."
666 )
667 return data
669 if max_length is None or flush_decoder:
670 max_length = -1
672 try:
673 if self._decoder:
674 data = self._decoder.decompress(data, max_length=max_length)
675 self._has_decoded_content = True
676 except self.DECODER_ERROR_CLASSES as e:
677 content_encoding = self.headers.get("content-encoding", "").lower()
678 raise DecodeError(
679 "Received response with content-encoding: %s, but "
680 "failed to decode it." % content_encoding,
681 e,
682 ) from e
683 if flush_decoder:
684 data += self._flush_decoder()
686 return data
688 def _flush_decoder(self) -> bytes:
689 """
690 Flushes the decoder. Should only be called if the decoder is actually
691 being used.
692 """
693 if self._decoder:
694 return self._decoder.decompress(b"") + self._decoder.flush()
695 return b""
697 # Compatibility methods for `io` module
698 def readinto(self, b: bytearray | memoryview[int]) -> int:
699 temp = self.read(len(b))
700 if len(temp) == 0:
701 return 0
702 else:
703 b[: len(temp)] = temp
704 return len(temp)
706 # Methods used by dependent libraries
707 def getheaders(self) -> HTTPHeaderDict:
708 return self.headers
710 def getheader(self, name: str, default: str | None = None) -> str | None:
711 return self.headers.get(name, default)
713 # Compatibility method for http.cookiejar
714 def info(self) -> HTTPHeaderDict:
715 return self.headers
717 def geturl(self) -> str | None:
718 return self.url
721class HTTPResponse(BaseHTTPResponse):
722 """
723 HTTP Response container.
725 Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
726 loaded and decoded on-demand when the ``data`` property is accessed. This
727 class is also compatible with the Python standard library's :mod:`io`
728 module, and can hence be treated as a readable object in the context of that
729 framework.
731 Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
733 :param preload_content:
734 If True, the response's body will be preloaded during construction.
736 :param decode_content:
737 If True, will attempt to decode the body based on the
738 'content-encoding' header.
740 :param original_response:
741 When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
742 object, it's convenient to include the original for debug purposes. It's
743 otherwise unused.
745 :param retries:
746 The retries contains the last :class:`~urllib3.util.retry.Retry` that
747 was used during the request.
749 :param enforce_content_length:
750 Enforce content length checking. Body returned by server must match
751 value of Content-Length header, if present. Otherwise, raise error.
752 """
754 def __init__(
755 self,
756 body: _TYPE_BODY = "",
757 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
758 status: int = 0,
759 version: int = 0,
760 version_string: str = "HTTP/?",
761 reason: str | None = None,
762 preload_content: bool = True,
763 decode_content: bool = True,
764 original_response: _HttplibHTTPResponse | None = None,
765 pool: HTTPConnectionPool | None = None,
766 connection: HTTPConnection | None = None,
767 msg: _HttplibHTTPMessage | None = None,
768 retries: Retry | None = None,
769 enforce_content_length: bool = True,
770 request_method: str | None = None,
771 request_url: str | None = None,
772 auto_close: bool = True,
773 sock_shutdown: typing.Callable[[int], None] | None = None,
774 ) -> None:
775 super().__init__(
776 headers=headers,
777 status=status,
778 version=version,
779 version_string=version_string,
780 reason=reason,
781 decode_content=decode_content,
782 request_url=request_url,
783 retries=retries,
784 )
786 self.enforce_content_length = enforce_content_length
787 self.auto_close = auto_close
789 self._body = None
790 self._uncached_read_occurred = False
791 self._fp: _HttplibHTTPResponse | None = None
792 self._original_response = original_response
793 self._fp_bytes_read = 0
794 self.msg = msg
796 if body and isinstance(body, (str, bytes)):
797 self._body = body
799 self._pool = pool
800 self._connection = connection
802 if hasattr(body, "read"):
803 self._fp = body # type: ignore[assignment]
804 self._sock_shutdown = sock_shutdown
806 # Are we using the chunked-style of transfer encoding?
807 self.chunk_left: int | None = None
809 # Determine length of response
810 self.length_remaining = self._init_length(request_method)
812 # Used to return the correct amount of bytes for partial read()s
813 self._decoded_buffer = BytesQueueBuffer()
815 # If requested, preload the body.
816 if preload_content and not self._body:
817 self._body = self.read(decode_content=decode_content)
819 def release_conn(self) -> None:
820 if not self._pool or not self._connection:
821 return None
823 self._pool._put_conn(self._connection)
824 self._connection = None
826 def drain_conn(self) -> None:
827 """
828 Read and discard any remaining HTTP response data in the response connection.
830 Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
831 """
832 try:
833 while self._raw_read(_READ_CHUNK_SIZE):
834 pass
835 except (HTTPError, OSError, BaseSSLError, HTTPException):
836 pass
837 if self._has_decoded_content:
838 # `_raw_read` skips decompression, so we should clean up the
839 # decoder to avoid keeping unnecessary data in memory.
840 self._decoded_buffer = BytesQueueBuffer()
841 self._decoder = None
843 @property
844 def data(self) -> bytes:
845 # For backwards-compat with earlier urllib3 0.4 and earlier.
846 if self._body:
847 return self._body # type: ignore[return-value]
849 if self._fp:
850 return self.read(cache_content=True)
852 return None # type: ignore[return-value]
854 @property
855 def connection(self) -> HTTPConnection | None:
856 return self._connection
858 def isclosed(self) -> bool:
859 return is_fp_closed(self._fp)
861 def tell(self) -> int:
862 """
863 Obtain the number of bytes pulled over the wire so far. May differ from
864 the amount of content returned by :meth:`HTTPResponse.read`
865 if bytes are encoded on the wire (e.g, compressed).
866 """
867 return self._fp_bytes_read
869 def _init_length(self, request_method: str | None) -> int | None:
870 """
871 Set initial length value for Response content if available.
872 """
873 length: int | None
874 content_length: str | None = self.headers.get("content-length")
876 if content_length is not None:
877 if self.chunked:
878 # This Response will fail with an IncompleteRead if it can't be
879 # received as chunked. This method falls back to attempt reading
880 # the response before raising an exception.
881 log.warning(
882 "Received response with both Content-Length and "
883 "Transfer-Encoding set. This is expressly forbidden "
884 "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
885 "attempting to process response as Transfer-Encoding: "
886 "chunked."
887 )
888 return None
890 try:
891 # RFC 7230 section 3.3.2 specifies multiple content lengths can
892 # be sent in a single Content-Length header
893 # (e.g. Content-Length: 42, 42). This line ensures the values
894 # are all valid ints and that as long as the `set` length is 1,
895 # all values are the same. Otherwise, the header is invalid.
896 lengths = {int(val) for val in content_length.split(",")}
897 if len(lengths) > 1:
898 raise InvalidHeader(
899 "Content-Length contained multiple "
900 "unmatching values (%s)" % content_length
901 )
902 length = lengths.pop()
903 except ValueError:
904 length = None
905 else:
906 if length < 0:
907 length = None
909 else: # if content_length is None
910 length = None
912 # Convert status to int for comparison
913 # In some cases, httplib returns a status of "_UNKNOWN"
914 try:
915 status = int(self.status)
916 except ValueError:
917 status = 0
919 # Check for responses that shouldn't include a body
920 if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
921 length = 0
923 return length
925 @contextmanager
926 def _error_catcher(self) -> typing.Generator[None]:
927 """
928 Catch low-level python exceptions, instead re-raising urllib3
929 variants, so that low-level exceptions are not leaked in the
930 high-level api.
932 On exit, release the connection back to the pool.
933 """
934 clean_exit = False
936 try:
937 try:
938 yield
940 except SocketTimeout as e:
941 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
942 # there is yet no clean way to get at it from this context.
943 raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
945 except BaseSSLError as e:
946 # SSL errors related to framing/MAC get wrapped and reraised here
947 raise SSLError(e) from e
949 except IncompleteRead as e:
950 if (
951 e.expected is not None
952 and e.partial is not None
953 and e.expected == -e.partial
954 ):
955 arg = "Response may not contain content."
956 else:
957 arg = f"Connection broken: {e!r}"
958 raise ProtocolError(arg, e) from e
960 except (HTTPException, OSError) as e:
961 raise ProtocolError(f"Connection broken: {e!r}", e) from e
963 # If no exception is thrown, we should avoid cleaning up
964 # unnecessarily.
965 clean_exit = True
966 finally:
967 # If we didn't terminate cleanly, we need to throw away our
968 # connection.
969 if not clean_exit:
970 # The response may not be closed but we're not going to use it
971 # anymore so close it now to ensure that the connection is
972 # released back to the pool.
973 if self._original_response:
974 self._original_response.close()
976 # Closing the response may not actually be sufficient to close
977 # everything, so if we have a hold of the connection close that
978 # too.
979 if self._connection:
980 self._connection.close()
982 # If we hold the original response but it's closed now, we should
983 # return the connection back to the pool.
984 if self._original_response and self._original_response.isclosed():
985 self.release_conn()
987 def _fp_read(
988 self,
989 amt: int | None = None,
990 *,
991 read1: bool = False,
992 ) -> bytes:
993 """
994 Read a response with the thought that reading the number of bytes
995 larger than can fit in a 32-bit int at a time via SSL in some
996 known cases leads to an overflow error that has to be prevented
997 if `amt` or `self.length_remaining` indicate that a problem may
998 happen.
1000 This happens to urllib3 injected with pyOpenSSL-backed SSL-support.
1001 """
1002 assert self._fp
1003 c_int_max = 2**31 - 1
1004 if (
1005 (amt and amt > c_int_max)
1006 or (
1007 amt is None
1008 and self.length_remaining
1009 and self.length_remaining > c_int_max
1010 )
1011 ) and util.IS_PYOPENSSL:
1012 if read1:
1013 return self._fp.read1(c_int_max)
1014 buffer = io.BytesIO()
1015 # Besides `max_chunk_amt` being a maximum chunk size, it
1016 # affects memory overhead of reading a response by this
1017 # method in CPython.
1018 # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
1019 # chunk size that does not lead to an overflow error, but
1020 # 256 MiB is a compromise.
1021 max_chunk_amt = 2**28
1022 while amt is None or amt != 0:
1023 if amt is not None:
1024 chunk_amt = min(amt, max_chunk_amt)
1025 amt -= chunk_amt
1026 else:
1027 chunk_amt = max_chunk_amt
1028 data = self._fp.read(chunk_amt)
1029 if not data:
1030 break
1031 buffer.write(data)
1032 del data # to reduce peak memory usage by `max_chunk_amt`.
1033 return buffer.getvalue()
1034 elif read1:
1035 return self._fp.read1(amt) if amt is not None else self._fp.read1()
1036 else:
1037 # StringIO doesn't like amt=None
1038 return self._fp.read(amt) if amt is not None else self._fp.read()
1040 def _raw_read(
1041 self,
1042 amt: int | None = None,
1043 *,
1044 read1: bool = False,
1045 ) -> bytes:
1046 """
1047 Reads `amt` of bytes from the socket.
1048 """
1049 if self._fp is None:
1050 return None # type: ignore[return-value]
1052 fp_closed = getattr(self._fp, "closed", False)
1054 with self._error_catcher():
1055 data = self._fp_read(amt, read1=read1) if not fp_closed else b""
1056 if amt is not None and amt != 0 and not data:
1057 # Platform-specific: Buggy versions of Python.
1058 # Close the connection when no data is returned
1059 #
1060 # This is redundant to what httplib/http.client _should_
1061 # already do. However, versions of python released before
1062 # December 15, 2012 (http://bugs.python.org/issue16298) do
1063 # not properly close the connection in all cases. There is
1064 # no harm in redundantly calling close.
1065 self._fp.close()
1066 if (
1067 self.enforce_content_length
1068 and self.length_remaining is not None
1069 and self.length_remaining != 0
1070 ):
1071 # This is an edge case that httplib failed to cover due
1072 # to concerns of backward compatibility. We're
1073 # addressing it here to make sure IncompleteRead is
1074 # raised during streaming, so all calls with incorrect
1075 # Content-Length are caught.
1076 raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
1077 elif read1 and (
1078 (amt != 0 and not data) or self.length_remaining == len(data)
1079 ):
1080 # All data has been read, but `self._fp.read1` in
1081 # CPython 3.12 and older doesn't always close
1082 # `http.client.HTTPResponse`, so we close it here.
1083 # See https://github.com/python/cpython/issues/113199
1084 self._fp.close()
1086 if data:
1087 self._fp_bytes_read += len(data)
1088 if self.length_remaining is not None:
1089 self.length_remaining -= len(data)
1090 return data
1092 def read(
1093 self,
1094 amt: int | None = None,
1095 decode_content: bool | None = None,
1096 cache_content: bool = False,
1097 ) -> bytes:
1098 """
1099 Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
1100 parameters: ``decode_content`` and ``cache_content``.
1102 :param amt:
1103 How much of the content to read. If specified, caching is skipped
1104 because it doesn't make sense to cache partial content as the full
1105 response.
1107 :param decode_content:
1108 If True, will attempt to decode the body based on the
1109 'content-encoding' header.
1111 :param cache_content:
1112 If True, will save the returned data such that the same result is
1113 returned despite of the state of the underlying file object. This
1114 is useful if you want the ``.data`` property to continue working
1115 after having ``.read()`` the file object. (Overridden if ``amt`` is
1116 set.)
1117 """
1118 self._init_decoder()
1119 if decode_content is None:
1120 decode_content = self.decode_content
1122 if amt and amt < 0:
1123 # Negative numbers and `None` should be treated the same.
1124 amt = None
1125 elif amt is not None:
1126 cache_content = False
1128 if (
1129 self._decoder
1130 and self._decoder.has_unconsumed_tail
1131 and len(self._decoded_buffer) < amt
1132 ):
1133 decoded_data = self._decode(
1134 b"",
1135 decode_content,
1136 flush_decoder=False,
1137 max_length=amt - len(self._decoded_buffer),
1138 )
1139 self._decoded_buffer.put(decoded_data)
1140 if len(self._decoded_buffer) >= amt:
1141 return self._decoded_buffer.get(amt)
1143 data = self._raw_read(amt)
1144 if not cache_content:
1145 self._uncached_read_occurred = True
1147 flush_decoder = amt is None or (amt != 0 and not data)
1149 if (
1150 not data
1151 and len(self._decoded_buffer) == 0
1152 and not (self._decoder and self._decoder.has_unconsumed_tail)
1153 ):
1154 return data
1156 if amt is None:
1157 data = self._decode(data, decode_content, flush_decoder)
1158 # It's possible that there is buffered decoded data after a
1159 # partial read.
1160 if decode_content and len(self._decoded_buffer) > 0:
1161 self._decoded_buffer.put(data)
1162 data = self._decoded_buffer.get_all()
1164 if cache_content and not self._uncached_read_occurred:
1165 self._body = data
1166 else:
1167 # do not waste memory on buffer when not decoding
1168 if not decode_content:
1169 if self._has_decoded_content:
1170 raise RuntimeError(
1171 "Calling read(decode_content=False) is not supported after "
1172 "read(decode_content=True) was called."
1173 )
1174 return data
1176 decoded_data = self._decode(
1177 data,
1178 decode_content,
1179 flush_decoder,
1180 max_length=amt - len(self._decoded_buffer),
1181 )
1182 self._decoded_buffer.put(decoded_data)
1184 while len(self._decoded_buffer) < amt and data:
1185 # TODO make sure to initially read enough data to get past the headers
1186 # For example, the GZ file header takes 10 bytes, we don't want to read
1187 # it one byte at a time
1188 data = self._raw_read(amt)
1189 decoded_data = self._decode(
1190 data,
1191 decode_content,
1192 flush_decoder,
1193 max_length=amt - len(self._decoded_buffer),
1194 )
1195 self._decoded_buffer.put(decoded_data)
1196 data = self._decoded_buffer.get(amt)
1198 return data
1200 def read1(
1201 self,
1202 amt: int | None = None,
1203 decode_content: bool | None = None,
1204 ) -> bytes:
1205 """
1206 Similar to ``http.client.HTTPResponse.read1`` and documented
1207 in :meth:`io.BufferedReader.read1`, but with an additional parameter:
1208 ``decode_content``.
1210 :param amt:
1211 How much of the content to read.
1213 :param decode_content:
1214 If True, will attempt to decode the body based on the
1215 'content-encoding' header.
1216 """
1217 if decode_content is None:
1218 decode_content = self.decode_content
1219 if amt and amt < 0:
1220 # Negative numbers and `None` should be treated the same.
1221 amt = None
1222 # try and respond without going to the network
1223 if self._has_decoded_content:
1224 if not decode_content:
1225 raise RuntimeError(
1226 "Calling read1(decode_content=False) is not supported after "
1227 "read1(decode_content=True) was called."
1228 )
1229 if (
1230 self._decoder
1231 and self._decoder.has_unconsumed_tail
1232 and (amt is None or len(self._decoded_buffer) < amt)
1233 ):
1234 decoded_data = self._decode(
1235 b"",
1236 decode_content,
1237 flush_decoder=False,
1238 max_length=(
1239 amt - len(self._decoded_buffer) if amt is not None else None
1240 ),
1241 )
1242 self._decoded_buffer.put(decoded_data)
1243 if len(self._decoded_buffer) > 0:
1244 if amt is None:
1245 return self._decoded_buffer.get_all()
1246 return self._decoded_buffer.get(amt)
1247 if amt == 0:
1248 return b""
1250 # FIXME, this method's type doesn't say returning None is possible
1251 data = self._raw_read(amt, read1=True)
1252 self._uncached_read_occurred = True
1253 if not decode_content or data is None:
1254 return data
1256 self._init_decoder()
1257 while True:
1258 flush_decoder = not data
1259 decoded_data = self._decode(
1260 data, decode_content, flush_decoder, max_length=amt
1261 )
1262 self._decoded_buffer.put(decoded_data)
1263 if decoded_data or flush_decoder:
1264 break
1265 data = self._raw_read(8192, read1=True)
1267 if amt is None:
1268 return self._decoded_buffer.get_all()
1269 return self._decoded_buffer.get(amt)
1271 def stream(
1272 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
1273 ) -> typing.Generator[bytes]:
1274 """
1275 A generator wrapper for the read() method. A call will block until
1276 ``amt`` bytes have been read from the connection or until the
1277 connection is closed.
1279 :param amt:
1280 How much of the content to read. The generator will return up to
1281 much data per iteration, but may return less. This is particularly
1282 likely when using compressed data. However, the empty string will
1283 never be returned.
1285 :param decode_content:
1286 If True, will attempt to decode the body based on the
1287 'content-encoding' header.
1288 """
1289 if amt == 0:
1290 return
1292 if self.chunked and self.supports_chunked_reads():
1293 yield from self.read_chunked(amt, decode_content=decode_content)
1294 else:
1295 while (
1296 not is_fp_closed(self._fp)
1297 or len(self._decoded_buffer) > 0
1298 or (self._decoder and self._decoder.has_unconsumed_tail)
1299 ):
1300 data = self.read(amt=amt, decode_content=decode_content)
1302 if data:
1303 yield data
1305 # Overrides from io.IOBase
1306 def readable(self) -> bool:
1307 return True
1309 def shutdown(self) -> None:
1310 if not self._sock_shutdown:
1311 raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set")
1312 if self._connection is None:
1313 raise RuntimeError(
1314 "Cannot shutdown as connection has already been released to the pool"
1315 )
1316 self._sock_shutdown(socket.SHUT_RD)
1318 def close(self) -> None:
1319 self._sock_shutdown = None
1321 if not self.closed and self._fp:
1322 self._fp.close()
1324 if self._connection:
1325 self._connection.close()
1327 if not self.auto_close:
1328 io.IOBase.close(self)
1330 @property
1331 def closed(self) -> bool:
1332 if not self.auto_close:
1333 return io.IOBase.closed.__get__(self) # type: ignore[no-any-return]
1334 elif self._fp is None:
1335 return True
1336 elif hasattr(self._fp, "isclosed"):
1337 return self._fp.isclosed()
1338 elif hasattr(self._fp, "closed"):
1339 return self._fp.closed
1340 else:
1341 return True
1343 def fileno(self) -> int:
1344 if self._fp is None:
1345 raise OSError("HTTPResponse has no file to get a fileno from")
1346 elif hasattr(self._fp, "fileno"):
1347 return self._fp.fileno()
1348 else:
1349 raise OSError(
1350 "The file-like object this HTTPResponse is wrapped "
1351 "around has no file descriptor"
1352 )
1354 def flush(self) -> None:
1355 if (
1356 self._fp is not None
1357 and hasattr(self._fp, "flush")
1358 and not getattr(self._fp, "closed", False)
1359 ):
1360 return self._fp.flush()
1362 def supports_chunked_reads(self) -> bool:
1363 """
1364 Checks if the underlying file-like object looks like a
1365 :class:`http.client.HTTPResponse` object. We do this by testing for
1366 the fp attribute. If it is present we assume it returns raw chunks as
1367 processed by read_chunked().
1368 """
1369 return hasattr(self._fp, "fp")
1371 def _update_chunk_length(self) -> None:
1372 # First, we'll figure out length of a chunk and then
1373 # we'll try to read it from socket.
1374 if self.chunk_left is not None:
1375 return None
1376 line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1) # type: ignore[union-attr]
1377 if len(line) > _MAX_CHUNK_LINE_LENGTH:
1378 self.close()
1379 raise ProtocolError(
1380 "Response chunk size line exceeded maximum allowed length"
1381 ) from None
1382 line = line.split(b";", 1)[0]
1383 try:
1384 self.chunk_left = int(line, 16)
1385 except ValueError:
1386 self.close()
1387 if line:
1388 # Invalid chunked protocol response, abort.
1389 raise InvalidChunkLength(self, line) from None
1390 else:
1391 # Truncated at start of next chunk
1392 raise ProtocolError("Response ended prematurely") from None
1394 def _handle_chunk(self, amt: int | None) -> bytes:
1395 returned_chunk = None
1396 if amt is None:
1397 chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1398 returned_chunk = chunk
1399 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1400 self.chunk_left = None
1401 elif self.chunk_left is not None and amt < self.chunk_left:
1402 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1403 self.chunk_left = self.chunk_left - amt
1404 returned_chunk = value
1405 elif amt == self.chunk_left:
1406 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1407 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1408 self.chunk_left = None
1409 returned_chunk = value
1410 else: # amt > self.chunk_left
1411 returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1412 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1413 self.chunk_left = None
1414 return returned_chunk # type: ignore[no-any-return]
1416 def read_chunked(
1417 self, amt: int | None = None, decode_content: bool | None = None
1418 ) -> typing.Generator[bytes]:
1419 """
1420 Similar to :meth:`HTTPResponse.read`, but with an additional
1421 parameter: ``decode_content``.
1423 :param amt:
1424 How much of the content to read. If specified, caching is skipped
1425 because it doesn't make sense to cache partial content as the full
1426 response.
1428 :param decode_content:
1429 If True, will attempt to decode the body based on the
1430 'content-encoding' header.
1431 """
1432 self._init_decoder()
1433 # FIXME: Rewrite this method and make it a class with a better structured logic.
1434 if not self.chunked:
1435 raise ResponseNotChunked(
1436 "Response is not chunked. "
1437 "Header 'transfer-encoding: chunked' is missing."
1438 )
1439 if not self.supports_chunked_reads():
1440 raise BodyNotHttplibCompatible(
1441 "Body should be http.client.HTTPResponse like. "
1442 "It should have have an fp attribute which returns raw chunks."
1443 )
1445 with self._error_catcher():
1446 # Don't bother reading the body of a HEAD request.
1447 if self._original_response and is_response_to_head(self._original_response):
1448 self._original_response.close()
1449 return None
1451 # If a response is already read and closed
1452 # then return immediately.
1453 if self._fp.fp is None: # type: ignore[union-attr]
1454 return None
1456 if amt == 0:
1457 return
1458 elif amt and amt < 0:
1459 # Negative numbers and `None` should be treated the same,
1460 # but httplib handles only `None` correctly.
1461 amt = None
1463 while True:
1464 # First, check if any data is left in the decoder's buffer.
1465 if self._decoder and self._decoder.has_unconsumed_tail:
1466 chunk = b""
1467 else:
1468 self._update_chunk_length()
1469 self._uncached_read_occurred = True
1470 if self.chunk_left == 0:
1471 break
1472 chunk = self._handle_chunk(amt)
1473 decoded = self._decode(
1474 chunk,
1475 decode_content=decode_content,
1476 flush_decoder=False,
1477 max_length=amt,
1478 )
1479 if decoded:
1480 yield decoded
1482 if decode_content:
1483 # On CPython and PyPy, we should never need to flush the
1484 # decoder. However, on Jython we *might* need to, so
1485 # lets defensively do it anyway.
1486 decoded = self._flush_decoder()
1487 if decoded: # Platform-specific: Jython.
1488 yield decoded
1490 # Chunk content ends with \r\n: discard it.
1491 while self._fp is not None:
1492 line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1)
1493 if len(line) > _MAX_CHUNK_LINE_LENGTH:
1494 raise ProtocolError(
1495 "Response chunk trailer line exceeded maximum allowed length"
1496 )
1497 if not line:
1498 # Some sites may not end with '\r\n'.
1499 break
1500 if line == b"\r\n":
1501 break
1503 # We read everything; close the "file".
1504 if self._original_response:
1505 self._original_response.close()
1507 @property
1508 def url(self) -> str | None:
1509 """
1510 Returns the URL that was the source of this response.
1511 If the request that generated this response redirected, this method
1512 will return the final redirect location.
1513 """
1514 return self._request_url
1516 @url.setter
1517 def url(self, url: str | None) -> None:
1518 self._request_url = url
1520 def __iter__(self) -> typing.Iterator[bytes]:
1521 buffer: list[bytes] = []
1522 for chunk in self.stream(decode_content=True):
1523 if b"\n" in chunk:
1524 chunks = chunk.split(b"\n")
1525 yield b"".join(buffer) + chunks[0] + b"\n"
1526 for x in chunks[1:-1]:
1527 yield x + b"\n"
1528 if chunks[-1]:
1529 buffer = [chunks[-1]]
1530 else:
1531 buffer = []
1532 else:
1533 buffer.append(chunk)
1534 if buffer:
1535 yield b"".join(buffer)