Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/urllib3/response.py: 19%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import collections
4import io
5import json as _json
6import logging
7import re
8import socket
9import sys
10import typing
11import warnings
12import zlib
13from contextlib import contextmanager
14from http.client import HTTPMessage as _HttplibHTTPMessage
15from http.client import HTTPResponse as _HttplibHTTPResponse
16from socket import timeout as SocketTimeout
18if typing.TYPE_CHECKING:
19 from ._base_connection import BaseHTTPConnection
21try:
22 try:
23 import brotlicffi as brotli # type: ignore[import-not-found]
24 except ImportError:
25 import brotli # type: ignore[import-not-found]
26except ImportError:
27 brotli = None
29from . import util
30from ._base_connection import _TYPE_BODY
31from ._collections import HTTPHeaderDict
32from .connection import BaseSSLError, HTTPConnection, HTTPException
33from .exceptions import (
34 BodyNotHttplibCompatible,
35 DecodeError,
36 DependencyWarning,
37 HTTPError,
38 IncompleteRead,
39 InvalidChunkLength,
40 InvalidHeader,
41 ProtocolError,
42 ReadTimeoutError,
43 ResponseNotChunked,
44 SSLError,
45)
46from .util.response import is_fp_closed, is_response_to_head
47from .util.retry import Retry
49if typing.TYPE_CHECKING:
50 from .connectionpool import HTTPConnectionPool
52log = logging.getLogger(__name__)
54# Read in 64 KiB chunks
55_READ_CHUNK_SIZE = 2**16
57# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG.
58# int() is more permissive, so we match against the grammar before calling it.
59_CONTENT_LENGTH_RE = re.compile(r"[0-9]+")
60_CHUNK_SIZE_RE = re.compile(rb"[0-9A-Fa-f]+")
62# Maximum length of a chunk-size line (and the trailing CRLF line) when reading a
63# chunked-transfer-encoded response. Matches ``http.client._MAXLINE`` so urllib3's
64# streaming path bounds these reads exactly like the stdlib ``.read()`` path does,
65# preventing a malicious server from forcing unbounded buffering (memory exhaustion)
66# via an unterminated chunk-size line.
67_MAX_CHUNK_LINE_LENGTH = 2**16
70class ContentDecoder:
71 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
72 raise NotImplementedError()
74 @property
75 def has_unconsumed_tail(self) -> bool:
76 raise NotImplementedError()
78 def flush(self) -> bytes:
79 raise NotImplementedError()
82class DeflateDecoder(ContentDecoder):
83 def __init__(self) -> None:
84 self._first_try = True
85 self._first_try_data = b""
86 self._unfed_data = b""
87 self._obj = zlib.decompressobj()
89 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
90 data = self._unfed_data + data
91 self._unfed_data = b""
92 # Data received after EOF cannot produce more output from this
93 # Deflate stream.
94 if self._obj.eof:
95 return b""
96 if not data and not self._obj.unconsumed_tail:
97 return data
98 original_max_length = max_length
99 if original_max_length < 0:
100 max_length = 0
101 elif original_max_length == 0:
102 # We should not pass 0 to the zlib decompressor because 0 is
103 # the default value that will make zlib decompress without a
104 # length limit.
105 # Data should be stored for subsequent calls.
106 self._unfed_data = data
107 return b""
109 # Subsequent calls always reuse `self._obj`. zlib requires
110 # passing the unconsumed tail if decompression is to continue.
111 if not self._first_try:
112 return self._obj.decompress(
113 self._obj.unconsumed_tail + data, max_length=max_length
114 )
116 # First call tries with RFC 1950 ZLIB format.
117 self._first_try_data += data
118 try:
119 decompressed = self._obj.decompress(data, max_length=max_length)
120 if decompressed:
121 self._first_try = False
122 self._first_try_data = b""
123 return decompressed
124 # On failure, it falls back to RFC 1951 DEFLATE format.
125 except zlib.error:
126 self._first_try = False
127 self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
128 try:
129 return self.decompress(
130 self._first_try_data, max_length=original_max_length
131 )
132 finally:
133 self._first_try_data = b""
135 @property
136 def has_unconsumed_tail(self) -> bool:
137 return bool(self._unfed_data) or (
138 bool(self._obj.unconsumed_tail)
139 and not self._first_try
140 and not self._obj.eof
141 )
143 def flush(self) -> bytes:
144 return self._obj.flush()
147class GzipDecoderState:
148 FIRST_MEMBER = 0
149 OTHER_MEMBERS = 1
150 SWALLOW_DATA = 2
153class GzipDecoder(ContentDecoder):
154 def __init__(self) -> None:
155 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
156 self._state = GzipDecoderState.FIRST_MEMBER
157 self._unconsumed_tail = b""
159 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
160 if self._state == GzipDecoderState.SWALLOW_DATA:
161 return b""
163 if max_length == 0:
164 # We should not pass 0 to the zlib decompressor because 0 is
165 # the default value that will make zlib decompress without a
166 # length limit.
167 # Data should be stored for subsequent calls.
168 self._unconsumed_tail += data
169 return b""
171 # zlib requires passing the unconsumed tail to the subsequent
172 # call if decompression is to continue.
173 data = self._unconsumed_tail + data
174 if not data and self._obj.eof:
175 return b""
177 ret: bytes | bytearray = b""
178 while True:
179 try:
180 chunk = self._obj.decompress(
181 data, max_length=max(max_length - len(ret), 0)
182 )
183 if not ret:
184 ret = chunk
185 elif chunk:
186 if isinstance(ret, bytes):
187 ret = bytearray(ret)
188 ret += chunk
189 del chunk
190 except zlib.error:
191 previous_state = self._state
192 # Ignore data after the first error
193 self._state = GzipDecoderState.SWALLOW_DATA
194 self._unconsumed_tail = b""
195 if previous_state == GzipDecoderState.OTHER_MEMBERS:
196 # Allow trailing garbage acceptable in other gzip clients
197 return ret if isinstance(ret, bytes) else bytes(ret)
198 raise
200 self._unconsumed_tail = data = (
201 self._obj.unconsumed_tail or self._obj.unused_data
202 )
203 # When the end of a gzip member is reached, a new decompressor
204 # must be created for unused (possibly future) data.
205 if data and self._obj.eof:
206 self._state = GzipDecoderState.OTHER_MEMBERS
207 self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
209 if max_length > 0 and len(ret) >= max_length:
210 break
212 if not data:
213 return ret if isinstance(ret, bytes) else bytes(ret)
215 return ret if isinstance(ret, bytes) else bytes(ret)
217 @property
218 def has_unconsumed_tail(self) -> bool:
219 return bool(self._unconsumed_tail)
221 def flush(self) -> bytes:
222 if self._state == GzipDecoderState.SWALLOW_DATA:
223 return b""
224 return self._obj.flush()
227if brotli is not None:
229 class BrotliDecoder(ContentDecoder):
230 # Supports both 'brotlipy' and 'Brotli' packages
231 # since they share an import name. The top branches
232 # are for 'brotlipy' and bottom branches for 'Brotli'
233 def __init__(self) -> None:
234 self._obj = brotli.Decompressor()
235 if hasattr(self._obj, "decompress"):
236 setattr(self, "_decompress", self._obj.decompress)
237 else:
238 setattr(self, "_decompress", self._obj.process)
240 # Requires Brotli >= 1.2.0 for `output_buffer_limit`.
241 def _decompress(self, data: bytes, output_buffer_limit: int = -1) -> bytes:
242 raise NotImplementedError()
244 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
245 try:
246 if max_length > 0:
247 return self._decompress(data, output_buffer_limit=max_length)
248 else:
249 return self._decompress(data)
250 except TypeError:
251 # Fallback for Brotli/brotlicffi/brotlipy versions without
252 # the `output_buffer_limit` parameter.
253 warnings.warn(
254 "Brotli >= 1.2.0 is required to prevent decompression bombs.",
255 DependencyWarning,
256 )
257 return self._decompress(data)
259 @property
260 def has_unconsumed_tail(self) -> bool:
261 try:
262 return not self._obj.can_accept_more_data()
263 except AttributeError:
264 return False
266 def flush(self) -> bytes:
267 if hasattr(self._obj, "flush"):
268 return self._obj.flush() # type: ignore[no-any-return]
269 return b""
272try:
273 if sys.version_info >= (3, 14):
274 from compression import zstd
275 else:
276 from backports import zstd
277except ImportError:
278 HAS_ZSTD = False
279else:
280 HAS_ZSTD = True
282 class ZstdDecoder(ContentDecoder):
283 def __init__(self) -> None:
284 self._obj = zstd.ZstdDecompressor()
286 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
287 if not data and not self.has_unconsumed_tail:
288 return b""
289 if self._obj.eof:
290 data = self._obj.unused_data + data
291 self._obj = zstd.ZstdDecompressor()
292 part = self._obj.decompress(data, max_length=max_length)
293 length = len(part)
294 data_parts = [part]
295 # Every loop iteration is supposed to read data from a separate frame.
296 # The loop breaks when:
297 # - enough data is read;
298 # - no more unused data is available;
299 # - end of the last read frame has not been reached (i.e.,
300 # more data has to be fed).
301 while (
302 self._obj.eof
303 and self._obj.unused_data
304 and (max_length < 0 or length < max_length)
305 ):
306 unused_data = self._obj.unused_data
307 if not self._obj.needs_input:
308 self._obj = zstd.ZstdDecompressor()
309 part = self._obj.decompress(
310 unused_data,
311 max_length=(max_length - length) if max_length > 0 else -1,
312 )
313 if part_length := len(part):
314 data_parts.append(part)
315 length += part_length
316 elif self._obj.needs_input:
317 break
318 return b"".join(data_parts)
320 @property
321 def has_unconsumed_tail(self) -> bool:
322 return not (self._obj.needs_input or self._obj.eof) or bool(
323 self._obj.unused_data
324 )
326 def flush(self) -> bytes:
327 if not self._obj.eof:
328 raise DecodeError("Zstandard data is incomplete")
329 return b""
332class MultiDecoder(ContentDecoder):
333 """
334 From RFC7231:
335 If one or more encodings have been applied to a representation, the
336 sender that applied the encodings MUST generate a Content-Encoding
337 header field that lists the content codings in the order in which
338 they were applied.
339 """
341 # Maximum allowed number of chained HTTP encodings in the
342 # Content-Encoding header.
343 max_decode_links = 5
345 def __init__(self, modes: str) -> None:
346 encodings = [m.strip() for m in modes.split(",")]
347 if len(encodings) > self.max_decode_links:
348 raise DecodeError(
349 "Too many content encodings in the chain: "
350 f"{len(encodings)} > {self.max_decode_links}"
351 )
352 self._decoders = [_get_decoder(e) for e in encodings]
354 def flush(self) -> bytes:
355 return self._decoders[0].flush()
357 def decompress(self, data: bytes, max_length: int = -1) -> bytes:
358 if max_length <= 0:
359 for d in reversed(self._decoders):
360 data = d.decompress(data)
361 return data
363 ret = bytearray()
364 # Every while loop iteration goes through all decoders once.
365 # It exits when enough data is read or no more data can be read.
366 # It is possible that the while loop iteration does not produce
367 # any data because we retrieve up to `max_length` from every
368 # decoder, and the amount of bytes may be insufficient for the
369 # next decoder to produce enough/any output.
370 while True:
371 any_data = False
372 for d in reversed(self._decoders):
373 data = d.decompress(data, max_length=max_length - len(ret))
374 if data:
375 any_data = True
376 # We should not break when no data is returned because
377 # next decoders may produce data even with empty input.
378 ret += data
379 if not any_data or len(ret) >= max_length:
380 return bytes(ret)
381 data = b""
383 @property
384 def has_unconsumed_tail(self) -> bool:
385 return any(d.has_unconsumed_tail for d in self._decoders)
388def _get_decoder(mode: str) -> ContentDecoder:
389 if "," in mode:
390 return MultiDecoder(mode)
392 # According to RFC 9110 section 8.4.1.3, recipients should
393 # consider x-gzip equivalent to gzip
394 if mode in ("gzip", "x-gzip"):
395 return GzipDecoder()
397 if brotli is not None and mode == "br":
398 return BrotliDecoder()
400 if HAS_ZSTD and mode == "zstd":
401 return ZstdDecoder()
403 return DeflateDecoder()
406class BytesQueueBuffer:
407 """Memory-efficient bytes buffer
409 To return decoded data in read() and still follow the BufferedIOBase API, we need a
410 buffer to always return the correct amount of bytes.
412 This buffer should be filled using calls to put()
414 Our maximum memory usage is determined by the sum of the size of:
416 * self.buffer, which contains the full data
417 * the largest chunk that we will copy in get()
418 """
420 def __init__(self) -> None:
421 self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque()
422 self._size: int = 0
424 def __len__(self) -> int:
425 return self._size
427 def put(self, data: bytes) -> None:
428 self.buffer.append(data)
429 self._size += len(data)
431 def get(self, n: int) -> bytes:
432 if n == 0:
433 return b""
434 elif not self.buffer:
435 raise RuntimeError("buffer is empty")
436 elif n < 0:
437 raise ValueError("n should be > 0")
439 if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes):
440 self._size -= n
441 return self.buffer.popleft()
443 fetched = 0
444 ret = io.BytesIO()
445 while fetched < n:
446 remaining = n - fetched
447 chunk = self.buffer.popleft()
448 chunk_length = len(chunk)
449 if remaining < chunk_length:
450 chunk = memoryview(chunk)
451 left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
452 ret.write(left_chunk)
453 self.buffer.appendleft(right_chunk)
454 self._size -= remaining
455 break
456 else:
457 ret.write(chunk)
458 self._size -= chunk_length
459 fetched += chunk_length
461 if not self.buffer:
462 break
464 return ret.getvalue()
466 def get_all(self) -> bytes:
467 buffer = self.buffer
468 if not buffer:
469 assert self._size == 0
470 return b""
471 if len(buffer) == 1:
472 result = buffer.pop()
473 if isinstance(result, memoryview):
474 result = result.tobytes()
475 else:
476 ret = io.BytesIO()
477 ret.writelines(buffer.popleft() for _ in range(len(buffer)))
478 result = ret.getvalue()
479 self._size = 0
480 return result
483class BaseHTTPResponse(io.IOBase):
484 CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
485 if brotli is not None:
486 CONTENT_DECODERS += ["br"]
487 if HAS_ZSTD:
488 CONTENT_DECODERS += ["zstd"]
489 REDIRECT_STATUSES = [301, 302, 303, 307, 308]
491 DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
492 if brotli is not None:
493 DECODER_ERROR_CLASSES += (brotli.error,)
495 if HAS_ZSTD:
496 DECODER_ERROR_CLASSES += (zstd.ZstdError,)
498 def __init__(
499 self,
500 *,
501 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
502 status: int,
503 version: int,
504 version_string: str,
505 reason: str | None,
506 decode_content: bool,
507 request_url: str | None,
508 retries: Retry | None = None,
509 ) -> None:
510 if isinstance(headers, HTTPHeaderDict):
511 self.headers = headers
512 else:
513 self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type]
514 self.status = status
515 self.version = version
516 self.version_string = version_string
517 self.reason = reason
518 self.decode_content = decode_content
519 self._has_decoded_content = False
520 self._request_url: str | None = request_url
521 self.retries = retries
523 self.chunked = False
524 tr_enc = self.headers.get("transfer-encoding", "").lower()
525 # Don't incur the penalty of creating a list and then discarding it
526 encodings = (enc.strip() for enc in tr_enc.split(","))
527 if "chunked" in encodings:
528 self.chunked = True
530 self._decoder: ContentDecoder | None = None
531 # Distinguish an uninitialized decoder from a response needing no decoder.
532 self._decoder_initialized = False
533 self.length_remaining: int | None
535 def get_redirect_location(self) -> str | None | typing.Literal[False]:
536 """
537 Should we redirect and where to?
539 :returns: Truthy redirect location string if we got a redirect status
540 code and valid location. ``None`` if redirect status and no
541 location. ``False`` if not a redirect status code.
542 """
543 if self.status in self.REDIRECT_STATUSES:
544 return self.headers.get("location")
545 return False
547 @property
548 def data(self) -> bytes:
549 raise NotImplementedError()
551 def json(self) -> typing.Any:
552 """
553 Deserializes the body of the HTTP response as a Python object.
555 The body of the HTTP response must be encoded using UTF-8, as per
556 `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
558 To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
559 your custom decoder instead.
561 If the body of the HTTP response is not decodable to UTF-8, a
562 `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
563 valid JSON document, a `json.JSONDecodeError` will be raised.
565 Read more :ref:`here <json_content>`.
567 :returns: The body of the HTTP response as a Python object.
568 """
569 data = self.data.decode("utf-8")
570 return _json.loads(data)
572 @property
573 def url(self) -> str | None:
574 raise NotImplementedError()
576 @url.setter
577 def url(self, url: str | None) -> None:
578 raise NotImplementedError()
580 @property
581 def connection(self) -> BaseHTTPConnection | None:
582 raise NotImplementedError()
584 @property
585 def retries(self) -> Retry | None:
586 return self._retries
588 @retries.setter
589 def retries(self, retries: Retry | None) -> None:
590 # Override the request_url if retries has a redirect location.
591 if retries is not None and retries.history:
592 self.url = retries.history[-1].redirect_location
593 self._retries = retries
595 def stream(
596 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
597 ) -> typing.Iterator[bytes]:
598 raise NotImplementedError()
600 def read(
601 self,
602 amt: int | None = None,
603 decode_content: bool | None = None,
604 cache_content: bool = False,
605 ) -> bytes:
606 raise NotImplementedError()
608 def read1(
609 self,
610 amt: int | None = None,
611 decode_content: bool | None = None,
612 ) -> bytes:
613 raise NotImplementedError()
615 def read_chunked(
616 self,
617 amt: int | None = None,
618 decode_content: bool | None = None,
619 ) -> typing.Iterator[bytes]:
620 raise NotImplementedError()
622 def release_conn(self) -> None:
623 raise NotImplementedError()
625 def drain_conn(self) -> None:
626 raise NotImplementedError()
628 def shutdown(self) -> None:
629 raise NotImplementedError()
631 def close(self) -> None:
632 raise NotImplementedError()
634 def _init_decoder(self) -> None:
635 """
636 Set-up the _decoder attribute if necessary.
637 """
638 if self._decoder_initialized:
639 return
641 if self._decoder is None:
642 # Note: content-encoding value should be case-insensitive, per RFC 7230
643 # Section 3.2
644 content_encoding = self.headers.get("content-encoding", "").lower()
645 if content_encoding in self.CONTENT_DECODERS:
646 self._decoder = _get_decoder(content_encoding)
647 elif "," in content_encoding:
648 encodings = [
649 e.strip()
650 for e in content_encoding.split(",")
651 if e.strip() in self.CONTENT_DECODERS
652 ]
653 if encodings:
654 self._decoder = _get_decoder(content_encoding)
656 self._decoder_initialized = True
658 def _decode(
659 self,
660 data: bytes,
661 decode_content: bool | None,
662 flush_decoder: bool,
663 max_length: int | None = None,
664 ) -> bytes:
665 """
666 Decode the data passed in and potentially flush the decoder.
667 """
668 if not decode_content:
669 if self._has_decoded_content:
670 raise RuntimeError(
671 "Calling read(decode_content=False) is not supported after "
672 "read(decode_content=True) was called."
673 )
674 return data
676 if max_length is None or flush_decoder:
677 max_length = -1
679 try:
680 if self._decoder:
681 data = self._decoder.decompress(data, max_length=max_length)
682 self._has_decoded_content = True
683 except self.DECODER_ERROR_CLASSES as e:
684 content_encoding = self.headers.get("content-encoding", "").lower()
685 raise DecodeError(
686 "Received response with content-encoding: %s, but "
687 "failed to decode it." % content_encoding,
688 e,
689 ) from e
690 if flush_decoder:
691 data += self._flush_decoder()
693 return data
695 def _flush_decoder(self) -> bytes:
696 """
697 Flushes the decoder. Should only be called if the decoder is actually
698 being used.
699 """
700 if self._decoder:
701 return self._decoder.decompress(b"") + self._decoder.flush()
702 return b""
704 # Compatibility methods for `io` module
705 def readinto(self, b: bytearray | memoryview[int]) -> int:
706 temp = self.read(len(b))
707 if len(temp) == 0:
708 return 0
709 else:
710 b[: len(temp)] = temp
711 return len(temp)
713 # Methods used by dependent libraries
714 def getheaders(self) -> HTTPHeaderDict:
715 return self.headers
717 def getheader(self, name: str, default: str | None = None) -> str | None:
718 return self.headers.get(name, default)
720 # Compatibility method for http.cookiejar
721 def info(self) -> HTTPHeaderDict:
722 return self.headers
724 def geturl(self) -> str | None:
725 return self.url
728class HTTPResponse(BaseHTTPResponse):
729 """
730 HTTP Response container.
732 Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
733 loaded and decoded on-demand when the ``data`` property is accessed. This
734 class is also compatible with the Python standard library's :mod:`io`
735 module, and can hence be treated as a readable object in the context of that
736 framework.
738 Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
740 :param preload_content:
741 If True, the response's body will be preloaded during construction.
743 :param decode_content:
744 If True, will attempt to decode the body based on the
745 'content-encoding' header.
747 :param original_response:
748 When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
749 object, it's convenient to include the original for debug purposes. It's
750 otherwise unused.
752 :param retries:
753 The retries contains the last :class:`~urllib3.util.retry.Retry` that
754 was used during the request.
756 :param enforce_content_length:
757 Enforce content length checking. Body returned by server must match
758 value of Content-Length header, if present. Otherwise, raise error.
759 """
761 def __init__(
762 self,
763 body: _TYPE_BODY = "",
764 headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
765 status: int = 0,
766 version: int = 0,
767 version_string: str = "HTTP/?",
768 reason: str | None = None,
769 preload_content: bool = True,
770 decode_content: bool = True,
771 original_response: _HttplibHTTPResponse | None = None,
772 pool: HTTPConnectionPool | None = None,
773 connection: HTTPConnection | None = None,
774 msg: _HttplibHTTPMessage | None = None,
775 retries: Retry | None = None,
776 enforce_content_length: bool = True,
777 request_method: str | None = None,
778 request_url: str | None = None,
779 auto_close: bool = True,
780 sock_shutdown: typing.Callable[[int], None] | None = None,
781 ) -> None:
782 super().__init__(
783 headers=headers,
784 status=status,
785 version=version,
786 version_string=version_string,
787 reason=reason,
788 decode_content=decode_content,
789 request_url=request_url,
790 retries=retries,
791 )
793 self.enforce_content_length = enforce_content_length
794 self.auto_close = auto_close
796 self._body = None
797 self._uncached_read_occurred = False
798 self._fp: _HttplibHTTPResponse | None = None
799 self._original_response = original_response
800 self._fp_bytes_read = 0
801 self.msg = msg
803 if body and isinstance(body, (str, bytes)):
804 self._body = body
806 self._pool = pool
807 self._connection = connection
809 if hasattr(body, "read"):
810 self._fp = body # type: ignore[assignment]
811 self._sock_shutdown = sock_shutdown
813 # Are we using the chunked-style of transfer encoding?
814 self.chunk_left: int | None = None
816 # Determine length of response
817 self.length_remaining = self._init_length(request_method)
819 # Used to return the correct amount of bytes for partial read()s
820 self._decoded_buffer = BytesQueueBuffer()
822 # If requested, preload the body.
823 if preload_content and not self._body:
824 self._body = self.read(decode_content=decode_content)
826 def release_conn(self) -> None:
827 if not self._pool or not self._connection:
828 return None
830 self._pool._put_conn(self._connection)
831 self._connection = None
833 def drain_conn(self) -> None:
834 """
835 Read and discard any remaining HTTP response data in the response connection.
837 Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
838 """
839 try:
840 while self._raw_read(_READ_CHUNK_SIZE):
841 pass
842 except (HTTPError, OSError, BaseSSLError, HTTPException):
843 pass
844 if self._has_decoded_content:
845 # `_raw_read` skips decompression, so we should clean up the
846 # decoder to avoid keeping unnecessary data in memory.
847 self._decoded_buffer = BytesQueueBuffer()
848 self._decoder = None
850 @property
851 def data(self) -> bytes:
852 # For backwards-compat with earlier urllib3 0.4 and earlier.
853 if self._body:
854 return self._body # type: ignore[return-value]
856 if self._fp:
857 return self.read(cache_content=True)
859 return None # type: ignore[return-value]
861 @property
862 def connection(self) -> HTTPConnection | None:
863 return self._connection
865 def isclosed(self) -> bool:
866 return is_fp_closed(self._fp)
868 def tell(self) -> int:
869 """
870 Obtain the number of bytes pulled over the wire so far. May differ from
871 the amount of content returned by :meth:`HTTPResponse.read`
872 if bytes are encoded on the wire (e.g, compressed).
873 """
874 return self._fp_bytes_read
876 def _init_length(self, request_method: str | None) -> int | None:
877 """
878 Set initial length value for Response content if available.
879 """
880 length: int | None
881 content_length: str | None = self.headers.get("content-length")
883 if content_length is not None:
884 if self.chunked:
885 # This Response will fail with an IncompleteRead if it can't be
886 # received as chunked. This method falls back to attempt reading
887 # the response before raising an exception.
888 log.warning(
889 "Received response with both Content-Length and "
890 "Transfer-Encoding set. This is expressly forbidden "
891 "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
892 "attempting to process response as Transfer-Encoding: "
893 "chunked."
894 )
895 return None
897 try:
898 # RFC 7230 section 3.3.2 specifies multiple content lengths can
899 # be sent in a single Content-Length header
900 # (e.g. Content-Length: 42, 42). This line ensures the values
901 # are all valid ints and that as long as the `set` length is 1,
902 # all values are the same. Otherwise, the header is invalid.
903 lengths = set()
904 for val in content_length.split(","):
905 # Only optional whitespace may surround each value; reject
906 # signs, underscores and unicode digits that int() accepts.
907 val = val.strip()
908 if not _CONTENT_LENGTH_RE.fullmatch(val):
909 raise ValueError("invalid Content-Length")
910 lengths.add(int(val))
911 if len(lengths) > 1:
912 raise InvalidHeader(
913 "Content-Length contained multiple "
914 "unmatching values (%s)" % content_length
915 )
916 length = lengths.pop()
917 except ValueError:
918 length = None
920 else: # if content_length is None
921 length = None
923 # Convert status to int for comparison
924 # In some cases, httplib returns a status of "_UNKNOWN"
925 try:
926 status = int(self.status)
927 except ValueError:
928 status = 0
930 # Check for responses that shouldn't include a body
931 if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
932 length = 0
934 return length
936 @contextmanager
937 def _error_catcher(self) -> typing.Generator[None]:
938 """
939 Catch low-level python exceptions, instead re-raising urllib3
940 variants, so that low-level exceptions are not leaked in the
941 high-level api.
943 On exit, release the connection back to the pool.
944 """
945 clean_exit = False
947 try:
948 try:
949 yield
951 except SocketTimeout as e:
952 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
953 # there is yet no clean way to get at it from this context.
954 raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
956 except BaseSSLError as e:
957 # SSL errors related to framing/MAC get wrapped and reraised here
958 raise SSLError(e) from e
960 except IncompleteRead as e:
961 if (
962 e.expected is not None
963 and e.partial is not None
964 and e.expected == -e.partial
965 ):
966 arg = "Response may not contain content."
967 else:
968 arg = f"Connection broken: {e!r}"
969 raise ProtocolError(arg, e) from e
971 except (HTTPException, OSError) as e:
972 raise ProtocolError(f"Connection broken: {e!r}", e) from e
974 # If no exception is thrown, we should avoid cleaning up
975 # unnecessarily.
976 clean_exit = True
977 finally:
978 # If we didn't terminate cleanly, we need to throw away our
979 # connection.
980 if not clean_exit:
981 # The response may not be closed but we're not going to use it
982 # anymore so close it now to ensure that the connection is
983 # released back to the pool.
984 if self._original_response:
985 self._original_response.close()
987 # Closing the response may not actually be sufficient to close
988 # everything, so if we have a hold of the connection close that
989 # too.
990 if self._connection:
991 self._connection.close()
993 # If we hold the original response but it's closed now, we should
994 # return the connection back to the pool.
995 if self._original_response and self._original_response.isclosed():
996 self.release_conn()
998 def _fp_read(
999 self,
1000 amt: int | None = None,
1001 *,
1002 read1: bool = False,
1003 ) -> bytes:
1004 """
1005 Read a response with the thought that reading the number of bytes
1006 larger than can fit in a 32-bit int at a time via SSL in some
1007 known cases leads to an overflow error that has to be prevented
1008 if `amt` or `self.length_remaining` indicate that a problem may
1009 happen.
1011 This happens to urllib3 injected with pyOpenSSL-backed SSL-support.
1012 """
1013 assert self._fp
1014 c_int_max = 2**31 - 1
1015 if (
1016 (amt and amt > c_int_max)
1017 or (
1018 amt is None
1019 and self.length_remaining
1020 and self.length_remaining > c_int_max
1021 )
1022 ) and util.IS_PYOPENSSL:
1023 if read1:
1024 return self._fp.read1(c_int_max)
1025 buffer = io.BytesIO()
1026 # Besides `max_chunk_amt` being a maximum chunk size, it
1027 # affects memory overhead of reading a response by this
1028 # method in CPython.
1029 # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
1030 # chunk size that does not lead to an overflow error, but
1031 # 256 MiB is a compromise.
1032 max_chunk_amt = 2**28
1033 while amt is None or amt != 0:
1034 if amt is not None:
1035 chunk_amt = min(amt, max_chunk_amt)
1036 amt -= chunk_amt
1037 else:
1038 chunk_amt = max_chunk_amt
1039 data = self._fp.read(chunk_amt)
1040 if not data:
1041 break
1042 buffer.write(data)
1043 del data # to reduce peak memory usage by `max_chunk_amt`.
1044 return buffer.getvalue()
1045 elif read1:
1046 return self._fp.read1(amt) if amt is not None else self._fp.read1()
1047 else:
1048 # StringIO doesn't like amt=None
1049 return self._fp.read(amt) if amt is not None else self._fp.read()
1051 def _raw_read(
1052 self,
1053 amt: int | None = None,
1054 *,
1055 read1: bool = False,
1056 ) -> bytes:
1057 """
1058 Reads `amt` of bytes from the socket.
1059 """
1060 if self._fp is None:
1061 return None # type: ignore[return-value]
1063 fp_closed = getattr(self._fp, "closed", False)
1065 with self._error_catcher():
1066 data = self._fp_read(amt, read1=read1) if not fp_closed else b""
1067 if amt is not None and amt != 0 and not data:
1068 # Platform-specific: Buggy versions of Python.
1069 # Close the connection when no data is returned
1070 #
1071 # This is redundant to what httplib/http.client _should_
1072 # already do. However, versions of python released before
1073 # December 15, 2012 (http://bugs.python.org/issue16298) do
1074 # not properly close the connection in all cases. There is
1075 # no harm in redundantly calling close.
1076 self._fp.close()
1077 if (
1078 self.enforce_content_length
1079 and self.length_remaining is not None
1080 and self.length_remaining != 0
1081 ):
1082 # This is an edge case that httplib failed to cover due
1083 # to concerns of backward compatibility. We're
1084 # addressing it here to make sure IncompleteRead is
1085 # raised during streaming, so all calls with incorrect
1086 # Content-Length are caught.
1087 raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
1088 elif read1 and (
1089 (amt != 0 and not data) or self.length_remaining == len(data)
1090 ):
1091 # All data has been read, but `self._fp.read1` in
1092 # CPython 3.12 and older doesn't always close
1093 # `http.client.HTTPResponse`, so we close it here.
1094 # See https://github.com/python/cpython/issues/113199
1095 self._fp.close()
1097 if data:
1098 self._fp_bytes_read += len(data)
1099 if self.length_remaining is not None:
1100 self.length_remaining -= len(data)
1101 return data
1103 def read(
1104 self,
1105 amt: int | None = None,
1106 decode_content: bool | None = None,
1107 cache_content: bool = False,
1108 ) -> bytes:
1109 """
1110 Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
1111 parameters: ``decode_content`` and ``cache_content``.
1113 :param amt:
1114 How much of the content to read. If specified, caching is skipped
1115 because it doesn't make sense to cache partial content as the full
1116 response.
1118 :param decode_content:
1119 If True, will attempt to decode the body based on the
1120 'content-encoding' header.
1122 :param cache_content:
1123 If True, will save the returned data such that the same result is
1124 returned despite of the state of the underlying file object. This
1125 is useful if you want the ``.data`` property to continue working
1126 after having ``.read()`` the file object. (Overridden if ``amt`` is
1127 set.)
1128 """
1129 self._init_decoder()
1130 if decode_content is None:
1131 decode_content = self.decode_content
1133 if amt and amt < 0:
1134 # Negative numbers and `None` should be treated the same.
1135 amt = None
1136 elif amt is not None:
1137 cache_content = False
1139 if (
1140 self._decoder
1141 and self._decoder.has_unconsumed_tail
1142 and len(self._decoded_buffer) < amt
1143 ):
1144 decoded_data = self._decode(
1145 b"",
1146 decode_content,
1147 flush_decoder=False,
1148 max_length=amt - len(self._decoded_buffer),
1149 )
1150 self._decoded_buffer.put(decoded_data)
1151 if len(self._decoded_buffer) >= amt:
1152 return self._decoded_buffer.get(amt)
1154 data = self._raw_read(amt)
1155 if not cache_content:
1156 self._uncached_read_occurred = True
1158 flush_decoder = amt is None or (amt != 0 and not data)
1160 if (
1161 not data
1162 and len(self._decoded_buffer) == 0
1163 and not (self._decoder and self._decoder.has_unconsumed_tail)
1164 and not (decode_content and self._has_decoded_content)
1165 ):
1166 return data
1168 if amt is None:
1169 data = self._decode(data, decode_content, flush_decoder)
1170 # It's possible that there is buffered decoded data after a
1171 # partial read.
1172 if decode_content and len(self._decoded_buffer) > 0:
1173 self._decoded_buffer.put(data)
1174 data = self._decoded_buffer.get_all()
1176 if cache_content and not self._uncached_read_occurred and data:
1177 self._body = data
1178 else:
1179 # do not waste memory on buffer when not decoding
1180 if not decode_content:
1181 if self._has_decoded_content:
1182 raise RuntimeError(
1183 "Calling read(decode_content=False) is not supported after "
1184 "read(decode_content=True) was called."
1185 )
1186 return data
1188 decoded_data = self._decode(
1189 data,
1190 decode_content,
1191 flush_decoder,
1192 max_length=amt - len(self._decoded_buffer),
1193 )
1194 self._decoded_buffer.put(decoded_data)
1196 while len(self._decoded_buffer) < amt and data:
1197 # TODO make sure to initially read enough data to get past the headers
1198 # For example, the GZ file header takes 10 bytes, we don't want to read
1199 # it one byte at a time
1200 data = self._raw_read(amt)
1201 flush_decoder = amt != 0 and not data
1202 decoded_data = self._decode(
1203 data,
1204 decode_content,
1205 flush_decoder,
1206 max_length=amt - len(self._decoded_buffer),
1207 )
1208 self._decoded_buffer.put(decoded_data)
1209 data = self._decoded_buffer.get(amt)
1211 return data
1213 def read1(
1214 self,
1215 amt: int | None = None,
1216 decode_content: bool | None = None,
1217 ) -> bytes:
1218 """
1219 Similar to ``http.client.HTTPResponse.read1`` and documented
1220 in :meth:`io.BufferedReader.read1`, but with an additional parameter:
1221 ``decode_content``.
1223 :param amt:
1224 How much of the content to read.
1226 :param decode_content:
1227 If True, will attempt to decode the body based on the
1228 'content-encoding' header.
1229 """
1230 if decode_content is None:
1231 decode_content = self.decode_content
1232 if amt and amt < 0:
1233 # Negative numbers and `None` should be treated the same.
1234 amt = None
1235 # try and respond without going to the network
1236 if self._has_decoded_content:
1237 if not decode_content:
1238 raise RuntimeError(
1239 "Calling read1(decode_content=False) is not supported after "
1240 "read1(decode_content=True) was called."
1241 )
1242 if (
1243 self._decoder
1244 and self._decoder.has_unconsumed_tail
1245 and (amt is None or len(self._decoded_buffer) < amt)
1246 ):
1247 decoded_data = self._decode(
1248 b"",
1249 decode_content,
1250 flush_decoder=False,
1251 max_length=(
1252 amt - len(self._decoded_buffer) if amt is not None else None
1253 ),
1254 )
1255 self._decoded_buffer.put(decoded_data)
1256 if len(self._decoded_buffer) > 0:
1257 if amt is None:
1258 return self._decoded_buffer.get_all()
1259 return self._decoded_buffer.get(amt)
1260 if amt == 0:
1261 return b""
1263 # FIXME, this method's type doesn't say returning None is possible
1264 data = self._raw_read(amt, read1=True)
1265 self._uncached_read_occurred = True
1266 if not decode_content or data is None:
1267 return data
1269 self._init_decoder()
1270 while True:
1271 flush_decoder = not data
1272 decoded_data = self._decode(
1273 data, decode_content, flush_decoder, max_length=amt
1274 )
1275 self._decoded_buffer.put(decoded_data)
1276 if decoded_data or flush_decoder:
1277 break
1278 data = self._raw_read(8192, read1=True)
1280 if amt is None:
1281 return self._decoded_buffer.get_all()
1282 return self._decoded_buffer.get(amt)
1284 def stream(
1285 self, amt: int | None = _READ_CHUNK_SIZE, decode_content: bool | None = None
1286 ) -> typing.Generator[bytes]:
1287 """
1288 A generator wrapper for the read() method. A call will block until
1289 ``amt`` bytes have been read from the connection or until the
1290 connection is closed.
1292 :param amt:
1293 How much of the content to read. The generator will return up to
1294 much data per iteration, but may return less. This is particularly
1295 likely when using compressed data. However, the empty string will
1296 never be returned.
1298 :param decode_content:
1299 If True, will attempt to decode the body based on the
1300 'content-encoding' header.
1301 """
1302 if amt == 0:
1303 return
1305 if self.chunked and self.supports_chunked_reads():
1306 yield from self.read_chunked(amt, decode_content=decode_content)
1307 else:
1308 while (
1309 not is_fp_closed(self._fp)
1310 or len(self._decoded_buffer) > 0
1311 or (self._decoder and self._decoder.has_unconsumed_tail)
1312 or self._has_decoded_content
1313 ):
1314 data = self.read(amt=amt, decode_content=decode_content)
1316 if not data:
1317 break
1318 yield data
1320 # Overrides from io.IOBase
1321 def readable(self) -> bool:
1322 return True
1324 def shutdown(self) -> None:
1325 if not self._sock_shutdown:
1326 raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set")
1327 if self._connection is None:
1328 raise RuntimeError(
1329 "Cannot shutdown as connection has already been released to the pool"
1330 )
1331 self._sock_shutdown(socket.SHUT_RD)
1333 def close(self) -> None:
1334 self._sock_shutdown = None
1336 if not self.closed and self._fp:
1337 self._fp.close()
1339 if self._connection:
1340 self._connection.close()
1342 if not self.auto_close:
1343 io.IOBase.close(self)
1345 @property
1346 def closed(self) -> bool:
1347 if not self.auto_close:
1348 return io.IOBase.closed.__get__(self) # type: ignore[no-any-return]
1349 elif self._fp is None:
1350 return True
1351 elif hasattr(self._fp, "isclosed"):
1352 return self._fp.isclosed()
1353 elif hasattr(self._fp, "closed"):
1354 return self._fp.closed
1355 else:
1356 return True
1358 def fileno(self) -> int:
1359 if self._fp is None:
1360 raise OSError("HTTPResponse has no file to get a fileno from")
1361 elif hasattr(self._fp, "fileno"):
1362 return self._fp.fileno()
1363 else:
1364 raise OSError(
1365 "The file-like object this HTTPResponse is wrapped "
1366 "around has no file descriptor"
1367 )
1369 def flush(self) -> None:
1370 if (
1371 self._fp is not None
1372 and hasattr(self._fp, "flush")
1373 and not getattr(self._fp, "closed", False)
1374 ):
1375 return self._fp.flush()
1377 def supports_chunked_reads(self) -> bool:
1378 """
1379 Checks if the underlying file-like object looks like a
1380 :class:`http.client.HTTPResponse` object. We do this by testing for
1381 the fp attribute. If it is present we assume it returns raw chunks as
1382 processed by read_chunked().
1383 """
1384 return hasattr(self._fp, "fp")
1386 def _update_chunk_length(self) -> None:
1387 # First, we'll figure out length of a chunk and then
1388 # we'll try to read it from socket.
1389 if self.chunk_left is not None:
1390 return None
1391 line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1) # type: ignore[union-attr]
1392 if len(line) > _MAX_CHUNK_LINE_LENGTH:
1393 self.close()
1394 raise ProtocolError(
1395 "Response chunk size line exceeded maximum allowed length"
1396 ) from None
1397 line = line.split(b";", 1)[0]
1398 try:
1399 chunk_size = line.rstrip()
1400 if not _CHUNK_SIZE_RE.fullmatch(chunk_size):
1401 raise ValueError("invalid chunk size")
1402 self.chunk_left = int(chunk_size, 16)
1403 except ValueError:
1404 self.close()
1405 if line:
1406 # Invalid chunked protocol response, abort.
1407 raise InvalidChunkLength(self, line) from None
1408 else:
1409 # Truncated at start of next chunk
1410 raise ProtocolError("Response ended prematurely") from None
1412 def _handle_chunk(self, amt: int | None) -> bytes:
1413 returned_chunk = None
1414 if amt is None:
1415 chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1416 returned_chunk = chunk
1417 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1418 self.chunk_left = None
1419 elif self.chunk_left is not None and amt < self.chunk_left:
1420 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1421 self.chunk_left = self.chunk_left - amt
1422 returned_chunk = value
1423 elif amt == self.chunk_left:
1424 value = self._fp._safe_read(amt) # type: ignore[union-attr]
1425 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1426 self.chunk_left = None
1427 returned_chunk = value
1428 else: # amt > self.chunk_left
1429 returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
1430 self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
1431 self.chunk_left = None
1432 return returned_chunk # type: ignore[no-any-return]
1434 def read_chunked(
1435 self, amt: int | None = None, decode_content: bool | None = None
1436 ) -> typing.Generator[bytes]:
1437 """
1438 Similar to :meth:`HTTPResponse.read`, but with an additional
1439 parameter: ``decode_content``.
1441 :param amt:
1442 How much of the content to read. If specified, caching is skipped
1443 because it doesn't make sense to cache partial content as the full
1444 response.
1446 :param decode_content:
1447 If True, will attempt to decode the body based on the
1448 'content-encoding' header.
1449 """
1450 self._init_decoder()
1451 # FIXME: Rewrite this method and make it a class with a better structured logic.
1452 if not self.chunked:
1453 raise ResponseNotChunked(
1454 "Response is not chunked. "
1455 "Header 'transfer-encoding: chunked' is missing."
1456 )
1457 if not self.supports_chunked_reads():
1458 raise BodyNotHttplibCompatible(
1459 "Body should be http.client.HTTPResponse like. "
1460 "It should have have an fp attribute which returns raw chunks."
1461 )
1463 with self._error_catcher():
1464 # Don't bother reading the body of a HEAD request.
1465 if self._original_response and is_response_to_head(self._original_response):
1466 self._original_response.close()
1467 return None
1469 # If a response is already read and closed
1470 # then return immediately.
1471 if self._fp.fp is None: # type: ignore[union-attr]
1472 return None
1474 if amt == 0:
1475 return
1476 elif amt and amt < 0:
1477 # Negative numbers and `None` should be treated the same,
1478 # but httplib handles only `None` correctly.
1479 amt = None
1481 while True:
1482 # First, check if any data is left in the decoder's buffer.
1483 if self._decoder and self._decoder.has_unconsumed_tail:
1484 chunk = b""
1485 else:
1486 self._update_chunk_length()
1487 self._uncached_read_occurred = True
1488 if self.chunk_left == 0:
1489 break
1490 chunk = self._handle_chunk(amt)
1491 decoded = self._decode(
1492 chunk,
1493 decode_content=decode_content,
1494 flush_decoder=False,
1495 max_length=amt,
1496 )
1497 if decoded:
1498 yield decoded
1500 if decode_content:
1501 # On CPython and PyPy, we should never need to flush the
1502 # decoder. However, on Jython we *might* need to, so
1503 # lets defensively do it anyway.
1504 decoded = self._flush_decoder()
1505 if decoded: # Platform-specific: Jython.
1506 yield decoded
1508 # Chunk content ends with \r\n: discard it.
1509 while self._fp is not None:
1510 line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1)
1511 if len(line) > _MAX_CHUNK_LINE_LENGTH:
1512 raise ProtocolError(
1513 "Response chunk trailer line exceeded maximum allowed length"
1514 )
1515 if not line:
1516 # Some sites may not end with '\r\n'.
1517 break
1518 if line == b"\r\n":
1519 break
1521 # We read everything; close the "file".
1522 if self._original_response:
1523 self._original_response.close()
1525 @property
1526 def url(self) -> str | None:
1527 """
1528 Returns the URL that was the source of this response.
1529 If the request that generated this response redirected, this method
1530 will return the final redirect location.
1531 """
1532 return self._request_url
1534 @url.setter
1535 def url(self, url: str | None) -> None:
1536 self._request_url = url
1538 def __iter__(self) -> typing.Iterator[bytes]:
1539 buffer: list[bytes] = []
1540 for chunk in self.stream(decode_content=True):
1541 if b"\n" in chunk:
1542 chunks = chunk.split(b"\n")
1543 yield b"".join(buffer) + chunks[0] + b"\n"
1544 for x in chunks[1:-1]:
1545 yield x + b"\n"
1546 if chunks[-1]:
1547 buffer = [chunks[-1]]
1548 else:
1549 buffer = []
1550 else:
1551 buffer.append(chunk)
1552 if buffer:
1553 yield b"".join(buffer)