Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from collections.abc import Generator
2from typing import Literal, NoReturn, TypeAlias, cast
3from urllib.parse import urlsplit
5from pip._vendor import requests, urllib3
6from pip._vendor.requests.models import Response
7from pip._vendor.urllib3.exceptions import TimeoutStateError
9from pip._internal.exceptions import (
10 ConnectionFailedError,
11 ConnectionTimeoutError,
12 NetworkConnectionError,
13 ProxyConnectionError,
14 SSLVerificationError,
15)
16from pip._internal.utils.misc import redact_auth_from_url
18TimeoutValue: TypeAlias = (
19 float | tuple[float | None, float | None] | urllib3.util.Timeout | None
20)
22# The following comments and HTTP headers were originally added by
23# Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
24#
25# We use Accept-Encoding: identity here because requests defaults to
26# accepting compressed responses. This breaks in a variety of ways
27# depending on how the server is configured.
28# - Some servers will notice that the file isn't a compressible file
29# and will leave the file alone and with an empty Content-Encoding
30# - Some servers will notice that the file is already compressed and
31# will leave the file alone, adding a Content-Encoding: gzip header
32# - Some servers won't notice anything at all and will take a file
33# that's already been compressed and compress it again, and set
34# the Content-Encoding: gzip header
35# By setting this to request only the identity encoding we're hoping
36# to eliminate the third case. Hopefully there does not exist a server
37# which when given a file will notice it is already compressed and that
38# you're not asking for a compressed file and will then decompress it
39# before sending because if that's the case I don't think it'll ever be
40# possible to make this work.
41HEADERS: dict[str, str] = {"Accept-Encoding": "identity"}
43DOWNLOAD_CHUNK_SIZE = 256 * 1024
46def raise_for_status(resp: Response) -> None:
47 http_error_msg = ""
48 if isinstance(resp.reason, bytes):
49 # We attempt to decode utf-8 first because some servers
50 # choose to localize their reason strings. If the string
51 # isn't utf-8, we fall back to iso-8859-1 for all other
52 # encodings.
53 try:
54 reason = resp.reason.decode("utf-8")
55 except UnicodeDecodeError:
56 reason = resp.reason.decode("iso-8859-1")
57 else:
58 reason = resp.reason
60 if 400 <= resp.status_code < 500:
61 http_error_msg = (
62 f"{resp.status_code} Client Error: {reason} for url: {resp.url}"
63 )
65 elif 500 <= resp.status_code < 600:
66 http_error_msg = (
67 f"{resp.status_code} Server Error: {reason} for url: {resp.url}"
68 )
70 if http_error_msg:
71 raise NetworkConnectionError(http_error_msg, response=resp)
74def response_chunks(
75 response: Response, chunk_size: int = DOWNLOAD_CHUNK_SIZE
76) -> Generator[bytes, None, None]:
77 """Given a requests Response, provide the data chunks."""
78 try:
79 # Special case for urllib3.
80 for chunk in response.raw.stream(
81 chunk_size,
82 # We use decode_content=False here because we don't
83 # want urllib3 to mess with the raw bytes we get
84 # from the server. If we decompress inside of
85 # urllib3 then we cannot verify the checksum
86 # because the checksum will be of the compressed
87 # file. This breakage will only occur if the
88 # server adds a Content-Encoding header, which
89 # depends on how the server was configured:
90 # - Some servers will notice that the file isn't a
91 # compressible file and will leave the file alone
92 # and with an empty Content-Encoding
93 # - Some servers will notice that the file is
94 # already compressed and will leave the file
95 # alone and will add a Content-Encoding: gzip
96 # header
97 # - Some servers won't notice anything at all and
98 # will take a file that's already been compressed
99 # and compress it again and set the
100 # Content-Encoding: gzip header
101 #
102 # By setting this not to decode automatically we
103 # hope to eliminate problems with the second case.
104 decode_content=False,
105 ):
106 yield chunk
107 except AttributeError:
108 # Standard file-like object.
109 while True:
110 chunk = response.raw.read(chunk_size)
111 if not chunk:
112 break
113 yield chunk
116def _parse_timeout(timeout: TimeoutValue, kind: Literal["connect", "read"]) -> float:
117 connect_timeout: float | None
118 read_timeout: float | None
119 if isinstance(timeout, tuple):
120 connect_timeout, read_timeout = timeout
121 elif isinstance(timeout, urllib3.util.Timeout):
122 connect_timeout = cast(float | None, timeout.connect_timeout)
123 try:
124 read_timeout = timeout.read_timeout
125 except TimeoutStateError:
126 read_timeout = None
127 else:
128 connect_timeout = read_timeout = timeout
130 if kind == "connect":
131 assert connect_timeout is not None
132 return connect_timeout
133 else:
134 assert read_timeout is not None
135 return read_timeout
138def _raise_timeout_error(
139 reason: urllib3.exceptions.TimeoutError,
140 url: str,
141 host: str,
142 timeout: TimeoutValue,
143) -> NoReturn:
144 if isinstance(reason, urllib3.exceptions.ConnectTimeoutError):
145 raise ConnectionTimeoutError(
146 url, host, kind="connect", timeout=_parse_timeout(timeout, "connect")
147 )
148 else:
149 raise ConnectionTimeoutError(
150 url, host, kind="read", timeout=_parse_timeout(timeout, "read")
151 )
154def raise_connection_error(
155 error: requests.ConnectionError | requests.Timeout,
156 *,
157 url: str,
158 timeout: TimeoutValue,
159) -> NoReturn:
160 """Raise a specific error for a given connection error, if possible.
162 Note: requests.ConnectionError is the parent class of
163 requests.ProxyError, requests.SSLError, and requests.ConnectTimeout
164 so these errors are also handled here.
165 """
166 url = redact_auth_from_url(url)
167 raw_hostname = urlsplit(url).hostname or urlsplit(url).netloc
168 reason = error.args[0] if error.args else error
170 # NewConnectionError is a subclass of TimeoutError for some reason...
171 if isinstance(reason, urllib3.exceptions.TimeoutError) and not isinstance(
172 reason, urllib3.exceptions.NewConnectionError
173 ):
174 # A bare timeout error can occur during non-streamed responses. Don't
175 # ask me how.
176 _raise_timeout_error(reason, url, raw_hostname, timeout)
177 if isinstance(reason, urllib3.exceptions.SSLError):
178 # A bare SSL error can occur during non-streamed responses, after the
179 # initial connection and TLS handshake have completed.
180 raise SSLVerificationError(url, raw_hostname, reason)
182 # At this point, all errors should be wrapped in MaxRetryError.
183 if not isinstance(reason, urllib3.exceptions.MaxRetryError):
184 raise ConnectionFailedError(url, raw_hostname, reason)
186 max_retry_error = reason
187 assert isinstance(max_retry_error.pool, urllib3.connectionpool.HTTPConnectionPool)
188 host = max_retry_error.pool.host
189 proxy = max_retry_error.pool.proxy
190 # Narrow the reason further to the specific error from the last retry.
191 reason = max_retry_error.reason
193 if isinstance(reason, urllib3.exceptions.SSLError):
194 raise SSLVerificationError(url, host, reason)
195 if isinstance(reason, urllib3.exceptions.TimeoutError) and not isinstance(
196 reason, urllib3.exceptions.NewConnectionError
197 ):
198 _raise_timeout_error(reason, url, host, timeout)
199 if isinstance(reason, urllib3.exceptions.ProxyError):
200 assert proxy is not None
201 raise ProxyConnectionError(url, redact_auth_from_url(str(proxy)), reason)
203 # Unknown error, give up and raise a generic error.
204 raise ConnectionFailedError(
205 url, host, reason if isinstance(reason, Exception) else max_retry_error
206 )