Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/urllib3/contrib/appengine.py: 30%

1""" 

2This module provides a pool manager that uses Google App Engine's 

3`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 

4 

5Example usage:: 

6 

7 from urllib3 import PoolManager 

8 from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox 

9 

10 if is_appengine_sandbox(): 

11 # AppEngineManager uses AppEngine's URLFetch API behind the scenes 

12 http = AppEngineManager() 

13 else: 

14 # PoolManager uses a socket-level API behind the scenes 

15 http = PoolManager() 

16 

17 r = http.request('GET', 'https://google.com/') 

18 

19There are `limitations <https://cloud.google.com/appengine/docs/python/\ 

20urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be 

21the best choice for your application. There are three options for using 

22urllib3 on Google App Engine: 

23 

241. You can use :class:`AppEngineManager` with URLFetch. URLFetch is 

25 cost-effective in many circumstances as long as your usage is within the 

26 limitations. 

272. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. 

28 Sockets also have `limitations and restrictions 

29 <https://cloud.google.com/appengine/docs/python/sockets/\ 

30 #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. 

31 To use sockets, be sure to specify the following in your ``app.yaml``:: 

32 

33 env_variables: 

34 GAE_USE_SOCKETS_HTTPLIB : 'true' 

35 

363. If you are using `App Engine Flexible 

37<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard 

38:class:`PoolManager` without any configuration or special environment variables. 

39""" 

40 

41from __future__ import absolute_import 

42 

43import io 

44import logging 

45import warnings 

46 

47from ..exceptions import ( 

48 HTTPError, 

49 HTTPWarning, 

50 MaxRetryError, 

51 ProtocolError, 

52 SSLError, 

53 TimeoutError, 

54) 

55from ..packages.six.moves.urllib.parse import urljoin 

56from ..request import RequestMethods 

57from ..response import HTTPResponse 

58from ..util.retry import Retry 

59from ..util.timeout import Timeout 

60from . import _appengine_environ 

61 

62try: 

63 from google.appengine.api import urlfetch 

64except ImportError: 

65 urlfetch = None 

66 

67 

68log = logging.getLogger(__name__) 

69 

70 

71class AppEnginePlatformWarning(HTTPWarning): 

72 pass 

73 

74 

75class AppEnginePlatformError(HTTPError): 

76 pass 

77 

78 

79class AppEngineManager(RequestMethods): 

80 """ 

81 Connection manager for Google App Engine sandbox applications. 

82 

83 This manager uses the URLFetch service directly instead of using the 

84 emulated httplib, and is subject to URLFetch limitations as described in 

85 the App Engine documentation `here 

86 <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 

87 

88 Notably it will raise an :class:`AppEnginePlatformError` if: 

89 * URLFetch is not available. 

90 * If you attempt to use this on App Engine Flexible, as full socket 

91 support is available. 

92 * If a request size is more than 10 megabytes. 

93 * If a response size is more than 32 megabytes. 

94 * If you use an unsupported request method such as OPTIONS. 

95 

96 Beyond those cases, it will raise normal urllib3 errors. 

97 """ 

98 

99 def __init__( 

100 self, 

101 headers=None, 

102 retries=None, 

103 validate_certificate=True, 

104 urlfetch_retries=True, 

105 ): 

106 if not urlfetch: 

107 raise AppEnginePlatformError( 

108 "URLFetch is not available in this environment." 

109 ) 

110 

111 warnings.warn( 

112 "urllib3 is using URLFetch on Google App Engine sandbox instead " 

113 "of sockets. To use sockets directly instead of URLFetch see " 

114 "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", 

115 AppEnginePlatformWarning, 

116 ) 

117 

118 RequestMethods.__init__(self, headers) 

119 self.validate_certificate = validate_certificate 

120 self.urlfetch_retries = urlfetch_retries 

121 

122 self.retries = retries or Retry.DEFAULT 

123 

124 def __enter__(self): 

125 return self 

126 

127 def __exit__(self, exc_type, exc_val, exc_tb): 

128 # Return False to re-raise any potential exceptions 

129 return False 

130 

131 def urlopen( 

132 self, 

133 method, 

134 url, 

135 body=None, 

136 headers=None, 

137 retries=None, 

138 redirect=True, 

139 timeout=Timeout.DEFAULT_TIMEOUT, 

140 **response_kw 

141 ): 

142 

143 retries = self._get_retries(retries, redirect) 

144 

145 try: 

146 follow_redirects = redirect and retries.redirect != 0 and retries.total 

147 response = urlfetch.fetch( 

148 url, 

149 payload=body, 

150 method=method, 

151 headers=headers or {}, 

152 allow_truncated=False, 

153 follow_redirects=self.urlfetch_retries and follow_redirects, 

154 deadline=self._get_absolute_timeout(timeout), 

155 validate_certificate=self.validate_certificate, 

156 ) 

157 except urlfetch.DeadlineExceededError as e: 

158 raise TimeoutError(self, e) 

159 

160 except urlfetch.InvalidURLError as e: 

161 if "too large" in str(e): 

162 raise AppEnginePlatformError( 

163 "URLFetch request too large, URLFetch only " 

164 "supports requests up to 10mb in size.", 

165 e, 

166 ) 

167 raise ProtocolError(e) 

168 

169 except urlfetch.DownloadError as e: 

170 if "Too many redirects" in str(e): 

171 raise MaxRetryError(self, url, reason=e) 

172 raise ProtocolError(e) 

173 

174 except urlfetch.ResponseTooLargeError as e: 

175 raise AppEnginePlatformError( 

176 "URLFetch response too large, URLFetch only supports" 

177 "responses up to 32mb in size.", 

178 e, 

179 ) 

180 

181 except urlfetch.SSLCertificateError as e: 

182 raise SSLError(e) 

183 

184 except urlfetch.InvalidMethodError as e: 

185 raise AppEnginePlatformError( 

186 "URLFetch does not support method: %s" % method, e 

187 ) 

188 

189 http_response = self._urlfetch_response_to_http_response( 

190 response, retries=retries, **response_kw 

191 ) 

192 

193 # Handle redirect? 

194 redirect_location = redirect and http_response.get_redirect_location() 

195 if redirect_location: 

196 # Check for redirect response 

197 if self.urlfetch_retries and retries.raise_on_redirect: 

198 raise MaxRetryError(self, url, "too many redirects") 

199 else: 

200 if http_response.status == 303: 

201 method = "GET" 

202 

203 try: 

204 retries = retries.increment( 

205 method, url, response=http_response, _pool=self 

206 ) 

207 except MaxRetryError: 

208 if retries.raise_on_redirect: 

209 raise MaxRetryError(self, url, "too many redirects") 

210 return http_response 

211 

212 retries.sleep_for_retry(http_response) 

213 log.debug("Redirecting %s -> %s", url, redirect_location) 

214 redirect_url = urljoin(url, redirect_location) 

215 return self.urlopen( 

216 method, 

217 redirect_url, 

218 body, 

219 headers, 

220 retries=retries, 

221 redirect=redirect, 

222 timeout=timeout, 

223 **response_kw 

224 ) 

225 

226 # Check if we should retry the HTTP response. 

227 has_retry_after = bool(http_response.headers.get("Retry-After")) 

228 if retries.is_retry(method, http_response.status, has_retry_after): 

229 retries = retries.increment(method, url, response=http_response, _pool=self) 

230 log.debug("Retry: %s", url) 

231 retries.sleep(http_response) 

232 return self.urlopen( 

233 method, 

234 url, 

235 body=body, 

236 headers=headers, 

237 retries=retries, 

238 redirect=redirect, 

239 timeout=timeout, 

240 **response_kw 

241 ) 

242 

243 return http_response 

244 

245 def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): 

246 

247 if is_prod_appengine(): 

248 # Production GAE handles deflate encoding automatically, but does 

249 # not remove the encoding header. 

250 content_encoding = urlfetch_resp.headers.get("content-encoding") 

251 

252 if content_encoding == "deflate": 

253 del urlfetch_resp.headers["content-encoding"] 

254 

255 transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") 

256 # We have a full response's content, 

257 # so let's make sure we don't report ourselves as chunked data. 

258 if transfer_encoding == "chunked": 

259 encodings = transfer_encoding.split(",") 

260 encodings.remove("chunked") 

261 urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) 

262 

263 original_response = HTTPResponse( 

264 # In order for decoding to work, we must present the content as 

265 # a file-like object. 

266 body=io.BytesIO(urlfetch_resp.content), 

267 msg=urlfetch_resp.header_msg, 

268 headers=urlfetch_resp.headers, 

269 status=urlfetch_resp.status_code, 

270 **response_kw 

271 ) 

272 

273 return HTTPResponse( 

274 body=io.BytesIO(urlfetch_resp.content), 

275 headers=urlfetch_resp.headers, 

276 status=urlfetch_resp.status_code, 

277 original_response=original_response, 

278 **response_kw 

279 ) 

280 

281 def _get_absolute_timeout(self, timeout): 

282 if timeout is Timeout.DEFAULT_TIMEOUT: 

283 return None # Defer to URLFetch's default. 

284 if isinstance(timeout, Timeout): 

285 if timeout._read is not None or timeout._connect is not None: 

286 warnings.warn( 

287 "URLFetch does not support granular timeout settings, " 

288 "reverting to total or default URLFetch timeout.", 

289 AppEnginePlatformWarning, 

290 ) 

291 return timeout.total 

292 return timeout 

293 

294 def _get_retries(self, retries, redirect): 

295 if not isinstance(retries, Retry): 

296 retries = Retry.from_int(retries, redirect=redirect, default=self.retries) 

297 

298 if retries.connect or retries.read or retries.redirect: 

299 warnings.warn( 

300 "URLFetch only supports total retries and does not " 

301 "recognize connect, read, or redirect retry parameters.", 

302 AppEnginePlatformWarning, 

303 ) 

304 

305 return retries 

306 

307 

308# Alias methods from _appengine_environ to maintain public API interface. 

309 

310is_appengine = _appengine_environ.is_appengine 

311is_appengine_sandbox = _appengine_environ.is_appengine_sandbox 

312is_local_appengine = _appengine_environ.is_local_appengine 

313is_prod_appengine = _appengine_environ.is_prod_appengine 

314is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms