Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tornado/httpclient.py: 26%

238 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-01 06:54 +0000

1"""Blocking and non-blocking HTTP client interfaces. 

2 

3This module defines a common interface shared by two implementations, 

4``simple_httpclient`` and ``curl_httpclient``. Applications may either 

5instantiate their chosen implementation class directly or use the 

6`AsyncHTTPClient` class from this module, which selects an implementation 

7that can be overridden with the `AsyncHTTPClient.configure` method. 

8 

9The default implementation is ``simple_httpclient``, and this is expected 

10to be suitable for most users' needs. However, some applications may wish 

11to switch to ``curl_httpclient`` for reasons such as the following: 

12 

13* ``curl_httpclient`` has some features not found in ``simple_httpclient``, 

14 including support for HTTP proxies and the ability to use a specified 

15 network interface. 

16 

17* ``curl_httpclient`` is more likely to be compatible with sites that are 

18 not-quite-compliant with the HTTP spec, or sites that use little-exercised 

19 features of HTTP. 

20 

21* ``curl_httpclient`` is faster. 

22 

23Note that if you are using ``curl_httpclient``, it is highly 

24recommended that you use a recent version of ``libcurl`` and 

25``pycurl``. Currently the minimum supported version of libcurl is 

267.22.0, and the minimum version of pycurl is 7.18.2. It is highly 

27recommended that your ``libcurl`` installation is built with 

28asynchronous DNS resolver (threaded or c-ares), otherwise you may 

29encounter various problems with request timeouts (for more 

30information, see 

31http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS 

32and comments in curl_httpclient.py). 

33 

34To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup:: 

35 

36 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") 

37""" 

38 

39import datetime 

40import functools 

41from io import BytesIO 

42import ssl 

43import time 

44import weakref 

45 

46from tornado.concurrent import ( 

47 Future, 

48 future_set_result_unless_cancelled, 

49 future_set_exception_unless_cancelled, 

50) 

51from tornado.escape import utf8, native_str 

52from tornado import gen, httputil 

53from tornado.ioloop import IOLoop 

54from tornado.util import Configurable 

55 

56from typing import Type, Any, Union, Dict, Callable, Optional, cast 

57 

58 

59class HTTPClient(object): 

60 """A blocking HTTP client. 

61 

62 This interface is provided to make it easier to share code between 

63 synchronous and asynchronous applications. Applications that are 

64 running an `.IOLoop` must use `AsyncHTTPClient` instead. 

65 

66 Typical usage looks like this:: 

67 

68 http_client = httpclient.HTTPClient() 

69 try: 

70 response = http_client.fetch("http://www.google.com/") 

71 print(response.body) 

72 except httpclient.HTTPError as e: 

73 # HTTPError is raised for non-200 responses; the response 

74 # can be found in e.response. 

75 print("Error: " + str(e)) 

76 except Exception as e: 

77 # Other errors are possible, such as IOError. 

78 print("Error: " + str(e)) 

79 http_client.close() 

80 

81 .. versionchanged:: 5.0 

82 

83 Due to limitations in `asyncio`, it is no longer possible to 

84 use the synchronous ``HTTPClient`` while an `.IOLoop` is running. 

85 Use `AsyncHTTPClient` instead. 

86 

87 """ 

88 

89 def __init__( 

90 self, 

91 async_client_class: "Optional[Type[AsyncHTTPClient]]" = None, 

92 **kwargs: Any 

93 ) -> None: 

94 # Initialize self._closed at the beginning of the constructor 

95 # so that an exception raised here doesn't lead to confusing 

96 # failures in __del__. 

97 self._closed = True 

98 self._io_loop = IOLoop(make_current=False) 

99 if async_client_class is None: 

100 async_client_class = AsyncHTTPClient 

101 

102 # Create the client while our IOLoop is "current", without 

103 # clobbering the thread's real current IOLoop (if any). 

104 async def make_client() -> "AsyncHTTPClient": 

105 await gen.sleep(0) 

106 assert async_client_class is not None 

107 return async_client_class(**kwargs) 

108 

109 self._async_client = self._io_loop.run_sync(make_client) 

110 self._closed = False 

111 

112 def __del__(self) -> None: 

113 self.close() 

114 

115 def close(self) -> None: 

116 """Closes the HTTPClient, freeing any resources used.""" 

117 if not self._closed: 

118 self._async_client.close() 

119 self._io_loop.close() 

120 self._closed = True 

121 

122 def fetch( 

123 self, request: Union["HTTPRequest", str], **kwargs: Any 

124 ) -> "HTTPResponse": 

125 """Executes a request, returning an `HTTPResponse`. 

126 

127 The request may be either a string URL or an `HTTPRequest` object. 

128 If it is a string, we construct an `HTTPRequest` using any additional 

129 kwargs: ``HTTPRequest(request, **kwargs)`` 

130 

131 If an error occurs during the fetch, we raise an `HTTPError` unless 

132 the ``raise_error`` keyword argument is set to False. 

133 """ 

134 response = self._io_loop.run_sync( 

135 functools.partial(self._async_client.fetch, request, **kwargs) 

136 ) 

137 return response 

138 

139 

140class AsyncHTTPClient(Configurable): 

141 """An non-blocking HTTP client. 

142 

143 Example usage:: 

144 

145 async def f(): 

146 http_client = AsyncHTTPClient() 

147 try: 

148 response = await http_client.fetch("http://www.google.com") 

149 except Exception as e: 

150 print("Error: %s" % e) 

151 else: 

152 print(response.body) 

153 

154 The constructor for this class is magic in several respects: It 

155 actually creates an instance of an implementation-specific 

156 subclass, and instances are reused as a kind of pseudo-singleton 

157 (one per `.IOLoop`). The keyword argument ``force_instance=True`` 

158 can be used to suppress this singleton behavior. Unless 

159 ``force_instance=True`` is used, no arguments should be passed to 

160 the `AsyncHTTPClient` constructor. The implementation subclass as 

161 well as arguments to its constructor can be set with the static 

162 method `configure()` 

163 

164 All `AsyncHTTPClient` implementations support a ``defaults`` 

165 keyword argument, which can be used to set default values for 

166 `HTTPRequest` attributes. For example:: 

167 

168 AsyncHTTPClient.configure( 

169 None, defaults=dict(user_agent="MyUserAgent")) 

170 # or with force_instance: 

171 client = AsyncHTTPClient(force_instance=True, 

172 defaults=dict(user_agent="MyUserAgent")) 

173 

174 .. versionchanged:: 5.0 

175 The ``io_loop`` argument (deprecated since version 4.1) has been removed. 

176 

177 """ 

178 

179 _instance_cache = None # type: Dict[IOLoop, AsyncHTTPClient] 

180 

181 @classmethod 

182 def configurable_base(cls) -> Type[Configurable]: 

183 return AsyncHTTPClient 

184 

185 @classmethod 

186 def configurable_default(cls) -> Type[Configurable]: 

187 from tornado.simple_httpclient import SimpleAsyncHTTPClient 

188 

189 return SimpleAsyncHTTPClient 

190 

191 @classmethod 

192 def _async_clients(cls) -> Dict[IOLoop, "AsyncHTTPClient"]: 

193 attr_name = "_async_client_dict_" + cls.__name__ 

194 if not hasattr(cls, attr_name): 

195 setattr(cls, attr_name, weakref.WeakKeyDictionary()) 

196 return getattr(cls, attr_name) 

197 

198 def __new__(cls, force_instance: bool = False, **kwargs: Any) -> "AsyncHTTPClient": 

199 io_loop = IOLoop.current() 

200 if force_instance: 

201 instance_cache = None 

202 else: 

203 instance_cache = cls._async_clients() 

204 if instance_cache is not None and io_loop in instance_cache: 

205 return instance_cache[io_loop] 

206 instance = super(AsyncHTTPClient, cls).__new__(cls, **kwargs) # type: ignore 

207 # Make sure the instance knows which cache to remove itself from. 

208 # It can't simply call _async_clients() because we may be in 

209 # __new__(AsyncHTTPClient) but instance.__class__ may be 

210 # SimpleAsyncHTTPClient. 

211 instance._instance_cache = instance_cache 

212 if instance_cache is not None: 

213 instance_cache[instance.io_loop] = instance 

214 return instance 

215 

216 def initialize(self, defaults: Optional[Dict[str, Any]] = None) -> None: 

217 self.io_loop = IOLoop.current() 

218 self.defaults = dict(HTTPRequest._DEFAULTS) 

219 if defaults is not None: 

220 self.defaults.update(defaults) 

221 self._closed = False 

222 

223 def close(self) -> None: 

224 """Destroys this HTTP client, freeing any file descriptors used. 

225 

226 This method is **not needed in normal use** due to the way 

227 that `AsyncHTTPClient` objects are transparently reused. 

228 ``close()`` is generally only necessary when either the 

229 `.IOLoop` is also being closed, or the ``force_instance=True`` 

230 argument was used when creating the `AsyncHTTPClient`. 

231 

232 No other methods may be called on the `AsyncHTTPClient` after 

233 ``close()``. 

234 

235 """ 

236 if self._closed: 

237 return 

238 self._closed = True 

239 if self._instance_cache is not None: 

240 cached_val = self._instance_cache.pop(self.io_loop, None) 

241 # If there's an object other than self in the instance 

242 # cache for our IOLoop, something has gotten mixed up. A 

243 # value of None appears to be possible when this is called 

244 # from a destructor (HTTPClient.__del__) as the weakref 

245 # gets cleared before the destructor runs. 

246 if cached_val is not None and cached_val is not self: 

247 raise RuntimeError("inconsistent AsyncHTTPClient cache") 

248 

249 def fetch( 

250 self, 

251 request: Union[str, "HTTPRequest"], 

252 raise_error: bool = True, 

253 **kwargs: Any 

254 ) -> "Future[HTTPResponse]": 

255 """Executes a request, asynchronously returning an `HTTPResponse`. 

256 

257 The request may be either a string URL or an `HTTPRequest` object. 

258 If it is a string, we construct an `HTTPRequest` using any additional 

259 kwargs: ``HTTPRequest(request, **kwargs)`` 

260 

261 This method returns a `.Future` whose result is an 

262 `HTTPResponse`. By default, the ``Future`` will raise an 

263 `HTTPError` if the request returned a non-200 response code 

264 (other errors may also be raised if the server could not be 

265 contacted). Instead, if ``raise_error`` is set to False, the 

266 response will always be returned regardless of the response 

267 code. 

268 

269 If a ``callback`` is given, it will be invoked with the `HTTPResponse`. 

270 In the callback interface, `HTTPError` is not automatically raised. 

271 Instead, you must check the response's ``error`` attribute or 

272 call its `~HTTPResponse.rethrow` method. 

273 

274 .. versionchanged:: 6.0 

275 

276 The ``callback`` argument was removed. Use the returned 

277 `.Future` instead. 

278 

279 The ``raise_error=False`` argument only affects the 

280 `HTTPError` raised when a non-200 response code is used, 

281 instead of suppressing all errors. 

282 """ 

283 if self._closed: 

284 raise RuntimeError("fetch() called on closed AsyncHTTPClient") 

285 if not isinstance(request, HTTPRequest): 

286 request = HTTPRequest(url=request, **kwargs) 

287 else: 

288 if kwargs: 

289 raise ValueError( 

290 "kwargs can't be used if request is an HTTPRequest object" 

291 ) 

292 # We may modify this (to add Host, Accept-Encoding, etc), 

293 # so make sure we don't modify the caller's object. This is also 

294 # where normal dicts get converted to HTTPHeaders objects. 

295 request.headers = httputil.HTTPHeaders(request.headers) 

296 request_proxy = _RequestProxy(request, self.defaults) 

297 future = Future() # type: Future[HTTPResponse] 

298 

299 def handle_response(response: "HTTPResponse") -> None: 

300 if response.error: 

301 if raise_error or not response._error_is_response_code: 

302 future_set_exception_unless_cancelled(future, response.error) 

303 return 

304 future_set_result_unless_cancelled(future, response) 

305 

306 self.fetch_impl(cast(HTTPRequest, request_proxy), handle_response) 

307 return future 

308 

309 def fetch_impl( 

310 self, request: "HTTPRequest", callback: Callable[["HTTPResponse"], None] 

311 ) -> None: 

312 raise NotImplementedError() 

313 

314 @classmethod 

315 def configure( 

316 cls, impl: "Union[None, str, Type[Configurable]]", **kwargs: Any 

317 ) -> None: 

318 """Configures the `AsyncHTTPClient` subclass to use. 

319 

320 ``AsyncHTTPClient()`` actually creates an instance of a subclass. 

321 This method may be called with either a class object or the 

322 fully-qualified name of such a class (or ``None`` to use the default, 

323 ``SimpleAsyncHTTPClient``) 

324 

325 If additional keyword arguments are given, they will be passed 

326 to the constructor of each subclass instance created. The 

327 keyword argument ``max_clients`` determines the maximum number 

328 of simultaneous `~AsyncHTTPClient.fetch()` operations that can 

329 execute in parallel on each `.IOLoop`. Additional arguments 

330 may be supported depending on the implementation class in use. 

331 

332 Example:: 

333 

334 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") 

335 """ 

336 super(AsyncHTTPClient, cls).configure(impl, **kwargs) 

337 

338 

339class HTTPRequest(object): 

340 """HTTP client request object.""" 

341 

342 _headers = None # type: Union[Dict[str, str], httputil.HTTPHeaders] 

343 

344 # Default values for HTTPRequest parameters. 

345 # Merged with the values on the request object by AsyncHTTPClient 

346 # implementations. 

347 _DEFAULTS = dict( 

348 connect_timeout=20.0, 

349 request_timeout=20.0, 

350 follow_redirects=True, 

351 max_redirects=5, 

352 decompress_response=True, 

353 proxy_password="", 

354 allow_nonstandard_methods=False, 

355 validate_cert=True, 

356 ) 

357 

358 def __init__( 

359 self, 

360 url: str, 

361 method: str = "GET", 

362 headers: Optional[Union[Dict[str, str], httputil.HTTPHeaders]] = None, 

363 body: Optional[Union[bytes, str]] = None, 

364 auth_username: Optional[str] = None, 

365 auth_password: Optional[str] = None, 

366 auth_mode: Optional[str] = None, 

367 connect_timeout: Optional[float] = None, 

368 request_timeout: Optional[float] = None, 

369 if_modified_since: Optional[Union[float, datetime.datetime]] = None, 

370 follow_redirects: Optional[bool] = None, 

371 max_redirects: Optional[int] = None, 

372 user_agent: Optional[str] = None, 

373 use_gzip: Optional[bool] = None, 

374 network_interface: Optional[str] = None, 

375 streaming_callback: Optional[Callable[[bytes], None]] = None, 

376 header_callback: Optional[Callable[[str], None]] = None, 

377 prepare_curl_callback: Optional[Callable[[Any], None]] = None, 

378 proxy_host: Optional[str] = None, 

379 proxy_port: Optional[int] = None, 

380 proxy_username: Optional[str] = None, 

381 proxy_password: Optional[str] = None, 

382 proxy_auth_mode: Optional[str] = None, 

383 allow_nonstandard_methods: Optional[bool] = None, 

384 validate_cert: Optional[bool] = None, 

385 ca_certs: Optional[str] = None, 

386 allow_ipv6: Optional[bool] = None, 

387 client_key: Optional[str] = None, 

388 client_cert: Optional[str] = None, 

389 body_producer: Optional[ 

390 Callable[[Callable[[bytes], None]], "Future[None]"] 

391 ] = None, 

392 expect_100_continue: bool = False, 

393 decompress_response: Optional[bool] = None, 

394 ssl_options: Optional[Union[Dict[str, Any], ssl.SSLContext]] = None, 

395 ) -> None: 

396 r"""All parameters except ``url`` are optional. 

397 

398 :arg str url: URL to fetch 

399 :arg str method: HTTP method, e.g. "GET" or "POST" 

400 :arg headers: Additional HTTP headers to pass on the request 

401 :type headers: `~tornado.httputil.HTTPHeaders` or `dict` 

402 :arg body: HTTP request body as a string (byte or unicode; if unicode 

403 the utf-8 encoding will be used) 

404 :type body: `str` or `bytes` 

405 :arg collections.abc.Callable body_producer: Callable used for 

406 lazy/asynchronous request bodies. 

407 It is called with one argument, a ``write`` function, and should 

408 return a `.Future`. It should call the write function with new 

409 data as it becomes available. The write function returns a 

410 `.Future` which can be used for flow control. 

411 Only one of ``body`` and ``body_producer`` may 

412 be specified. ``body_producer`` is not supported on 

413 ``curl_httpclient``. When using ``body_producer`` it is recommended 

414 to pass a ``Content-Length`` in the headers as otherwise chunked 

415 encoding will be used, and many servers do not support chunked 

416 encoding on requests. New in Tornado 4.0 

417 :arg str auth_username: Username for HTTP authentication 

418 :arg str auth_password: Password for HTTP authentication 

419 :arg str auth_mode: Authentication mode; default is "basic". 

420 Allowed values are implementation-defined; ``curl_httpclient`` 

421 supports "basic" and "digest"; ``simple_httpclient`` only supports 

422 "basic" 

423 :arg float connect_timeout: Timeout for initial connection in seconds, 

424 default 20 seconds (0 means no timeout) 

425 :arg float request_timeout: Timeout for entire request in seconds, 

426 default 20 seconds (0 means no timeout) 

427 :arg if_modified_since: Timestamp for ``If-Modified-Since`` header 

428 :type if_modified_since: `datetime` or `float` 

429 :arg bool follow_redirects: Should redirects be followed automatically 

430 or return the 3xx response? Default True. 

431 :arg int max_redirects: Limit for ``follow_redirects``, default 5. 

432 :arg str user_agent: String to send as ``User-Agent`` header 

433 :arg bool decompress_response: Request a compressed response from 

434 the server and decompress it after downloading. Default is True. 

435 New in Tornado 4.0. 

436 :arg bool use_gzip: Deprecated alias for ``decompress_response`` 

437 since Tornado 4.0. 

438 :arg str network_interface: Network interface or source IP to use for request. 

439 See ``curl_httpclient`` note below. 

440 :arg collections.abc.Callable streaming_callback: If set, ``streaming_callback`` will 

441 be run with each chunk of data as it is received, and 

442 ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in 

443 the final response. 

444 :arg collections.abc.Callable header_callback: If set, ``header_callback`` will 

445 be run with each header line as it is received (including the 

446 first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line 

447 containing only ``\r\n``. All lines include the trailing newline 

448 characters). ``HTTPResponse.headers`` will be empty in the final 

449 response. This is most useful in conjunction with 

450 ``streaming_callback``, because it's the only way to get access to 

451 header data while the request is in progress. 

452 :arg collections.abc.Callable prepare_curl_callback: If set, will be called with 

453 a ``pycurl.Curl`` object to allow the application to make additional 

454 ``setopt`` calls. 

455 :arg str proxy_host: HTTP proxy hostname. To use proxies, 

456 ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username``, 

457 ``proxy_pass`` and ``proxy_auth_mode`` are optional. Proxies are 

458 currently only supported with ``curl_httpclient``. 

459 :arg int proxy_port: HTTP proxy port 

460 :arg str proxy_username: HTTP proxy username 

461 :arg str proxy_password: HTTP proxy password 

462 :arg str proxy_auth_mode: HTTP proxy Authentication mode; 

463 default is "basic". supports "basic" and "digest" 

464 :arg bool allow_nonstandard_methods: Allow unknown values for ``method`` 

465 argument? Default is False. 

466 :arg bool validate_cert: For HTTPS requests, validate the server's 

467 certificate? Default is True. 

468 :arg str ca_certs: filename of CA certificates in PEM format, 

469 or None to use defaults. See note below when used with 

470 ``curl_httpclient``. 

471 :arg str client_key: Filename for client SSL key, if any. See 

472 note below when used with ``curl_httpclient``. 

473 :arg str client_cert: Filename for client SSL certificate, if any. 

474 See note below when used with ``curl_httpclient``. 

475 :arg ssl.SSLContext ssl_options: `ssl.SSLContext` object for use in 

476 ``simple_httpclient`` (unsupported by ``curl_httpclient``). 

477 Overrides ``validate_cert``, ``ca_certs``, ``client_key``, 

478 and ``client_cert``. 

479 :arg bool allow_ipv6: Use IPv6 when available? Default is True. 

480 :arg bool expect_100_continue: If true, send the 

481 ``Expect: 100-continue`` header and wait for a continue response 

482 before sending the request body. Only supported with 

483 ``simple_httpclient``. 

484 

485 .. note:: 

486 

487 When using ``curl_httpclient`` certain options may be 

488 inherited by subsequent fetches because ``pycurl`` does 

489 not allow them to be cleanly reset. This applies to the 

490 ``ca_certs``, ``client_key``, ``client_cert``, and 

491 ``network_interface`` arguments. If you use these 

492 options, you should pass them on every request (you don't 

493 have to always use the same values, but it's not possible 

494 to mix requests that specify these options with ones that 

495 use the defaults). 

496 

497 .. versionadded:: 3.1 

498 The ``auth_mode`` argument. 

499 

500 .. versionadded:: 4.0 

501 The ``body_producer`` and ``expect_100_continue`` arguments. 

502 

503 .. versionadded:: 4.2 

504 The ``ssl_options`` argument. 

505 

506 .. versionadded:: 4.5 

507 The ``proxy_auth_mode`` argument. 

508 """ 

509 # Note that some of these attributes go through property setters 

510 # defined below. 

511 self.headers = headers # type: ignore 

512 if if_modified_since: 

513 self.headers["If-Modified-Since"] = httputil.format_timestamp( 

514 if_modified_since 

515 ) 

516 self.proxy_host = proxy_host 

517 self.proxy_port = proxy_port 

518 self.proxy_username = proxy_username 

519 self.proxy_password = proxy_password 

520 self.proxy_auth_mode = proxy_auth_mode 

521 self.url = url 

522 self.method = method 

523 self.body = body # type: ignore 

524 self.body_producer = body_producer 

525 self.auth_username = auth_username 

526 self.auth_password = auth_password 

527 self.auth_mode = auth_mode 

528 self.connect_timeout = connect_timeout 

529 self.request_timeout = request_timeout 

530 self.follow_redirects = follow_redirects 

531 self.max_redirects = max_redirects 

532 self.user_agent = user_agent 

533 if decompress_response is not None: 

534 self.decompress_response = decompress_response # type: Optional[bool] 

535 else: 

536 self.decompress_response = use_gzip 

537 self.network_interface = network_interface 

538 self.streaming_callback = streaming_callback 

539 self.header_callback = header_callback 

540 self.prepare_curl_callback = prepare_curl_callback 

541 self.allow_nonstandard_methods = allow_nonstandard_methods 

542 self.validate_cert = validate_cert 

543 self.ca_certs = ca_certs 

544 self.allow_ipv6 = allow_ipv6 

545 self.client_key = client_key 

546 self.client_cert = client_cert 

547 self.ssl_options = ssl_options 

548 self.expect_100_continue = expect_100_continue 

549 self.start_time = time.time() 

550 

551 @property 

552 def headers(self) -> httputil.HTTPHeaders: 

553 # TODO: headers may actually be a plain dict until fairly late in 

554 # the process (AsyncHTTPClient.fetch), but practically speaking, 

555 # whenever the property is used they're already HTTPHeaders. 

556 return self._headers # type: ignore 

557 

558 @headers.setter 

559 def headers(self, value: Union[Dict[str, str], httputil.HTTPHeaders]) -> None: 

560 if value is None: 

561 self._headers = httputil.HTTPHeaders() 

562 else: 

563 self._headers = value # type: ignore 

564 

565 @property 

566 def body(self) -> bytes: 

567 return self._body 

568 

569 @body.setter 

570 def body(self, value: Union[bytes, str]) -> None: 

571 self._body = utf8(value) 

572 

573 

574class HTTPResponse(object): 

575 """HTTP Response object. 

576 

577 Attributes: 

578 

579 * ``request``: HTTPRequest object 

580 

581 * ``code``: numeric HTTP status code, e.g. 200 or 404 

582 

583 * ``reason``: human-readable reason phrase describing the status code 

584 

585 * ``headers``: `tornado.httputil.HTTPHeaders` object 

586 

587 * ``effective_url``: final location of the resource after following any 

588 redirects 

589 

590 * ``buffer``: ``cStringIO`` object for response body 

591 

592 * ``body``: response body as bytes (created on demand from ``self.buffer``) 

593 

594 * ``error``: Exception object, if any 

595 

596 * ``request_time``: seconds from request start to finish. Includes all 

597 network operations from DNS resolution to receiving the last byte of 

598 data. Does not include time spent in the queue (due to the 

599 ``max_clients`` option). If redirects were followed, only includes 

600 the final request. 

601 

602 * ``start_time``: Time at which the HTTP operation started, based on 

603 `time.time` (not the monotonic clock used by `.IOLoop.time`). May 

604 be ``None`` if the request timed out while in the queue. 

605 

606 * ``time_info``: dictionary of diagnostic timing information from the 

607 request. Available data are subject to change, but currently uses timings 

608 available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, 

609 plus ``queue``, which is the delay (if any) introduced by waiting for 

610 a slot under `AsyncHTTPClient`'s ``max_clients`` setting. 

611 

612 .. versionadded:: 5.1 

613 

614 Added the ``start_time`` attribute. 

615 

616 .. versionchanged:: 5.1 

617 

618 The ``request_time`` attribute previously included time spent in the queue 

619 for ``simple_httpclient``, but not in ``curl_httpclient``. Now queueing time 

620 is excluded in both implementations. ``request_time`` is now more accurate for 

621 ``curl_httpclient`` because it uses a monotonic clock when available. 

622 """ 

623 

624 # I'm not sure why these don't get type-inferred from the references in __init__. 

625 error = None # type: Optional[BaseException] 

626 _error_is_response_code = False 

627 request = None # type: HTTPRequest 

628 

629 def __init__( 

630 self, 

631 request: HTTPRequest, 

632 code: int, 

633 headers: Optional[httputil.HTTPHeaders] = None, 

634 buffer: Optional[BytesIO] = None, 

635 effective_url: Optional[str] = None, 

636 error: Optional[BaseException] = None, 

637 request_time: Optional[float] = None, 

638 time_info: Optional[Dict[str, float]] = None, 

639 reason: Optional[str] = None, 

640 start_time: Optional[float] = None, 

641 ) -> None: 

642 if isinstance(request, _RequestProxy): 

643 self.request = request.request 

644 else: 

645 self.request = request 

646 self.code = code 

647 self.reason = reason or httputil.responses.get(code, "Unknown") 

648 if headers is not None: 

649 self.headers = headers 

650 else: 

651 self.headers = httputil.HTTPHeaders() 

652 self.buffer = buffer 

653 self._body = None # type: Optional[bytes] 

654 if effective_url is None: 

655 self.effective_url = request.url 

656 else: 

657 self.effective_url = effective_url 

658 self._error_is_response_code = False 

659 if error is None: 

660 if self.code < 200 or self.code >= 300: 

661 self._error_is_response_code = True 

662 self.error = HTTPError(self.code, message=self.reason, response=self) 

663 else: 

664 self.error = None 

665 else: 

666 self.error = error 

667 self.start_time = start_time 

668 self.request_time = request_time 

669 self.time_info = time_info or {} 

670 

671 @property 

672 def body(self) -> bytes: 

673 if self.buffer is None: 

674 return b"" 

675 elif self._body is None: 

676 self._body = self.buffer.getvalue() 

677 

678 return self._body 

679 

680 def rethrow(self) -> None: 

681 """If there was an error on the request, raise an `HTTPError`.""" 

682 if self.error: 

683 raise self.error 

684 

685 def __repr__(self) -> str: 

686 args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) 

687 return "%s(%s)" % (self.__class__.__name__, args) 

688 

689 

690class HTTPClientError(Exception): 

691 """Exception thrown for an unsuccessful HTTP request. 

692 

693 Attributes: 

694 

695 * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is 

696 used when no HTTP response was received, e.g. for a timeout. 

697 

698 * ``response`` - `HTTPResponse` object, if any. 

699 

700 Note that if ``follow_redirects`` is False, redirects become HTTPErrors, 

701 and you can look at ``error.response.headers['Location']`` to see the 

702 destination of the redirect. 

703 

704 .. versionchanged:: 5.1 

705 

706 Renamed from ``HTTPError`` to ``HTTPClientError`` to avoid collisions with 

707 `tornado.web.HTTPError`. The name ``tornado.httpclient.HTTPError`` remains 

708 as an alias. 

709 """ 

710 

711 def __init__( 

712 self, 

713 code: int, 

714 message: Optional[str] = None, 

715 response: Optional[HTTPResponse] = None, 

716 ) -> None: 

717 self.code = code 

718 self.message = message or httputil.responses.get(code, "Unknown") 

719 self.response = response 

720 super().__init__(code, message, response) 

721 

722 def __str__(self) -> str: 

723 return "HTTP %d: %s" % (self.code, self.message) 

724 

725 # There is a cyclic reference between self and self.response, 

726 # which breaks the default __repr__ implementation. 

727 # (especially on pypy, which doesn't have the same recursion 

728 # detection as cpython). 

729 __repr__ = __str__ 

730 

731 

732HTTPError = HTTPClientError 

733 

734 

735class _RequestProxy(object): 

736 """Combines an object with a dictionary of defaults. 

737 

738 Used internally by AsyncHTTPClient implementations. 

739 """ 

740 

741 def __init__( 

742 self, request: HTTPRequest, defaults: Optional[Dict[str, Any]] 

743 ) -> None: 

744 self.request = request 

745 self.defaults = defaults 

746 

747 def __getattr__(self, name: str) -> Any: 

748 request_attr = getattr(self.request, name) 

749 if request_attr is not None: 

750 return request_attr 

751 elif self.defaults is not None: 

752 return self.defaults.get(name, None) 

753 else: 

754 return None 

755 

756 

757def main() -> None: 

758 from tornado.options import define, options, parse_command_line 

759 

760 define("print_headers", type=bool, default=False) 

761 define("print_body", type=bool, default=True) 

762 define("follow_redirects", type=bool, default=True) 

763 define("validate_cert", type=bool, default=True) 

764 define("proxy_host", type=str) 

765 define("proxy_port", type=int) 

766 args = parse_command_line() 

767 client = HTTPClient() 

768 for arg in args: 

769 try: 

770 response = client.fetch( 

771 arg, 

772 follow_redirects=options.follow_redirects, 

773 validate_cert=options.validate_cert, 

774 proxy_host=options.proxy_host, 

775 proxy_port=options.proxy_port, 

776 ) 

777 except HTTPError as e: 

778 if e.response is not None: 

779 response = e.response 

780 else: 

781 raise 

782 if options.print_headers: 

783 print(response.headers) 

784 if options.print_body: 

785 print(native_str(response.body)) 

786 client.close() 

787 

788 

789if __name__ == "__main__": 

790 main()