Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/h11/_readers.py: 24%

1# Code to read HTTP data 

2# 

3# Strategy: each reader is a callable which takes a ReceiveBuffer object, and 

4# either: 

5# 1) consumes some of it and returns an Event 

6# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate() 

7# and it might raise a LocalProtocolError, so simpler just to always use 

8# this) 

9# 3) returns None, meaning "I need more data" 

10# 

11# If they have a .read_eof attribute, then this will be called if an EOF is 

12# received -- but this is optional. Either way, the actual ConnectionClosed 

13# event will be generated afterwards. 

14# 

15# READERS is a dict describing how to pick a reader. It maps states to either: 

16# - a reader 

17# - or, for body readers, a dict of per-framing reader factories 

18 

19import re 

20from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union 

21 

22from ._abnf import chunk_header, header_field, request_line, status_line 

23from ._events import Data, EndOfMessage, InformationalResponse, Request, Response 

24from ._receivebuffer import ReceiveBuffer 

25from ._state import ( 

26 CLIENT, 

27 CLOSED, 

28 DONE, 

29 IDLE, 

30 MUST_CLOSE, 

31 SEND_BODY, 

32 SEND_RESPONSE, 

33 SERVER, 

34) 

35from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate 

36 

37__all__ = ["READERS"] 

38 

39header_field_re = re.compile(header_field.encode("ascii")) 

40obs_fold_re = re.compile(rb"[ \t]+") 

41 

42 

43def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: 

44 it = iter(lines) 

45 last: Optional[bytes] = None 

46 for line in it: 

47 match = obs_fold_re.match(line) 

48 if match: 

49 if last is None: 

50 raise LocalProtocolError("continuation line at start of headers") 

51 if not isinstance(last, bytearray): 

52 # Cast to a mutable type, avoiding copy on append to ensure O(n) time 

53 last = bytearray(last) 

54 last += b" " 

55 last += line[match.end() :] 

56 else: 

57 if last is not None: 

58 yield last 

59 last = line 

60 if last is not None: 

61 yield last 

62 

63 

64def _decode_header_lines( 

65 lines: Iterable[bytes], 

66) -> Iterable[Tuple[bytes, bytes]]: 

67 for line in _obsolete_line_fold(lines): 

68 matches = validate(header_field_re, line, "illegal header line: {!r}", line) 

69 yield (matches["field_name"], matches["field_value"]) 

70 

71 

72request_line_re = re.compile(request_line.encode("ascii")) 

73 

74 

75def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: 

76 lines = buf.maybe_extract_lines() 

77 if lines is None: 

78 if buf.is_next_line_obviously_invalid_request_line(): 

79 raise LocalProtocolError("illegal request line") 

80 return None 

81 if not lines: 

82 raise LocalProtocolError("no request line received") 

83 matches = validate( 

84 request_line_re, lines[0], "illegal request line: {!r}", lines[0] 

85 ) 

86 return Request( 

87 headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches 

88 ) 

89 

90 

91status_line_re = re.compile(status_line.encode("ascii")) 

92 

93 

94def maybe_read_from_SEND_RESPONSE_server( 

95 buf: ReceiveBuffer, 

96) -> Union[InformationalResponse, Response, None]: 

97 lines = buf.maybe_extract_lines() 

98 if lines is None: 

99 if buf.is_next_line_obviously_invalid_request_line(): 

100 raise LocalProtocolError("illegal request line") 

101 return None 

102 if not lines: 

103 raise LocalProtocolError("no response line received") 

104 matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0]) 

105 http_version = ( 

106 b"1.1" if matches["http_version"] is None else matches["http_version"] 

107 ) 

108 reason = b"" if matches["reason"] is None else matches["reason"] 

109 status_code = int(matches["status_code"]) 

110 class_: Union[Type[InformationalResponse], Type[Response]] = ( 

111 InformationalResponse if status_code < 200 else Response 

112 ) 

113 return class_( 

114 headers=list(_decode_header_lines(lines[1:])), 

115 _parsed=True, 

116 status_code=status_code, 

117 reason=reason, 

118 http_version=http_version, 

119 ) 

120 

121 

122class ContentLengthReader: 

123 def __init__(self, length: int) -> None: 

124 self._length = length 

125 self._remaining = length 

126 

127 def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: 

128 if self._remaining == 0: 

129 return EndOfMessage() 

130 data = buf.maybe_extract_at_most(self._remaining) 

131 if data is None: 

132 return None 

133 self._remaining -= len(data) 

134 return Data(data=data) 

135 

136 def read_eof(self) -> NoReturn: 

137 raise RemoteProtocolError( 

138 "peer closed connection without sending complete message body " 

139 "(received {} bytes, expected {})".format( 

140 self._length - self._remaining, self._length 

141 ) 

142 ) 

143 

144 

145chunk_header_re = re.compile(chunk_header.encode("ascii")) 

146 

147 

148class ChunkedReader: 

149 def __init__(self) -> None: 

150 self._bytes_in_chunk = 0 

151 # After reading a chunk, we have to throw away the trailing \r\n; if 

152 # this is >0 then we discard that many bytes before resuming regular 

153 # de-chunkification. 

154 self._bytes_to_discard = 0 

155 self._reading_trailer = False 

156 

157 def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: 

158 if self._reading_trailer: 

159 lines = buf.maybe_extract_lines() 

160 if lines is None: 

161 return None 

162 return EndOfMessage(headers=list(_decode_header_lines(lines))) 

163 if self._bytes_to_discard > 0: 

164 data = buf.maybe_extract_at_most(self._bytes_to_discard) 

165 if data is None: 

166 return None 

167 self._bytes_to_discard -= len(data) 

168 if self._bytes_to_discard > 0: 

169 return None 

170 # else, fall through and read some more 

171 assert self._bytes_to_discard == 0 

172 if self._bytes_in_chunk == 0: 

173 # We need to refill our chunk count 

174 chunk_header = buf.maybe_extract_next_line() 

175 if chunk_header is None: 

176 return None 

177 matches = validate( 

178 chunk_header_re, 

179 chunk_header, 

180 "illegal chunk header: {!r}", 

181 chunk_header, 

182 ) 

183 # XX FIXME: we discard chunk extensions. Does anyone care? 

184 self._bytes_in_chunk = int(matches["chunk_size"], base=16) 

185 if self._bytes_in_chunk == 0: 

186 self._reading_trailer = True 

187 return self(buf) 

188 chunk_start = True 

189 else: 

190 chunk_start = False 

191 assert self._bytes_in_chunk > 0 

192 data = buf.maybe_extract_at_most(self._bytes_in_chunk) 

193 if data is None: 

194 return None 

195 self._bytes_in_chunk -= len(data) 

196 if self._bytes_in_chunk == 0: 

197 self._bytes_to_discard = 2 

198 chunk_end = True 

199 else: 

200 chunk_end = False 

201 return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end) 

202 

203 def read_eof(self) -> NoReturn: 

204 raise RemoteProtocolError( 

205 "peer closed connection without sending complete message body " 

206 "(incomplete chunked read)" 

207 ) 

208 

209 

210class Http10Reader: 

211 def __call__(self, buf: ReceiveBuffer) -> Optional[Data]: 

212 data = buf.maybe_extract_at_most(999999999) 

213 if data is None: 

214 return None 

215 return Data(data=data) 

216 

217 def read_eof(self) -> EndOfMessage: 

218 return EndOfMessage() 

219 

220 

221def expect_nothing(buf: ReceiveBuffer) -> None: 

222 if buf: 

223 raise LocalProtocolError("Got data when expecting EOF") 

224 return None 

225 

226 

227ReadersType = Dict[ 

228 Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]], 

229 Union[Callable[..., Any], Dict[str, Callable[..., Any]]], 

230] 

231 

232READERS: ReadersType = { 

233 (CLIENT, IDLE): maybe_read_from_IDLE_client, 

234 (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server, 

235 (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server, 

236 (CLIENT, DONE): expect_nothing, 

237 (CLIENT, MUST_CLOSE): expect_nothing, 

238 (CLIENT, CLOSED): expect_nothing, 

239 (SERVER, DONE): expect_nothing, 

240 (SERVER, MUST_CLOSE): expect_nothing, 

241 (SERVER, CLOSED): expect_nothing, 

242 SEND_BODY: { 

243 "chunked": ChunkedReader, 

244 "content-length": ContentLengthReader, 

245 "http/1.0": Http10Reader, 

246 }, 

247}