Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pip/_vendor/msgpack/fallback.py: 11%

1"""Fallback pure Python implementation of msgpack""" 

2from datetime import datetime as _DateTime 

3import sys 

4import struct 

5 

6 

7PY2 = sys.version_info[0] == 2 

8if PY2: 

9 int_types = (int, long) 

10 

11 def dict_iteritems(d): 

12 return d.iteritems() 

13 

14else: 

15 int_types = int 

16 unicode = str 

17 xrange = range 

18 

19 def dict_iteritems(d): 

20 return d.items() 

21 

22 

23if sys.version_info < (3, 5): 

24 # Ugly hack... 

25 RecursionError = RuntimeError 

26 

27 def _is_recursionerror(e): 

28 return ( 

29 len(e.args) == 1 

30 and isinstance(e.args[0], str) 

31 and e.args[0].startswith("maximum recursion depth exceeded") 

32 ) 

33 

34else: 

35 

36 def _is_recursionerror(e): 

37 return True 

38 

39 

40if hasattr(sys, "pypy_version_info"): 

41 # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own 

42 # StringBuilder is fastest. 

43 from __pypy__ import newlist_hint 

44 

45 try: 

46 from __pypy__.builders import BytesBuilder as StringBuilder 

47 except ImportError: 

48 from __pypy__.builders import StringBuilder 

49 USING_STRINGBUILDER = True 

50 

51 class StringIO(object): 

52 def __init__(self, s=b""): 

53 if s: 

54 self.builder = StringBuilder(len(s)) 

55 self.builder.append(s) 

56 else: 

57 self.builder = StringBuilder() 

58 

59 def write(self, s): 

60 if isinstance(s, memoryview): 

61 s = s.tobytes() 

62 elif isinstance(s, bytearray): 

63 s = bytes(s) 

64 self.builder.append(s) 

65 

66 def getvalue(self): 

67 return self.builder.build() 

68 

69else: 

70 USING_STRINGBUILDER = False 

71 from io import BytesIO as StringIO 

72 

73 newlist_hint = lambda size: [] 

74 

75 

76from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError 

77 

78from .ext import ExtType, Timestamp 

79 

80 

81EX_SKIP = 0 

82EX_CONSTRUCT = 1 

83EX_READ_ARRAY_HEADER = 2 

84EX_READ_MAP_HEADER = 3 

85 

86TYPE_IMMEDIATE = 0 

87TYPE_ARRAY = 1 

88TYPE_MAP = 2 

89TYPE_RAW = 3 

90TYPE_BIN = 4 

91TYPE_EXT = 5 

92 

93DEFAULT_RECURSE_LIMIT = 511 

94 

95 

96def _check_type_strict(obj, t, type=type, tuple=tuple): 

97 if type(t) is tuple: 

98 return type(obj) in t 

99 else: 

100 return type(obj) is t 

101 

102 

103def _get_data_from_buffer(obj): 

104 view = memoryview(obj) 

105 if view.itemsize != 1: 

106 raise ValueError("cannot unpack from multi-byte object") 

107 return view 

108 

109 

110def unpackb(packed, **kwargs): 

111 """ 

112 Unpack an object from `packed`. 

113 

114 Raises ``ExtraData`` when *packed* contains extra bytes. 

115 Raises ``ValueError`` when *packed* is incomplete. 

116 Raises ``FormatError`` when *packed* is not valid msgpack. 

117 Raises ``StackError`` when *packed* contains too nested. 

118 Other exceptions can be raised during unpacking. 

119 

120 See :class:`Unpacker` for options. 

121 """ 

122 unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs) 

123 unpacker.feed(packed) 

124 try: 

125 ret = unpacker._unpack() 

126 except OutOfData: 

127 raise ValueError("Unpack failed: incomplete input") 

128 except RecursionError as e: 

129 if _is_recursionerror(e): 

130 raise StackError 

131 raise 

132 if unpacker._got_extradata(): 

133 raise ExtraData(ret, unpacker._get_extradata()) 

134 return ret 

135 

136 

137if sys.version_info < (2, 7, 6): 

138 

139 def _unpack_from(f, b, o=0): 

140 """Explicit type cast for legacy struct.unpack_from""" 

141 return struct.unpack_from(f, bytes(b), o) 

142 

143else: 

144 _unpack_from = struct.unpack_from 

145 

146_NO_FORMAT_USED = "" 

147_MSGPACK_HEADERS = { 

148 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), 

149 0xC5: (2, ">H", TYPE_BIN), 

150 0xC6: (4, ">I", TYPE_BIN), 

151 0xC7: (2, "Bb", TYPE_EXT), 

152 0xC8: (3, ">Hb", TYPE_EXT), 

153 0xC9: (5, ">Ib", TYPE_EXT), 

154 0xCA: (4, ">f"), 

155 0xCB: (8, ">d"), 

156 0xCC: (1, _NO_FORMAT_USED), 

157 0xCD: (2, ">H"), 

158 0xCE: (4, ">I"), 

159 0xCF: (8, ">Q"), 

160 0xD0: (1, "b"), 

161 0xD1: (2, ">h"), 

162 0xD2: (4, ">i"), 

163 0xD3: (8, ">q"), 

164 0xD4: (1, "b1s", TYPE_EXT), 

165 0xD5: (2, "b2s", TYPE_EXT), 

166 0xD6: (4, "b4s", TYPE_EXT), 

167 0xD7: (8, "b8s", TYPE_EXT), 

168 0xD8: (16, "b16s", TYPE_EXT), 

169 0xD9: (1, _NO_FORMAT_USED, TYPE_RAW), 

170 0xDA: (2, ">H", TYPE_RAW), 

171 0xDB: (4, ">I", TYPE_RAW), 

172 0xDC: (2, ">H", TYPE_ARRAY), 

173 0xDD: (4, ">I", TYPE_ARRAY), 

174 0xDE: (2, ">H", TYPE_MAP), 

175 0xDF: (4, ">I", TYPE_MAP), 

176} 

177 

178 

179class Unpacker(object): 

180 """Streaming unpacker. 

181 

182 Arguments: 

183 

184 :param file_like: 

185 File-like object having `.read(n)` method. 

186 If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. 

187 

188 :param int read_size: 

189 Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) 

190 

191 :param bool use_list: 

192 If true, unpack msgpack array to Python list. 

193 Otherwise, unpack to Python tuple. (default: True) 

194 

195 :param bool raw: 

196 If true, unpack msgpack raw to Python bytes. 

197 Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). 

198 

199 :param int timestamp: 

200 Control how timestamp type is unpacked: 

201 

202 0 - Timestamp 

203 1 - float (Seconds from the EPOCH) 

204 2 - int (Nanoseconds from the EPOCH) 

205 3 - datetime.datetime (UTC). Python 2 is not supported. 

206 

207 :param bool strict_map_key: 

208 If true (default), only str or bytes are accepted for map (dict) keys. 

209 

210 :param callable object_hook: 

211 When specified, it should be callable. 

212 Unpacker calls it with a dict argument after unpacking msgpack map. 

213 (See also simplejson) 

214 

215 :param callable object_pairs_hook: 

216 When specified, it should be callable. 

217 Unpacker calls it with a list of key-value pairs after unpacking msgpack map. 

218 (See also simplejson) 

219 

220 :param str unicode_errors: 

221 The error handler for decoding unicode. (default: 'strict') 

222 This option should be used only when you have msgpack data which 

223 contains invalid UTF-8 string. 

224 

225 :param int max_buffer_size: 

226 Limits size of data waiting unpacked. 0 means 2**32-1. 

227 The default value is 100*1024*1024 (100MiB). 

228 Raises `BufferFull` exception when it is insufficient. 

229 You should set this parameter when unpacking data from untrusted source. 

230 

231 :param int max_str_len: 

232 Deprecated, use *max_buffer_size* instead. 

233 Limits max length of str. (default: max_buffer_size) 

234 

235 :param int max_bin_len: 

236 Deprecated, use *max_buffer_size* instead. 

237 Limits max length of bin. (default: max_buffer_size) 

238 

239 :param int max_array_len: 

240 Limits max length of array. 

241 (default: max_buffer_size) 

242 

243 :param int max_map_len: 

244 Limits max length of map. 

245 (default: max_buffer_size//2) 

246 

247 :param int max_ext_len: 

248 Deprecated, use *max_buffer_size* instead. 

249 Limits max size of ext type. (default: max_buffer_size) 

250 

251 Example of streaming deserialize from file-like object:: 

252 

253 unpacker = Unpacker(file_like) 

254 for o in unpacker: 

255 process(o) 

256 

257 Example of streaming deserialize from socket:: 

258 

259 unpacker = Unpacker() 

260 while True: 

261 buf = sock.recv(1024**2) 

262 if not buf: 

263 break 

264 unpacker.feed(buf) 

265 for o in unpacker: 

266 process(o) 

267 

268 Raises ``ExtraData`` when *packed* contains extra bytes. 

269 Raises ``OutOfData`` when *packed* is incomplete. 

270 Raises ``FormatError`` when *packed* is not valid msgpack. 

271 Raises ``StackError`` when *packed* contains too nested. 

272 Other exceptions can be raised during unpacking. 

273 """ 

274 

275 def __init__( 

276 self, 

277 file_like=None, 

278 read_size=0, 

279 use_list=True, 

280 raw=False, 

281 timestamp=0, 

282 strict_map_key=True, 

283 object_hook=None, 

284 object_pairs_hook=None, 

285 list_hook=None, 

286 unicode_errors=None, 

287 max_buffer_size=100 * 1024 * 1024, 

288 ext_hook=ExtType, 

289 max_str_len=-1, 

290 max_bin_len=-1, 

291 max_array_len=-1, 

292 max_map_len=-1, 

293 max_ext_len=-1, 

294 ): 

295 if unicode_errors is None: 

296 unicode_errors = "strict" 

297 

298 if file_like is None: 

299 self._feeding = True 

300 else: 

301 if not callable(file_like.read): 

302 raise TypeError("`file_like.read` must be callable") 

303 self.file_like = file_like 

304 self._feeding = False 

305 

306 #: array of bytes fed. 

307 self._buffer = bytearray() 

308 #: Which position we currently reads 

309 self._buff_i = 0 

310 

311 # When Unpacker is used as an iterable, between the calls to next(), 

312 # the buffer is not "consumed" completely, for efficiency sake. 

313 # Instead, it is done sloppily. To make sure we raise BufferFull at 

314 # the correct moments, we have to keep track of how sloppy we were. 

315 # Furthermore, when the buffer is incomplete (that is: in the case 

316 # we raise an OutOfData) we need to rollback the buffer to the correct 

317 # state, which _buf_checkpoint records. 

318 self._buf_checkpoint = 0 

319 

320 if not max_buffer_size: 

321 max_buffer_size = 2**31 - 1 

322 if max_str_len == -1: 

323 max_str_len = max_buffer_size 

324 if max_bin_len == -1: 

325 max_bin_len = max_buffer_size 

326 if max_array_len == -1: 

327 max_array_len = max_buffer_size 

328 if max_map_len == -1: 

329 max_map_len = max_buffer_size // 2 

330 if max_ext_len == -1: 

331 max_ext_len = max_buffer_size 

332 

333 self._max_buffer_size = max_buffer_size 

334 if read_size > self._max_buffer_size: 

335 raise ValueError("read_size must be smaller than max_buffer_size") 

336 self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) 

337 self._raw = bool(raw) 

338 self._strict_map_key = bool(strict_map_key) 

339 self._unicode_errors = unicode_errors 

340 self._use_list = use_list 

341 if not (0 <= timestamp <= 3): 

342 raise ValueError("timestamp must be 0..3") 

343 self._timestamp = timestamp 

344 self._list_hook = list_hook 

345 self._object_hook = object_hook 

346 self._object_pairs_hook = object_pairs_hook 

347 self._ext_hook = ext_hook 

348 self._max_str_len = max_str_len 

349 self._max_bin_len = max_bin_len 

350 self._max_array_len = max_array_len 

351 self._max_map_len = max_map_len 

352 self._max_ext_len = max_ext_len 

353 self._stream_offset = 0 

354 

355 if list_hook is not None and not callable(list_hook): 

356 raise TypeError("`list_hook` is not callable") 

357 if object_hook is not None and not callable(object_hook): 

358 raise TypeError("`object_hook` is not callable") 

359 if object_pairs_hook is not None and not callable(object_pairs_hook): 

360 raise TypeError("`object_pairs_hook` is not callable") 

361 if object_hook is not None and object_pairs_hook is not None: 

362 raise TypeError( 

363 "object_pairs_hook and object_hook are mutually " "exclusive" 

364 ) 

365 if not callable(ext_hook): 

366 raise TypeError("`ext_hook` is not callable") 

367 

368 def feed(self, next_bytes): 

369 assert self._feeding 

370 view = _get_data_from_buffer(next_bytes) 

371 if len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size: 

372 raise BufferFull 

373 

374 # Strip buffer before checkpoint before reading file. 

375 if self._buf_checkpoint > 0: 

376 del self._buffer[: self._buf_checkpoint] 

377 self._buff_i -= self._buf_checkpoint 

378 self._buf_checkpoint = 0 

379 

380 # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython 

381 self._buffer.extend(view) 

382 

383 def _consume(self): 

384 """Gets rid of the used parts of the buffer.""" 

385 self._stream_offset += self._buff_i - self._buf_checkpoint 

386 self._buf_checkpoint = self._buff_i 

387 

388 def _got_extradata(self): 

389 return self._buff_i < len(self._buffer) 

390 

391 def _get_extradata(self): 

392 return self._buffer[self._buff_i :] 

393 

394 def read_bytes(self, n): 

395 ret = self._read(n, raise_outofdata=False) 

396 self._consume() 

397 return ret 

398 

399 def _read(self, n, raise_outofdata=True): 

400 # (int) -> bytearray 

401 self._reserve(n, raise_outofdata=raise_outofdata) 

402 i = self._buff_i 

403 ret = self._buffer[i : i + n] 

404 self._buff_i = i + len(ret) 

405 return ret 

406 

407 def _reserve(self, n, raise_outofdata=True): 

408 remain_bytes = len(self._buffer) - self._buff_i - n 

409 

410 # Fast path: buffer has n bytes already 

411 if remain_bytes >= 0: 

412 return 

413 

414 if self._feeding: 

415 self._buff_i = self._buf_checkpoint 

416 raise OutOfData 

417 

418 # Strip buffer before checkpoint before reading file. 

419 if self._buf_checkpoint > 0: 

420 del self._buffer[: self._buf_checkpoint] 

421 self._buff_i -= self._buf_checkpoint 

422 self._buf_checkpoint = 0 

423 

424 # Read from file 

425 remain_bytes = -remain_bytes 

426 if remain_bytes + len(self._buffer) > self._max_buffer_size: 

427 raise BufferFull 

428 while remain_bytes > 0: 

429 to_read_bytes = max(self._read_size, remain_bytes) 

430 read_data = self.file_like.read(to_read_bytes) 

431 if not read_data: 

432 break 

433 assert isinstance(read_data, bytes) 

434 self._buffer += read_data 

435 remain_bytes -= len(read_data) 

436 

437 if len(self._buffer) < n + self._buff_i and raise_outofdata: 

438 self._buff_i = 0 # rollback 

439 raise OutOfData 

440 

441 def _read_header(self): 

442 typ = TYPE_IMMEDIATE 

443 n = 0 

444 obj = None 

445 self._reserve(1) 

446 b = self._buffer[self._buff_i] 

447 self._buff_i += 1 

448 if b & 0b10000000 == 0: 

449 obj = b 

450 elif b & 0b11100000 == 0b11100000: 

451 obj = -1 - (b ^ 0xFF) 

452 elif b & 0b11100000 == 0b10100000: 

453 n = b & 0b00011111 

454 typ = TYPE_RAW 

455 if n > self._max_str_len: 

456 raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) 

457 obj = self._read(n) 

458 elif b & 0b11110000 == 0b10010000: 

459 n = b & 0b00001111 

460 typ = TYPE_ARRAY 

461 if n > self._max_array_len: 

462 raise ValueError( 

463 "%s exceeds max_array_len(%s)" % (n, self._max_array_len) 

464 ) 

465 elif b & 0b11110000 == 0b10000000: 

466 n = b & 0b00001111 

467 typ = TYPE_MAP 

468 if n > self._max_map_len: 

469 raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) 

470 elif b == 0xC0: 

471 obj = None 

472 elif b == 0xC2: 

473 obj = False 

474 elif b == 0xC3: 

475 obj = True 

476 elif 0xC4 <= b <= 0xC6: 

477 size, fmt, typ = _MSGPACK_HEADERS[b] 

478 self._reserve(size) 

479 if len(fmt) > 0: 

480 n = _unpack_from(fmt, self._buffer, self._buff_i)[0] 

481 else: 

482 n = self._buffer[self._buff_i] 

483 self._buff_i += size 

484 if n > self._max_bin_len: 

485 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) 

486 obj = self._read(n) 

487 elif 0xC7 <= b <= 0xC9: 

488 size, fmt, typ = _MSGPACK_HEADERS[b] 

489 self._reserve(size) 

490 L, n = _unpack_from(fmt, self._buffer, self._buff_i) 

491 self._buff_i += size 

492 if L > self._max_ext_len: 

493 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) 

494 obj = self._read(L) 

495 elif 0xCA <= b <= 0xD3: 

496 size, fmt = _MSGPACK_HEADERS[b] 

497 self._reserve(size) 

498 if len(fmt) > 0: 

499 obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] 

500 else: 

501 obj = self._buffer[self._buff_i] 

502 self._buff_i += size 

503 elif 0xD4 <= b <= 0xD8: 

504 size, fmt, typ = _MSGPACK_HEADERS[b] 

505 if self._max_ext_len < size: 

506 raise ValueError( 

507 "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) 

508 ) 

509 self._reserve(size + 1) 

510 n, obj = _unpack_from(fmt, self._buffer, self._buff_i) 

511 self._buff_i += size + 1 

512 elif 0xD9 <= b <= 0xDB: 

513 size, fmt, typ = _MSGPACK_HEADERS[b] 

514 self._reserve(size) 

515 if len(fmt) > 0: 

516 (n,) = _unpack_from(fmt, self._buffer, self._buff_i) 

517 else: 

518 n = self._buffer[self._buff_i] 

519 self._buff_i += size 

520 if n > self._max_str_len: 

521 raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) 

522 obj = self._read(n) 

523 elif 0xDC <= b <= 0xDD: 

524 size, fmt, typ = _MSGPACK_HEADERS[b] 

525 self._reserve(size) 

526 (n,) = _unpack_from(fmt, self._buffer, self._buff_i) 

527 self._buff_i += size 

528 if n > self._max_array_len: 

529 raise ValueError( 

530 "%s exceeds max_array_len(%s)" % (n, self._max_array_len) 

531 ) 

532 elif 0xDE <= b <= 0xDF: 

533 size, fmt, typ = _MSGPACK_HEADERS[b] 

534 self._reserve(size) 

535 (n,) = _unpack_from(fmt, self._buffer, self._buff_i) 

536 self._buff_i += size 

537 if n > self._max_map_len: 

538 raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) 

539 else: 

540 raise FormatError("Unknown header: 0x%x" % b) 

541 return typ, n, obj 

542 

543 def _unpack(self, execute=EX_CONSTRUCT): 

544 typ, n, obj = self._read_header() 

545 

546 if execute == EX_READ_ARRAY_HEADER: 

547 if typ != TYPE_ARRAY: 

548 raise ValueError("Expected array") 

549 return n 

550 if execute == EX_READ_MAP_HEADER: 

551 if typ != TYPE_MAP: 

552 raise ValueError("Expected map") 

553 return n 

554 # TODO should we eliminate the recursion? 

555 if typ == TYPE_ARRAY: 

556 if execute == EX_SKIP: 

557 for i in xrange(n): 

558 # TODO check whether we need to call `list_hook` 

559 self._unpack(EX_SKIP) 

560 return 

561 ret = newlist_hint(n) 

562 for i in xrange(n): 

563 ret.append(self._unpack(EX_CONSTRUCT)) 

564 if self._list_hook is not None: 

565 ret = self._list_hook(ret) 

566 # TODO is the interaction between `list_hook` and `use_list` ok? 

567 return ret if self._use_list else tuple(ret) 

568 if typ == TYPE_MAP: 

569 if execute == EX_SKIP: 

570 for i in xrange(n): 

571 # TODO check whether we need to call hooks 

572 self._unpack(EX_SKIP) 

573 self._unpack(EX_SKIP) 

574 return 

575 if self._object_pairs_hook is not None: 

576 ret = self._object_pairs_hook( 

577 (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) 

578 for _ in xrange(n) 

579 ) 

580 else: 

581 ret = {} 

582 for _ in xrange(n): 

583 key = self._unpack(EX_CONSTRUCT) 

584 if self._strict_map_key and type(key) not in (unicode, bytes): 

585 raise ValueError( 

586 "%s is not allowed for map key" % str(type(key)) 

587 ) 

588 if not PY2 and type(key) is str: 

589 key = sys.intern(key) 

590 ret[key] = self._unpack(EX_CONSTRUCT) 

591 if self._object_hook is not None: 

592 ret = self._object_hook(ret) 

593 return ret 

594 if execute == EX_SKIP: 

595 return 

596 if typ == TYPE_RAW: 

597 if self._raw: 

598 obj = bytes(obj) 

599 else: 

600 obj = obj.decode("utf_8", self._unicode_errors) 

601 return obj 

602 if typ == TYPE_BIN: 

603 return bytes(obj) 

604 if typ == TYPE_EXT: 

605 if n == -1: # timestamp 

606 ts = Timestamp.from_bytes(bytes(obj)) 

607 if self._timestamp == 1: 

608 return ts.to_unix() 

609 elif self._timestamp == 2: 

610 return ts.to_unix_nano() 

611 elif self._timestamp == 3: 

612 return ts.to_datetime() 

613 else: 

614 return ts 

615 else: 

616 return self._ext_hook(n, bytes(obj)) 

617 assert typ == TYPE_IMMEDIATE 

618 return obj 

619 

620 def __iter__(self): 

621 return self 

622 

623 def __next__(self): 

624 try: 

625 ret = self._unpack(EX_CONSTRUCT) 

626 self._consume() 

627 return ret 

628 except OutOfData: 

629 self._consume() 

630 raise StopIteration 

631 except RecursionError: 

632 raise StackError 

633 

634 next = __next__ 

635 

636 def skip(self): 

637 self._unpack(EX_SKIP) 

638 self._consume() 

639 

640 def unpack(self): 

641 try: 

642 ret = self._unpack(EX_CONSTRUCT) 

643 except RecursionError: 

644 raise StackError 

645 self._consume() 

646 return ret 

647 

648 def read_array_header(self): 

649 ret = self._unpack(EX_READ_ARRAY_HEADER) 

650 self._consume() 

651 return ret 

652 

653 def read_map_header(self): 

654 ret = self._unpack(EX_READ_MAP_HEADER) 

655 self._consume() 

656 return ret 

657 

658 def tell(self): 

659 return self._stream_offset 

660 

661 

662class Packer(object): 

663 """ 

664 MessagePack Packer 

665 

666 Usage:: 

667 

668 packer = Packer() 

669 astream.write(packer.pack(a)) 

670 astream.write(packer.pack(b)) 

671 

672 Packer's constructor has some keyword arguments: 

673 

674 :param callable default: 

675 Convert user type to builtin type that Packer supports. 

676 See also simplejson's document. 

677 

678 :param bool use_single_float: 

679 Use single precision float type for float. (default: False) 

680 

681 :param bool autoreset: 

682 Reset buffer after each pack and return its content as `bytes`. (default: True). 

683 If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. 

684 

685 :param bool use_bin_type: 

686 Use bin type introduced in msgpack spec 2.0 for bytes. 

687 It also enables str8 type for unicode. (default: True) 

688 

689 :param bool strict_types: 

690 If set to true, types will be checked to be exact. Derived classes 

691 from serializable types will not be serialized and will be 

692 treated as unsupported type and forwarded to default. 

693 Additionally tuples will not be serialized as lists. 

694 This is useful when trying to implement accurate serialization 

695 for python types. 

696 

697 :param bool datetime: 

698 If set to true, datetime with tzinfo is packed into Timestamp type. 

699 Note that the tzinfo is stripped in the timestamp. 

700 You can get UTC datetime with `timestamp=3` option of the Unpacker. 

701 (Python 2 is not supported). 

702 

703 :param str unicode_errors: 

704 The error handler for encoding unicode. (default: 'strict') 

705 DO NOT USE THIS!! This option is kept for very specific usage. 

706 

707 Example of streaming deserialize from file-like object:: 

708 

709 unpacker = Unpacker(file_like) 

710 for o in unpacker: 

711 process(o) 

712 

713 Example of streaming deserialize from socket:: 

714 

715 unpacker = Unpacker() 

716 while True: 

717 buf = sock.recv(1024**2) 

718 if not buf: 

719 break 

720 unpacker.feed(buf) 

721 for o in unpacker: 

722 process(o) 

723 

724 Raises ``ExtraData`` when *packed* contains extra bytes. 

725 Raises ``OutOfData`` when *packed* is incomplete. 

726 Raises ``FormatError`` when *packed* is not valid msgpack. 

727 Raises ``StackError`` when *packed* contains too nested. 

728 Other exceptions can be raised during unpacking. 

729 """ 

730 

731 def __init__( 

732 self, 

733 default=None, 

734 use_single_float=False, 

735 autoreset=True, 

736 use_bin_type=True, 

737 strict_types=False, 

738 datetime=False, 

739 unicode_errors=None, 

740 ): 

741 self._strict_types = strict_types 

742 self._use_float = use_single_float 

743 self._autoreset = autoreset 

744 self._use_bin_type = use_bin_type 

745 self._buffer = StringIO() 

746 if PY2 and datetime: 

747 raise ValueError("datetime is not supported in Python 2") 

748 self._datetime = bool(datetime) 

749 self._unicode_errors = unicode_errors or "strict" 

750 if default is not None: 

751 if not callable(default): 

752 raise TypeError("default must be callable") 

753 self._default = default 

754 

755 def _pack( 

756 self, 

757 obj, 

758 nest_limit=DEFAULT_RECURSE_LIMIT, 

759 check=isinstance, 

760 check_type_strict=_check_type_strict, 

761 ): 

762 default_used = False 

763 if self._strict_types: 

764 check = check_type_strict 

765 list_types = list 

766 else: 

767 list_types = (list, tuple) 

768 while True: 

769 if nest_limit < 0: 

770 raise ValueError("recursion limit exceeded") 

771 if obj is None: 

772 return self._buffer.write(b"\xc0") 

773 if check(obj, bool): 

774 if obj: 

775 return self._buffer.write(b"\xc3") 

776 return self._buffer.write(b"\xc2") 

777 if check(obj, int_types): 

778 if 0 <= obj < 0x80: 

779 return self._buffer.write(struct.pack("B", obj)) 

780 if -0x20 <= obj < 0: 

781 return self._buffer.write(struct.pack("b", obj)) 

782 if 0x80 <= obj <= 0xFF: 

783 return self._buffer.write(struct.pack("BB", 0xCC, obj)) 

784 if -0x80 <= obj < 0: 

785 return self._buffer.write(struct.pack(">Bb", 0xD0, obj)) 

786 if 0xFF < obj <= 0xFFFF: 

787 return self._buffer.write(struct.pack(">BH", 0xCD, obj)) 

788 if -0x8000 <= obj < -0x80: 

789 return self._buffer.write(struct.pack(">Bh", 0xD1, obj)) 

790 if 0xFFFF < obj <= 0xFFFFFFFF: 

791 return self._buffer.write(struct.pack(">BI", 0xCE, obj)) 

792 if -0x80000000 <= obj < -0x8000: 

793 return self._buffer.write(struct.pack(">Bi", 0xD2, obj)) 

794 if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF: 

795 return self._buffer.write(struct.pack(">BQ", 0xCF, obj)) 

796 if -0x8000000000000000 <= obj < -0x80000000: 

797 return self._buffer.write(struct.pack(">Bq", 0xD3, obj)) 

798 if not default_used and self._default is not None: 

799 obj = self._default(obj) 

800 default_used = True 

801 continue 

802 raise OverflowError("Integer value out of range") 

803 if check(obj, (bytes, bytearray)): 

804 n = len(obj) 

805 if n >= 2**32: 

806 raise ValueError("%s is too large" % type(obj).__name__) 

807 self._pack_bin_header(n) 

808 return self._buffer.write(obj) 

809 if check(obj, unicode): 

810 obj = obj.encode("utf-8", self._unicode_errors) 

811 n = len(obj) 

812 if n >= 2**32: 

813 raise ValueError("String is too large") 

814 self._pack_raw_header(n) 

815 return self._buffer.write(obj) 

816 if check(obj, memoryview): 

817 n = obj.nbytes 

818 if n >= 2**32: 

819 raise ValueError("Memoryview is too large") 

820 self._pack_bin_header(n) 

821 return self._buffer.write(obj) 

822 if check(obj, float): 

823 if self._use_float: 

824 return self._buffer.write(struct.pack(">Bf", 0xCA, obj)) 

825 return self._buffer.write(struct.pack(">Bd", 0xCB, obj)) 

826 if check(obj, (ExtType, Timestamp)): 

827 if check(obj, Timestamp): 

828 code = -1 

829 data = obj.to_bytes() 

830 else: 

831 code = obj.code 

832 data = obj.data 

833 assert isinstance(code, int) 

834 assert isinstance(data, bytes)