Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/ijson-3.2.0.post0-py3.8-linux-x86_64.egg/ijson/common.py: 58%

1''' 

2Backend independent higher level interfaces, common exceptions. 

3''' 

4import decimal 

5import inspect 

6import warnings 

7 

8from ijson import compat, utils 

9 

10 

11class JSONError(Exception): 

12 ''' 

13 Base exception for all parsing errors. 

14 ''' 

15 pass 

16 

17 

18class IncompleteJSONError(JSONError): 

19 ''' 

20 Raised when the parser can't read expected data from a stream. 

21 ''' 

22 pass 

23 

24 

25@utils.coroutine 

26def parse_basecoro(target): 

27 ''' 

28 A coroutine dispatching parsing events with the information about their 

29 location with the JSON object tree. Events are tuples 

30 ``(prefix, type, value)``. 

31 

32 Available types and values are: 

33 

34 ('null', None) 

35 ('boolean', <True or False>) 

36 ('number', <int or Decimal>) 

37 ('string', <unicode>) 

38 ('map_key', <str>) 

39 ('start_map', None) 

40 ('end_map', None) 

41 ('start_array', None) 

42 ('end_array', None) 

43 

44 Prefixes represent the path to the nested elements from the root of the JSON 

45 document. For example, given this document:: 

46 

47 { 

48 "array": [1, 2], 

49 "map": { 

50 "key": "value" 

51 } 

52 } 

53 

54 the parser would yield events: 

55 

56 ('', 'start_map', None) 

57 ('', 'map_key', 'array') 

58 ('array', 'start_array', None) 

59 ('array.item', 'number', 1) 

60 ('array.item', 'number', 2) 

61 ('array', 'end_array', None) 

62 ('', 'map_key', 'map') 

63 ('map', 'start_map', None) 

64 ('map', 'map_key', 'key') 

65 ('map.key', 'string', u'value') 

66 ('map', 'end_map', None) 

67 ('', 'end_map', None) 

68 

69 ''' 

70 path = [] 

71 while True: 

72 event, value = yield 

73 if event == 'map_key': 

74 prefix = '.'.join(path[:-1]) 

75 path[-1] = value 

76 elif event == 'start_map': 

77 prefix = '.'.join(path) 

78 path.append(None) 

79 elif event == 'end_map': 

80 path.pop() 

81 prefix = '.'.join(path) 

82 elif event == 'start_array': 

83 prefix = '.'.join(path) 

84 path.append('item') 

85 elif event == 'end_array': 

86 path.pop() 

87 prefix = '.'.join(path) 

88 else: # any scalar value 

89 prefix = '.'.join(path) 

90 target.send((prefix, event, value)) 

91 

92 

93class ObjectBuilder(object): 

94 ''' 

95 Incrementally builds an object from JSON parser events. Events are passed 

96 into the `event` function that accepts two parameters: event type and 

97 value. The object being built is available at any time from the `value` 

98 attribute. 

99 

100 Example:: 

101 

102 >>> from ijson import basic_parse 

103 >>> from ijson.common import ObjectBuilder 

104 >>> from ijson.compat import BytesIO 

105 

106 >>> builder = ObjectBuilder() 

107 >>> f = BytesIO(b'{"key": "value"}') 

108 >>> for event, value in basic_parse(f): 

109 ... builder.event(event, value) 

110 >>> builder.value == {'key': 'value'} 

111 True 

112 

113 ''' 

114 def __init__(self, map_type=None): 

115 def initial_set(value): 

116 self.value = value 

117 self.containers = [initial_set] 

118 self.map_type = map_type or dict 

119 

120 def event(self, event, value): 

121 if event == 'map_key': 

122 self.key = value 

123 elif event == 'start_map': 

124 mappable = self.map_type() 

125 self.containers[-1](mappable) 

126 def setter(value): 

127 mappable[self.key] = value 

128 self.containers.append(setter) 

129 elif event == 'start_array': 

130 array = [] 

131 self.containers[-1](array) 

132 self.containers.append(array.append) 

133 elif event == 'end_array' or event == 'end_map': 

134 self.containers.pop() 

135 else: 

136 self.containers[-1](value) 

137 

138 

139@utils.coroutine 

140def items_basecoro(target, prefix, map_type=None): 

141 ''' 

142 An couroutine dispatching native Python objects constructed from the events 

143 under a given prefix. 

144 ''' 

145 while True: 

146 current, event, value = (yield) 

147 if current == prefix: 

148 if event in ('start_map', 'start_array'): 

149 object_depth = 1 

150 builder = ObjectBuilder(map_type=map_type) 

151 while object_depth: 

152 builder.event(event, value) 

153 current, event, value = (yield) 

154 if event in ('start_map', 'start_array'): 

155 object_depth += 1 

156 elif event in ('end_map', 'end_array'): 

157 object_depth -= 1 

158 del builder.containers[:] 

159 target.send(builder.value) 

160 else: 

161 target.send(value) 

162 

163 

164@utils.coroutine 

165def kvitems_basecoro(target, prefix, map_type=None): 

166 ''' 

167 An coroutine dispatching (key, value) pairs constructed from the events 

168 under a given prefix. The prefix should point to JSON objects 

169 ''' 

170 builder = None 

171 while True: 

172 path, event, value = (yield) 

173 while path == prefix and event == 'map_key': 

174 object_depth = 0 

175 key = value 

176 builder = ObjectBuilder(map_type=map_type) 

177 path, event, value = (yield) 

178 if event == 'start_map': 

179 object_depth += 1 

180 while ( 

181 (event != 'map_key' or object_depth != 0) and 

182 (event != 'end_map' or object_depth != -1)): 

183 builder.event(event, value) 

184 path, event, value = (yield) 

185 if event == 'start_map': 

186 object_depth += 1 

187 elif event == 'end_map': 

188 object_depth -= 1 

189 del builder.containers[:] 

190 target.send((key, builder.value)) 

191 

192 

193def integer_or_decimal(str_value): 

194 ''' 

195 Converts string with a numeric value into an int or a Decimal. 

196 Used in different backends for consistent number representation. 

197 ''' 

198 if not ('.' in str_value or 'e' in str_value or 'E' in str_value): 

199 return int(str_value) 

200 return decimal.Decimal(str_value) 

201 

202def integer_or_float(str_value): 

203 ''' 

204 Converts string with a numeric value into an int or a float. 

205 Used in different backends for consistent number representation. 

206 ''' 

207 if not ('.' in str_value or 'e' in str_value or 'E' in str_value): 

208 return int(str_value) 

209 return float(str_value) 

210 

211def number(str_value): 

212 warnings.warn("number() function will be removed in a later release", DeprecationWarning) 

213 return integer_or_decimal(str_value) 

214 

215def file_source(f, buf_size=64*1024): 

216 '''A generator that yields data from a file-like object''' 

217 f = compat.bytes_reader(f) 

218 while True: 

219 data = f.read(buf_size) 

220 yield data 

221 if not data: 

222 break 

223 

224 

225def _basic_parse_pipeline(backend, config): 

226 return ( 

227 (backend['basic_parse_basecoro'], [], config), 

228 ) 

229 

230 

231def _parse_pipeline(backend, config): 

232 return ( 

233 (backend['parse_basecoro'], [], {}), 

234 (backend['basic_parse_basecoro'], [], config) 

235 ) 

236 

237 

238def _items_pipeline(backend, prefix, map_type, config): 

239 return ( 

240 (backend['items_basecoro'], (prefix,), {'map_type': map_type}), 

241 (backend['parse_basecoro'], [], {}), 

242 (backend['basic_parse_basecoro'], [], config) 

243 ) 

244 

245 

246def _kvitems_pipeline(backend, prefix, map_type, config): 

247 return ( 

248 (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}), 

249 (backend['parse_basecoro'], [], {}), 

250 (backend['basic_parse_basecoro'], [], config) 

251 ) 

252 

253 

254def _make_basic_parse_coro(backend): 

255 def basic_parse_coro(target, **config): 

256 return utils.chain( 

257 target, 

258 *_basic_parse_pipeline(backend, config) 

259 ) 

260 return basic_parse_coro 

261 

262 

263def _make_parse_coro(backend): 

264 def parse_coro(target, **config): 

265 return utils.chain( 

266 target, 

267 *_parse_pipeline(backend, config) 

268 ) 

269 return parse_coro 

270 

271 

272def _make_items_coro(backend): 

273 def items_coro(target, prefix, map_type=None, **config): 

274 return utils.chain( 

275 target, 

276 *_items_pipeline(backend, prefix, map_type, config) 

277 ) 

278 return items_coro 

279 

280 

281def _make_kvitems_coro(backend): 

282 def kvitems_coro(target, prefix, map_type=None, **config): 

283 return utils.chain( 

284 target, 

285 *_kvitems_pipeline(backend, prefix, map_type, config) 

286 ) 

287 return kvitems_coro 

288 

289 

290def is_awaitablefunction(func): 

291 """True if `func` is an awaitable function""" 

292 return ( 

293 inspect.iscoroutinefunction(func) or ( 

294 inspect.isgeneratorfunction(func) and 

295 (func.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE) 

296 ) 

297 ) 

298 

299def is_async_file(f): 

300 """True if `f` has an asynchronous `read` method""" 

301 return ( 

302 compat.IS_PY35 and hasattr(f, 'read') and 

303 is_awaitablefunction(f.read) 

304 ) 

305 

306def is_file(x): 

307 """True if x has a `read` method""" 

308 return hasattr(x, 'read') 

309 

310 

311def is_iterable(x): 

312 """True if x can be iterated over""" 

313 return hasattr(x, '__iter__') 

314 

315 

316def _get_source(source): 

317 if isinstance(source, compat.bytetype): 

318 return compat.BytesIO(source) 

319 elif isinstance(source, compat.texttype): 

320 return compat.StringIO(source) 

321 return source 

322 

323 

324def _make_basic_parse_gen(backend): 

325 def basic_parse_gen(file_obj, buf_size=64*1024, **config): 

326 return utils.coros2gen( 

327 file_source(file_obj, buf_size=buf_size), 

328 *_basic_parse_pipeline(backend, config) 

329 ) 

330 return basic_parse_gen 

331 

332 

333def _make_parse_gen(backend): 

334 def parse_gen(file_obj, buf_size=64*1024, **config): 

335 return utils.coros2gen( 

336 file_source(file_obj, buf_size=buf_size), 

337 *_parse_pipeline(backend, config) 

338 ) 

339 return parse_gen 

340 

341 

342def _make_items_gen(backend): 

343 def items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): 

344 return utils.coros2gen( 

345 file_source(file_obj, buf_size=buf_size), 

346 *_items_pipeline(backend, prefix, map_type, config) 

347 ) 

348 return items_gen 

349 

350 

351def _make_kvitems_gen(backend): 

352 def kvitems_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): 

353 return utils.coros2gen( 

354 file_source(file_obj, buf_size=buf_size), 

355 *_kvitems_pipeline(backend, prefix, map_type, config) 

356 ) 

357 return kvitems_gen 

358 

359 

360def _make_basic_parse(backend): 

361 def basic_parse(source, buf_size=64*1024, **config): 

362 source = _get_source(source) 

363 if is_async_file(source): 

364 return backend['basic_parse_async']( 

365 source, buf_size=buf_size, **config 

366 ) 

367 elif is_file(source): 

368 return backend['basic_parse_gen']( 

369 source, buf_size=buf_size, **config 

370 ) 

371 raise ValueError("Unknown source type: %r" % type(source)) 

372 return basic_parse 

373 

374 

375def _make_parse(backend): 

376 def parse(source, buf_size=64*1024, **config): 

377 source = _get_source(source) 

378 if is_async_file(source): 

379 return backend['parse_async']( 

380 source, buf_size=buf_size, **config 

381 ) 

382 elif is_file(source): 

383 return backend['parse_gen']( 

384 source, buf_size=buf_size, **config 

385 ) 

386 elif is_iterable(source): 

387 return utils.coros2gen(source, 

388 (parse_basecoro, (), {}) 

389 ) 

390 raise ValueError("Unknown source type: %r" % type(source)) 

391 return parse 

392 

393 

394def _make_items(backend): 

395 def items(source, prefix, map_type=None, buf_size=64*1024, **config): 

396 source = _get_source(source) 

397 if is_async_file(source): 

398 return backend['items_async']( 

399 source, prefix, map_type=map_type, buf_size=buf_size, **config 

400 ) 

401 elif is_file(source): 

402 return backend['items_gen']( 

403 source, prefix, map_type=map_type, buf_size=buf_size, **config 

404 ) 

405 elif is_iterable(source): 

406 return utils.coros2gen(source, 

407 (backend['items_basecoro'], (prefix,), {'map_type': map_type}) 

408 ) 

409 raise ValueError("Unknown source type: %r" % type(source)) 

410 return items 

411 

412 

413def _make_kvitems(backend): 

414 def kvitems(source, prefix, map_type=None, buf_size=64*1024, **config): 

415 source = _get_source(source) 

416 if is_async_file(source): 

417 return backend['kvitems_async']( 

418 source, prefix, map_type=map_type, buf_size=buf_size, **config 

419 ) 

420 elif is_file(source): 

421 return backend['kvitems_gen']( 

422 source, prefix, map_type=map_type, buf_size=buf_size, **config 

423 ) 

424 elif is_iterable(source): 

425 return utils.coros2gen(source, 

426 (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}) 

427 ) 

428 raise ValueError("Unknown source type: %r" % type(source)) 

429 return kvitems 

430 

431 

432_common_functions_warn = ''' 

433Don't use the ijson.common.* functions; instead go directly with the ijson.* ones. 

434See the documentation for more information. 

435''' 

436 

437def parse(events): 

438 """Like ijson.parse, but takes events generated via ijson.basic_parse instead 

439 of a file""" 

440 warnings.warn(_common_functions_warn, DeprecationWarning) 

441 return utils.coros2gen(events, 

442 (parse_basecoro, (), {}) 

443 ) 

444 

445 

446def kvitems(events, prefix, map_type=None): 

447 """Like ijson.kvitems, but takes events generated via ijson.parse instead of 

448 a file""" 

449 warnings.warn(_common_functions_warn, DeprecationWarning) 

450 return utils.coros2gen(events, 

451 (kvitems_basecoro, (prefix,), {'map_type': map_type}) 

452 ) 

453 

454 

455def items(events, prefix, map_type=None): 

456 """Like ijson.items, but takes events generated via ijson.parse instead of 

457 a file""" 

458 warnings.warn(_common_functions_warn, DeprecationWarning) 

459 return utils.coros2gen(events, 

460 (items_basecoro, (prefix,), {'map_type': map_type}) 

461 ) 

462 

463 

464def enrich_backend(backend): 

465 ''' 

466 Provides a backend with any missing coroutines/generators/async-iterables 

467 it might be missing by using the generic ones written in python. 

468 ''' 

469 backend['backend'] = backend['__name__'].split('.')[-1] 

470 for name in ('basic_parse', 'parse', 'items', 'kvitems'): 

471 basecoro_name = name + '_basecoro' 

472 if basecoro_name not in backend: 

473 backend[basecoro_name] = globals()[basecoro_name] 

474 coro_name = name + '_coro' 

475 if coro_name not in backend: 

476 factory = globals()['_make_' + coro_name] 

477 backend[coro_name] = factory(backend) 

478 gen_name = name + '_gen' 

479 if gen_name not in backend: 

480 factory = globals()['_make_' + gen_name] 

481 backend[gen_name] = factory(backend) 

482 if compat.IS_PY35: 

483 from . import utils35 

484 async_name = name + '_async' 

485 if async_name not in backend: 

486 factory = getattr(utils35, '_make_' + async_name) 

487 backend[async_name] = factory(backend) 

488 factory = globals()['_make_' + name] 

489 backend[name] = factory(backend)