Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/ijson/common.py: 43%

1'''

2Backend independent higher level interfaces, common exceptions.

3'''

4import decimal

5import inspect

6import io

7import warnings

9from ijson import compat, utils, utils35

12class JSONError(Exception):

13 '''

14 Base exception for all parsing errors.

15 '''

16 pass

19class IncompleteJSONError(JSONError):

20 '''

21 Raised when the parser can't read expected data from a stream.

22 '''

23 pass

26@utils.coroutine

27def parse_basecoro(target):

28 '''

29 A coroutine dispatching parsing events with the information about their

30 location with the JSON object tree. Events are tuples

31 ``(prefix, type, value)``.

33 Available types and values are:

35 ('null', None)

36 ('boolean', <True or False>)

37 ('number', <int or Decimal>)

38 ('string', <unicode>)

39 ('map_key', <str>)

40 ('start_map', None)

41 ('end_map', None)

42 ('start_array', None)

43 ('end_array', None)

45 Prefixes represent the path to the nested elements from the root of the JSON

46 document. For example, given this document::

48 {

49 "array": [1, 2],

50 "map": {

51 "key": "value"

52 }

53 }

55 the parser would yield events:

57 ('', 'start_map', None)

58 ('', 'map_key', 'array')

59 ('array', 'start_array', None)

60 ('array.item', 'number', 1)

61 ('array.item', 'number', 2)

62 ('array', 'end_array', None)

63 ('', 'map_key', 'map')

64 ('map', 'start_map', None)

65 ('map', 'map_key', 'key')

66 ('map.key', 'string', u'value')

67 ('map', 'end_map', None)

68 ('', 'end_map', None)

70 '''

71 path = []

72 while True:

73 event, value = yield

74 if event == 'map_key':

75 prefix = '.'.join(path[:-1])

76 path[-1] = value

77 elif event == 'start_map':

78 prefix = '.'.join(path)

79 path.append(None)

80 elif event == 'end_map':

81 path.pop()

82 prefix = '.'.join(path)

83 elif event == 'start_array':

84 prefix = '.'.join(path)

85 path.append('item')

86 elif event == 'end_array':

87 path.pop()

88 prefix = '.'.join(path)

89 else: # any scalar value

90 prefix = '.'.join(path)

91 target.send((prefix, event, value))

94class ObjectBuilder:

95 '''

96 Incrementally builds an object from JSON parser events. Events are passed

97 into the `event` function that accepts two parameters: event type and

98 value. The object being built is available at any time from the `value`

99 attribute.

100

101 Example::

102

103 >>> from io import BytesIO

104 >>> from ijson import basic_parse

105 >>> from ijson.common import ObjectBuilder

106

107 >>> builder = ObjectBuilder()

108 >>> f = BytesIO(b'{"key": "value"}')

109 >>> for event, value in basic_parse(f):

110 ... builder.event(event, value)

111 >>> builder.value == {'key': 'value'}

112 True

113

114 '''

115 def __init__(self, map_type=None):

116 self.value = None

117 self.containers = []

118 self.map_type = map_type or dict

119

120 def _add(self, value):

121 # Storing the in-progress containers themselves (rather than setter

122 # closures, which would reference back to self and create reference

123 # cycles) keeps the builder collectable by plain refcounting

124 if not self.containers:

125 self.value = value

126 else:

127 top = self.containers[-1]

128 if isinstance(top, list):

129 top.append(value)

130 else:

131 top[self.key] = value

132

133 def event(self, event, value):

134 if event == 'map_key':

135 self.key = value

136 elif event == 'start_map':

137 mappable = self.map_type()

138 self._add(mappable)

139 self.containers.append(mappable)

140 elif event == 'start_array':

141 array = []

142 self._add(array)

143 self.containers.append(array)

144 elif event == 'end_array' or event == 'end_map':

145 self.containers.pop()

146 else:

147 self._add(value)

148

149

150@utils.coroutine

151def items_basecoro(target, prefix, map_type=None):

152 '''

153 An couroutine dispatching native Python objects constructed from the events

154 under a given prefix.

155 '''

156 while True:

157 current, event, value = (yield)

158 if current == prefix:

159 if event in ('start_map', 'start_array'):

160 object_depth = 1

161 builder = ObjectBuilder(map_type=map_type)

162 while object_depth:

163 builder.event(event, value)

164 current, event, value = (yield)

165 if event in ('start_map', 'start_array'):

166 object_depth += 1

167 elif event in ('end_map', 'end_array'):

168 object_depth -= 1

169 target.send(builder.value)

170 else:

171 target.send(value)

172

173

174@utils.coroutine

175def kvitems_basecoro(target, prefix, map_type=None):

176 '''

177 An coroutine dispatching (key, value) pairs constructed from the events

178 under a given prefix. The prefix should point to JSON objects

179 '''

180 builder = None

181 while True:

182 path, event, value = (yield)

183 while path == prefix and event == 'map_key':

184 object_depth = 0

185 key = value

186 builder = ObjectBuilder(map_type=map_type)

187 path, event, value = (yield)

188 if event == 'start_map':

189 object_depth += 1

190 while (

191 (event != 'map_key' or object_depth != 0) and

192 (event != 'end_map' or object_depth != -1)):

193 builder.event(event, value)

194 path, event, value = (yield)

195 if event == 'start_map':

196 object_depth += 1

197 elif event == 'end_map':

198 object_depth -= 1

199 target.send((key, builder.value))

200

201

202def integer_or_decimal(str_value):

203 '''

204 Converts string with a numeric value into an int or a Decimal.

205 Used in different backends for consistent number representation.

206 '''

207 if not ('.' in str_value or 'e' in str_value or 'E' in str_value):

208 return int(str_value)

209 return decimal.Decimal(str_value)

210

211def integer_or_float(str_value):

212 '''

213 Converts string with a numeric value into an int or a float.

214 Used in different backends for consistent number representation.

215 '''

216 if not ('.' in str_value or 'e' in str_value or 'E' in str_value):

217 return int(str_value)

218 return float(str_value)

219

220def number(str_value):

221 warnings.warn("number() function will be removed in a later release", DeprecationWarning)

222 return integer_or_decimal(str_value)

223

224def file_source(f, buf_size=64*1024):

225 '''A generator that yields data from a file-like object'''

226 f = compat.bytes_reader(f)

227 while True:

228 data = f.read(buf_size)

229 yield data

230 if not data:

231 break

232

233

234def _basic_parse_pipeline(backend, config):

235 return (

236 (backend['basic_parse_basecoro'], [], config),

237 )

238

239

240def _parse_pipeline(backend, config):

241 return (

242 (backend['parse_basecoro'], [], {}),

243 (backend['basic_parse_basecoro'], [], config)

244 )

245

246

247def _items_pipeline(backend, prefix, map_type, config):

248 return (

249 (backend['items_basecoro'], (prefix,), {'map_type': map_type}),

250 (backend['parse_basecoro'], [], {}),

251 (backend['basic_parse_basecoro'], [], config)

252 )

253

254

255def _kvitems_pipeline(backend, prefix, map_type, config):

256 return (

257 (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}),

258 (backend['parse_basecoro'], [], {}),

259 (backend['basic_parse_basecoro'], [], config)

260 )

261

262

263def _make_basic_parse_coro(backend):

264 def basic_parse_coro(target, **config):

265 return utils.chain(

266 target,

267 *_basic_parse_pipeline(backend, config)

268 )

269 return basic_parse_coro

270

271

272def _make_parse_coro(backend):

273 def parse_coro(target, **config):

274 return utils.chain(

275 target,

276 *_parse_pipeline(backend, config)

277 )

278 return parse_coro

279

280

281def _make_items_coro(backend):

282 def items_coro(target, prefix, map_type=None, **config):

283 return utils.chain(

284 target,

285 *_items_pipeline(backend, prefix, map_type, config)

286 )

287 return items_coro

288

289

290def _make_kvitems_coro(backend):

291 def kvitems_coro(target, prefix, map_type=None, **config):

292 return utils.chain(

293 target,

294 *_kvitems_pipeline(backend, prefix, map_type, config)

295 )

296 return kvitems_coro

297

298

299def is_awaitablefunction(func):

300 """True if `func` is an awaitable function"""

301 return (

302 inspect.iscoroutinefunction(func) or (

303 inspect.isgeneratorfunction(func) and

304 (func.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE)

305 )

306 )

307

308def is_async_file(f):

309 """True if `f` has an asynchronous `read` method"""

310 return (

311 hasattr(f, 'read') and

312 is_awaitablefunction(f.read)

313 )

314

315def is_file(x):

316 """True if x has a `read` method"""

317 return hasattr(x, 'read')

318

319

320def is_iterable(x):

321 """True if x can be iterated over"""

322 return hasattr(x, '__iter__')

323

324

325def _get_source(source):

326 if isinstance(source, bytes):

327 return io.BytesIO(source)

328 elif isinstance(source, str):

329 return io.StringIO(source)

330 return source

331

332

333def _make_basic_parse_gen(backend):

334 def basic_parse_gen(file_obj, buf_size=64*1024, **config):

335 return utils.coros2gen(

336 file_source(file_obj, buf_size=buf_size),

337 *_basic_parse_pipeline(backend, config)

338 )

339 return basic_parse_gen

340

341

342def _make_parse_gen(backend):

343 def parse_gen(file_obj, buf_size=64*1024, **config):

344 return utils.coros2gen(

345 file_source(file_obj, buf_size=buf_size),

346 *_parse_pipeline(backend, config)

347 )

348 return parse_gen

349

350

351def _make_items_gen(backend):

352 def items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config):

353 return utils.coros2gen(

354 file_source(file_obj, buf_size=buf_size),

355 *_items_pipeline(backend, prefix, map_type, config)

356 )

357 return items_gen

358

359

360def _make_kvitems_gen(backend):

361 def kvitems_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config):

362 return utils.coros2gen(

363 file_source(file_obj, buf_size=buf_size),

364 *_kvitems_pipeline(backend, prefix, map_type, config)

365 )

366 return kvitems_gen

367

368

369def _make_basic_parse(backend):

370 def basic_parse(source, buf_size=64*1024, **config):

371 source = _get_source(source)

372 if is_async_file(source):

373 return backend['basic_parse_async'](

374 source, buf_size=buf_size, **config

375 )

376 elif is_file(source):

377 return backend['basic_parse_gen'](

378 source, buf_size=buf_size, **config

379 )

380 raise ValueError("Unknown source type: %r" % type(source))

381 return basic_parse

382

383

384def _make_parse(backend):

385 def parse(source, buf_size=64*1024, **config):

386 source = _get_source(source)

387 if is_async_file(source):

388 return backend['parse_async'](

389 source, buf_size=buf_size, **config

390 )

391 elif is_file(source):

392 return backend['parse_gen'](

393 source, buf_size=buf_size, **config

394 )

395 elif is_iterable(source):

396 return utils.coros2gen(source,

397 (backend['parse_basecoro'], (), {})

398 )

399 raise ValueError("Unknown source type: %r" % type(source))

400 return parse

401

402

403def _make_items(backend):

404 def items(source, prefix, map_type=None, buf_size=64*1024, **config):

405 source = _get_source(source)

406 if is_async_file(source):

407 return backend['items_async'](

408 source, prefix, map_type=map_type, buf_size=buf_size, **config

409 )

410 elif is_file(source):

411 return backend['items_gen'](

412 source, prefix, map_type=map_type, buf_size=buf_size, **config

413 )

414 elif is_iterable(source):

415 return utils.coros2gen(source,

416 (backend['items_basecoro'], (prefix,), {'map_type': map_type})

417 )

418 raise ValueError("Unknown source type: %r" % type(source))

419 return items

420

421

422def _make_kvitems(backend):

423 def kvitems(source, prefix, map_type=None, buf_size=64*1024, **config):

424 source = _get_source(source)

425 if is_async_file(source):

426 return backend['kvitems_async'](

427 source, prefix, map_type=map_type, buf_size=buf_size, **config

428 )

429 elif is_file(source):

430 return backend['kvitems_gen'](

431 source, prefix, map_type=map_type, buf_size=buf_size, **config

432 )

433 elif is_iterable(source):

434 return utils.coros2gen(source,

435 (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type})

436 )

437 raise ValueError("Unknown source type: %r" % type(source))

438 return kvitems

439

440

441_common_functions_warn = '''

442Don't use the ijson.common.* functions; instead go directly with the ijson.* ones.

443See the documentation for more information.

444'''

445

446def parse(events):

447 """Like ijson.parse, but takes events generated via ijson.basic_parse instead

448 of a file"""

449 warnings.warn(_common_functions_warn, DeprecationWarning)

450 return utils.coros2gen(events,

451 (parse_basecoro, (), {})

452 )

453

454

455def kvitems(events, prefix, map_type=None):

456 """Like ijson.kvitems, but takes events generated via ijson.parse instead of

457 a file"""

458 warnings.warn(_common_functions_warn, DeprecationWarning)

459 return utils.coros2gen(events,

460 (kvitems_basecoro, (prefix,), {'map_type': map_type})

461 )

462

463

464def items(events, prefix, map_type=None):

465 """Like ijson.items, but takes events generated via ijson.parse instead of

466 a file"""

467 warnings.warn(_common_functions_warn, DeprecationWarning)

468 return utils.coros2gen(events,

469 (items_basecoro, (prefix,), {'map_type': map_type})

470 )

471

472

473class BackendCapabilities:

474 '''

475 Capabilities supported by a backend.

476 '''

477

478 __slots__ = {

479 'c_comments': 'C-ctyle comments (non-standard in JSON)',

480 'multiple_values': 'Multiple top-level values (non-standard in JSON)',

481 'invalid_leading_zeros_detection': 'Detection of leading zeros in numbers, marking them as invalid',

482 'incomplete_json_tokens_detection': 'Documents with incomplete JSON tokens',

483 'int64': '64 bit integers supported when running with ``use_float=True``',

484 }

485

486 def __init__(self):

487 self.c_comments = True

488 self.multiple_values = True

489 self.invalid_leading_zeros_detection = True

490 self.incomplete_json_tokens_detection = True

491 self.int64 = True

492

493

494def enrich_backend(backend, **capabilities_overrides):

495 '''

496 Provides a backend with any missing coroutines/generators/async-iterables

497 it might be missing by using the generic ones written in python.

498 '''

499 # Backends unset some of these

500 capabilities = BackendCapabilities()

501 for name, value in capabilities_overrides.items():

502 setattr(capabilities, name, value)

503 backend['capabilities'] = capabilities

504 backend['backend'] = backend['__name__'].split('.')[-1]

505 backend['backend_name'] = backend['backend']

506 for name in ('basic_parse', 'parse', 'items', 'kvitems'):

507 basecoro_name = name + '_basecoro'

508 if basecoro_name not in backend:

509 backend[basecoro_name] = globals()[basecoro_name]

510 coro_name = name + '_coro'

511 if coro_name not in backend:

512 factory = globals()['_make_' + coro_name]

513 backend[coro_name] = factory(backend)

514 gen_name = name + '_gen'

515 if gen_name not in backend:

516 factory = globals()['_make_' + gen_name]

517 backend[gen_name] = factory(backend)

518 async_name = name + '_async'

519 if async_name not in backend:

520 factory = getattr(utils35, '_make_' + async_name)

521 backend[async_name] = factory(backend)

522 factory = globals()['_make_' + name]

523 backend[name] = factory(backend)