Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/nbformat/validator.py: 15%

1"""Notebook format validators."""

3# Copyright (c) IPython Development Team.

4# Distributed under the terms of the Modified BSD License.

5from __future__ import annotations

7import json

8import pprint

9import warnings

10from copy import deepcopy

11from itertools import chain

12from pathlib import Path

13from typing import Any

15from ._imports import import_item

16from .corpus.words import generate_corpus_id

17from .json_compat import ValidationError, _validator_for_name, get_current_validator

18from .reader import get_version

19from .warnings import DuplicateCellId, MissingIDFieldWarning

21validators: dict[tuple[str, int | None, int | None, bool], Any] = {}

24__all__ = [

25 "NotebookValidationError",

26 "ValidationError",

27 "better_validation_error",

28 "get_validator",

29 "isvalid",

30 "iter_validate",

31 "normalize",

32 "validate",

33]

36def _relax_additional_properties(obj):

37 """relax any `additionalProperties`"""

38 if isinstance(obj, dict):

39 for key, value in obj.items():

40 value = ( # noqa: PLW2901

41 True if key == "additionalProperties" else _relax_additional_properties(value)

42 )

43 obj[key] = value

44 elif isinstance(obj, list):

45 for i, value in enumerate(obj):

46 obj[i] = _relax_additional_properties(value)

47 return obj

50def _allow_undefined(schema):

51 schema["definitions"]["cell"]["oneOf"].append({"$ref": "#/definitions/unrecognized_cell"})

52 schema["definitions"]["output"]["oneOf"].append({"$ref": "#/definitions/unrecognized_output"})

53 return schema

56def get_validator(version=None, version_minor=None, relax_add_props=False, name=None):

57 """Load the JSON schema into a Validator"""

58 if version is None:

59 from . import current_nbformat # noqa:PLC0415

61 version = current_nbformat

63 v = import_item("nbformat.v%s" % version)

64 current_minor = getattr(v, "nbformat_minor", 0)

65 if version_minor is None:

66 version_minor = current_minor

68 current_validator = _validator_for_name(name) if name else get_current_validator()

70 # `relax_add_props` is part of the key: it produces a different schema, so a

71 # relaxed validator must not be handed out to callers asking for a strict one.

72 version_tuple = (current_validator.name, version, version_minor, relax_add_props)

74 if version_tuple not in validators:

75 try:

76 schema_json = _get_schema_json(v, version=version, version_minor=version_minor)

77 except AttributeError:

78 return None

80 if current_minor < version_minor:

81 # notebook from the future, relax all `additionalProperties: False` requirements

82 schema_json = _relax_additional_properties(schema_json)

83 # and allow undefined cell types and outputs

84 schema_json = _allow_undefined(schema_json)

86 if relax_add_props:

87 # this allows properties to be added for intermediate

88 # representations while validating for all other kinds of errors

89 schema_json = _relax_additional_properties(schema_json)

91 validators[version_tuple] = current_validator(schema_json)

93 return validators[version_tuple]

96def _get_schema_json(v, version=None, version_minor=None):

97 """

98 Gets the json schema from a given imported library and nbformat version.

99 """

100 if (version, version_minor) in v.nbformat_schema:

101 schema_path = str(Path(v.__file__).parent / v.nbformat_schema[(version, version_minor)])

102 elif version_minor > v.nbformat_minor:

103 # load the latest schema

104 schema_path = str(Path(v.__file__).parent / v.nbformat_schema[(None, None)])

105 else:

106 msg = "Cannot find appropriate nbformat schema file."

107 raise AttributeError(msg)

108 with Path(schema_path).open(encoding="utf8") as f:

109 schema_json = json.load(f)

110 return schema_json # noqa: RET504

111

112

113def isvalid(nbjson, ref=None, version=None, version_minor=None):

114 """Checks whether the given notebook JSON conforms to the current

115 notebook format schema. Returns True if the JSON is valid, and

116 False otherwise.

117

118 To see the individual errors that were encountered, please use the

119 `validate` function instead.

120 """

121 orig = deepcopy(nbjson)

122 try:

123 with warnings.catch_warnings():

124 warnings.filterwarnings("ignore", category=MissingIDFieldWarning)

125 _validate(nbjson, ref, version, version_minor, repair_duplicate_cell_ids=False)

126 except ValidationError:

127 return False

128 else:

129 return True

130 finally:

131 if nbjson != orig:

132 raise AssertionError

133

134

135def _format_as_index(indices):

136 """

137 (from jsonschema._utils.format_as_index, copied to avoid relying on private API)

138

139 Construct a single string containing indexing operations for the indices.

140

141 For example, [1, 2, "foo"] -> [1][2]["foo"]

142 """

143

144 if not indices:

145 return ""

146 return "[%s]" % "][".join(repr(index) for index in indices)

147

148

149_ITEM_LIMIT = 16

150_STR_LIMIT = 64

151

152

153def _truncate_obj(obj):

154 """Truncate objects for use in validation tracebacks

155

156 Cell and output lists are squashed, as are long strings, lists, and dicts.

157 """

158 if isinstance(obj, dict):

159 truncated_dict = {k: _truncate_obj(v) for k, v in list(obj.items())[:_ITEM_LIMIT]}

160 if isinstance(truncated_dict.get("cells"), list):

161 truncated_dict["cells"] = ["...%i cells..." % len(obj["cells"])]

162 if isinstance(truncated_dict.get("outputs"), list):

163 truncated_dict["outputs"] = ["...%i outputs..." % len(obj["outputs"])]

164

165 if len(obj) > _ITEM_LIMIT:

166 truncated_dict["..."] = "%i keys truncated" % (len(obj) - _ITEM_LIMIT)

167 return truncated_dict

168 if isinstance(obj, list):

169 truncated_list = [_truncate_obj(item) for item in obj[:_ITEM_LIMIT]]

170 if len(obj) > _ITEM_LIMIT:

171 truncated_list.append("...%i items truncated..." % (len(obj) - _ITEM_LIMIT))

172 return truncated_list

173 if isinstance(obj, str):

174 truncated_str = obj[:_STR_LIMIT]

175 if len(obj) > _STR_LIMIT:

176 truncated_str += "..."

177 return truncated_str

178 return obj

179

180

181class NotebookValidationError(ValidationError): # type:ignore[misc]

182 """Schema ValidationError with truncated representation

183

184 to avoid massive verbose tracebacks.

185 """

186

187 def __init__(self, original, ref=None):

188 """Initialize the error class."""

189 self.original = original

190 self.ref = getattr(self.original, "ref", ref)

191 self.message = self.original.message

192

193 def __getattr__(self, key):

194 """Get an attribute from the error."""

195 return getattr(self.original, key)

196

197 def __unicode__(self):

198 """Custom str for validation errors

199

200 avoids dumping full schema and notebook to logs

201 """

202 error = self.original

203 instance = _truncate_obj(error.instance)

204

205 return "\n".join(

206 [

207 error.message,

208 "",

209 "Failed validating {!r} in {}{}:".format(

210 error.validator,

211 self.ref or "notebook",

212 _format_as_index(list(error.relative_schema_path)[:-1]),

213 ),

214 "",

215 "On instance%s:" % _format_as_index(error.relative_path),

216 pprint.pformat(instance, width=78),

217 ]

218 )

219

220 __str__ = __unicode__

221

222

223def better_validation_error(error, version, version_minor):

224 """Get better ValidationError on oneOf failures

225

226 oneOf errors aren't informative.

227 if it's a cell type or output_type error,

228 try validating directly based on the type for a better error message

229 """

230 if not len(error.schema_path):

231 return error

232 key = error.schema_path[-1]

233 ref = None

234 if key.endswith("Of"):

235 if isinstance(error.instance, dict):

236 if "cell_type" in error.instance:

237 ref = error.instance["cell_type"] + "_cell"

238 elif "output_type" in error.instance:

239 ref = error.instance["output_type"]

240

241 if ref:

242 try:

243 validate(

244 error.instance,

245 ref,

246 version=version,

247 version_minor=version_minor,

248 )

249 except ValidationError as sub_error:

250 # keep extending relative path

251 error.relative_path.extend(sub_error.relative_path)

252 sub_error.relative_path = error.relative_path

253 better = better_validation_error(sub_error, version, version_minor)

254 if better.ref is None:

255 better.ref = ref

256 return better

257 except Exception: # noqa: S110, BLE001

258 # if it fails for some reason,

259 # let the original error through

260 pass

261 return NotebookValidationError(error, ref)

262

263

264def normalize(

265 nbdict: Any,

266 version: int | None = None,

267 version_minor: int | None = None,

268 *,

269 relax_add_props: bool = False,

270 strip_invalid_metadata: bool = False,

271) -> tuple[int, Any]:

272 """

273 Normalise a notebook prior to validation.

274

275 This tries to implement a couple of normalisation steps to standardise

276 notebooks and make validation easier.

277

278 You should in general not rely on this function and make sure the notebooks

279 that reach nbformat are already in a normal form. If not you likely have a bug,

280 and may have security issues.

281

282 Parameters

283 ----------

284 nbdict : dict

285 notebook document

286 version : int

287 version_minor : int

288 relax_add_props : bool

289 Whether to allow extra property in the Json schema validating the

290 notebook.

291 strip_invalid_metadata : bool

292 Whether to strip metadata that does not exist in the Json schema when

293 validating the notebook.

294

295 Returns

296 -------

297 changes : int

298 number of changes in the notebooks

299 notebook : dict

300 deep-copy of the original object with relevant changes.

301

302 """

303 nbdict = deepcopy(nbdict)

304 nbdict_version, nbdict_version_minor = get_version(nbdict)

305 if version is None:

306 version = nbdict_version

307 if version_minor is None:

308 version_minor = nbdict_version_minor

309 return _normalize(

310 nbdict,

311 version,

312 version_minor,

313 True,

314 relax_add_props=relax_add_props,

315 strip_invalid_metadata=strip_invalid_metadata,

316 )

317

318

319def _normalize(

320 nbdict: Any,

321 version: int,

322 version_minor: int,

323 repair_duplicate_cell_ids: bool,

324 relax_add_props: bool,

325 strip_invalid_metadata: bool,

326) -> tuple[int, Any]:

327 """

328 Private normalisation routine.

329

330 This function attempts to normalize the `nbdict` passed to it.

331

332 As `_normalize()` is currently used both in `validate()` (for

333 historical reasons), and in the `normalize()` public function,

334 `_normalize()` does currently mutate `nbdict`.

335 Ideally, once `validate()` stops calling `_normalize()`, `_normalize()`

336 may stop mutating `nbdict`.

337

338 """

339 changes = 0

340

341 if (version, version_minor) >= (4, 5):

342 # if we support cell ids ensure default ids are provided

343 for cell in nbdict["cells"]:

344 if "id" not in cell:

345 warnings.warn(

346 "Cell is missing an id field, this will become"

347 " a hard error in future nbformat versions. You may want"

348 " to use `normalize()` on your notebooks before validations"

349 " (available since nbformat 5.1.4). Previous versions of nbformat"

350 " are fixing this issue transparently, and will stop doing so"

351 " in the future.",

352 MissingIDFieldWarning,

353 stacklevel=3,

354 )

355 # Generate cell ids if any are missing

356 if repair_duplicate_cell_ids:

357 cell["id"] = generate_corpus_id()

358 changes += 1

359

360 # if we support cell ids check for uniqueness when validating the whole notebook

361 seen_ids = set()

362 for cell in nbdict["cells"]:

363 if "id" not in cell:

364 continue

365 cell_id = cell["id"]

366 if cell_id in seen_ids:

367 # Best effort to repair if we find a duplicate id

368 if repair_duplicate_cell_ids:

369 new_id = generate_corpus_id()

370 cell["id"] = new_id

371 changes += 1

372 warnings.warn(

373 f"Non-unique cell id {cell_id!r} detected. Corrected to {new_id!r}.",

374 DuplicateCellId,

375 stacklevel=3,

376 )

377 else:

378 msg = f"Non-unique cell id '{cell_id}' detected."

379 raise ValidationError(msg)

380 seen_ids.add(cell_id)

381 if strip_invalid_metadata:

382 changes += _strip_invalida_metadata(

383 nbdict, version, version_minor, relax_add_props=relax_add_props

384 )

385 return changes, nbdict

386

387

388def validate(

389 nbdict: Any = None,

390 ref: str | None = None,

391 version: int | None = None,

392 version_minor: int | None = None,

393 relax_add_props: bool = False,

394 nbjson: Any = None,

395) -> None:

396 """Checks whether the given notebook dict-like object

397 conforms to the relevant notebook format schema.

398

399 Parameters

400 ----------

401 nbdict : dict

402 notebook document

403 ref : optional, str

404 reference to the subset of the schema we want to validate against.

405 for example ``"markdown_cell"``, `"code_cell"` ....

406 version : int

407 version_minor : int

408 relax_add_props : bool

409 Whether to allow extra properties in the JSON schema validating the notebook.

410 When True, all known fields are validated, but unknown fields are ignored.

411 nbjson

412

413 Returns

414 -------

415 None

416

417 Raises

418 ------

419 ValidationError if not valid.

420

421 Notes

422 -----

423 Please explicitly call `normalize` if you need to normalize notebooks.

424 """

425 # backwards compatibility for nbjson argument

426 if nbdict is not None:

427 pass

428 elif nbjson is not None:

429 nbdict = nbjson

430 else:

431 msg = "validate() missing 1 required argument: 'nbdict'"

432 raise TypeError(msg)

433

434 _validate(nbdict, ref, version, version_minor, relax_add_props)

435

436

437def _validate(

438 nbdict: Any,

439 ref: str | None = None,

440 version: int | None = None,

441 version_minor: int | None = None,

442 relax_add_props: bool = False,

443 *,

444 repair_duplicate_cell_ids: bool = True,

445 strip_invalid_metadata: bool = False,

446) -> None:

447 """Validate a notebook, with explicit control over normalization behavior.

448

449 Internal callers (`isvalid`, `normalize`) use this helper to set

450 `repair_duplicate_cell_ids` and `strip_invalid_metadata` explicitly; the

451 public `validate` always uses the defaults.

452 """

453 assert isinstance(ref, str) or ref is None

454

455 if ref is None:

456 # if ref is not specified, we have a whole notebook, so we can get the version

457 nbdict_version, nbdict_version_minor = get_version(nbdict)

458 if version is None:

459 version = nbdict_version

460 if version_minor is None:

461 version_minor = nbdict_version_minor

462 # if ref is specified, and we don't have a version number, assume we're validating against 1.0

463 elif version is None:

464 version, version_minor = 1, 0

465

466 if ref is None:

467 assert isinstance(version, int)

468 assert isinstance(version_minor, int)

469 _normalize(

470 nbdict,

471 version,

472 version_minor,

473 repair_duplicate_cell_ids,

474 relax_add_props=relax_add_props,

475 strip_invalid_metadata=strip_invalid_metadata,

476 )

477

478 for error in iter_validate(

479 nbdict,

480 ref=ref,

481 version=version,

482 version_minor=version_minor,

483 relax_add_props=relax_add_props,

484 strip_invalid_metadata=strip_invalid_metadata,

485 ):

486 raise error

487

488

489def _get_errors(

490 nbdict: Any, version: int, version_minor: int, relax_add_props: bool, *args: Any

491) -> Any:

492 validator = get_validator(version, version_minor, relax_add_props=relax_add_props)

493 if not validator:

494 msg = f"No schema for validating v{version}.{version_minor} notebooks"

495 raise ValidationError(msg)

496 # Peek at the first error rather than draining the iterator: callers that only

497 # need a verdict (or only the first error) should not pay for a full traversal.

498 # `iter()` once and reuse it, since a backend may hand back a list.

499 errors = iter(validator.iter_errors(nbdict, *args))

500 first = next(errors, None)

501 if first is None:

502 return iter(())

503 # jsonschema gives the best error messages.

504 if validator.name != "jsonschema":

505 validator = get_validator(

506 version=version,

507 version_minor=version_minor,

508 relax_add_props=relax_add_props,

509 name="jsonschema",

510 )

511 return validator.iter_errors(nbdict, *args)

512 return chain((first,), errors)

513

514

515def _strip_invalida_metadata(

516 nbdict: Any, version: int, version_minor: int, relax_add_props: bool

517) -> int:

518 """

519 This function tries to extract metadata errors from the validator and fix

520 them if necessary. This mostly mean stripping unknown keys from metadata

521 fields, or removing metadata fields altogether.

522

523 Parameters

524 ----------

525 nbdict : dict

526 notebook document

527 version : int

528 version_minor : int

529 relax_add_props : bool

530 Whether to allow extra property in the Json schema validating the

531 notebook.

532

533 Returns

534 -------

535 int

536 number of modifications

537

538 """

539 errors = _get_errors(nbdict, version, version_minor, relax_add_props)

540 changes = 0

541 # only the presence of an error matters here, so don't drain the iterator

542 if next(iter(errors), None) is not None:

543 # jsonschema gives a better error tree.

544 validator = get_validator(

545 version=version,

546 version_minor=version_minor,

547 relax_add_props=relax_add_props,

548 name="jsonschema",

549 )

550 if not validator:

551 msg = f"No jsonschema for validating v{version}.{version_minor} notebooks"

552 raise ValidationError(msg)

553 errors = validator.iter_errors(nbdict)

554 error_tree = validator.error_tree(errors)

555 if "metadata" in error_tree:

556 for key in error_tree["metadata"]:

557 nbdict["metadata"].pop(key, None)

558 changes += 1

559

560 if "cells" in error_tree:

561 number_of_cells = len(nbdict.get("cells", 0))

562 for cell_idx in range(number_of_cells):

563 # Cells don't report individual metadata keys as having failed validation

564 # Instead it reports that it failed to validate against each cell-type definition.

565 # We have to delve into why those definitions failed to uncover which metadata

566 # keys are misbehaving.

567 if "oneOf" in error_tree["cells"][cell_idx].errors:

568 intended_cell_type = nbdict["cells"][cell_idx]["cell_type"]

569 schemas_by_index = [

570 ref["$ref"]

571 for ref in error_tree["cells"][cell_idx].errors["oneOf"].schema["oneOf"]

572 ]

573 cell_type_definition_name = f"#/definitions/{intended_cell_type}_cell"

574 if cell_type_definition_name in schemas_by_index:

575 schema_index = schemas_by_index.index(cell_type_definition_name)

576 for error in error_tree["cells"][cell_idx].errors["oneOf"].context:

577 rel_path = error.relative_path

578 error_for_intended_schema = error.schema_path[0] == schema_index

579 is_top_level_metadata_key = (

580 len(rel_path) == 2 and rel_path[0] == "metadata"

581 )

582 if error_for_intended_schema and is_top_level_metadata_key:

583 nbdict["cells"][cell_idx]["metadata"].pop(rel_path[1], None)

584 changes += 1

585

586 return changes

587

588

589def iter_validate(

590 nbdict=None,

591 ref=None,

592 version=None,

593 version_minor=None,

594 relax_add_props=False,

595 nbjson=None,

596 strip_invalid_metadata=False,

597):

598 """Checks whether the given notebook dict-like object conforms to the

599 relevant notebook format schema.

600

601 Returns a generator of all ValidationErrors if not valid.

602

603 Notes

604 -----

605 To fix: For security reasons, this function should *never* mutate its `nbdict` argument, and

606 should *never* try to validate a mutated or modified version of its notebook.

607

608 """

609 # backwards compatibility for nbjson argument

610 if nbdict is not None:

611 pass

612 elif nbjson is not None:

613 nbdict = nbjson

614 else:

615 msg = "iter_validate() missing 1 required argument: 'nbdict'"

616 raise TypeError(msg)

617

618 if version is None:

619 version, version_minor = get_version(nbdict)

620

621 if ref:

622 try:

623 errors = _get_errors(

624 nbdict,

625 version,

626 version_minor,

627 relax_add_props,

628 {"$ref": "#/definitions/%s" % ref},

629 )

630 except ValidationError as e:

631 yield e

632 return

633

634 else:

635 if strip_invalid_metadata:

636 _strip_invalida_metadata(nbdict, version, version_minor, relax_add_props)

637

638 # Validate one more time to ensure that us removing metadata

639 # didn't cause another complex validation issue in the schema.

640 # Also to ensure that higher-level errors produced by individual metadata validation

641 # failures are removed.

642 try:

643 errors = _get_errors(nbdict, version, version_minor, relax_add_props)

644 except ValidationError as e:

645 yield e

646 return

647

648 for error in errors:

649 yield better_validation_error(error, version, version_minor)