Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tomlkit/container.py: 65%

1from __future__ import annotations

3import copy

4import math

6from collections.abc import Iterator

7from typing import TYPE_CHECKING

8from typing import Any

11if TYPE_CHECKING:

12 from typing import Self

14from tomlkit._compat import decode

15from tomlkit._types import _CustomDict

16from tomlkit._utils import merge_dicts

17from tomlkit.exceptions import KeyAlreadyPresent

18from tomlkit.exceptions import NonExistentKey

19from tomlkit.exceptions import TOMLKitError

20from tomlkit.items import AoT

21from tomlkit.items import Comment

22from tomlkit.items import Item

23from tomlkit.items import Key

24from tomlkit.items import Null

25from tomlkit.items import SingleKey

26from tomlkit.items import Table

27from tomlkit.items import Trivia

28from tomlkit.items import Whitespace

29from tomlkit.items import item as _item

32_NOT_SET = object()

35class Container(_CustomDict): # type: ignore[type-arg]

36 """

37 A container for items within a TOMLDocument.

39 This class implements the `dict` interface with copy/deepcopy protocol.

40 """

42 def __init__(self, parsed: bool = False) -> None:

43 self._map: dict[Key, int | tuple[int, ...]] = {}

44 self._body: list[tuple[Key | None, Item]] = []

45 self._parsed = parsed

46 self._table_keys: list[Key] = []

47 # number of already-validated fragments and the temp container they

48 # were merged into, per out-of-order key; lets parse-time validation

49 # resume where the previous pass stopped instead of re-merging every

50 # fragment (quadratic) on each append

51 self._validation_cache: dict[Key, tuple[int, Container]] = {}

52 # superset of the keys mapped to an index tuple, so validating all

53 # out-of-order tables doesn't have to scan every key in the map;

54 # stale entries are filtered by the per-key isinstance check

55 self._out_of_order_keys: set[Key] = set()

57 @property

58 def body(self) -> list[tuple[Key | None, Item]]:

59 return self._body

61 def unwrap(self) -> dict[str, Any]:

62 """Returns as pure python object (ppo)"""

63 unwrapped: dict[str, Any] = {}

64 # Resolve each key straight from _map, which already holds the parsed

65 # Key objects and their body index, instead of via self.items(): the

66 # inherited MutableMapping iteration goes through __getitem__, which

67 # rebuilds a SingleKey from the bare string on every key only to throw

68 # it away. Out-of-order keys (a tuple index) still go through

69 # OutOfOrderTableProxy so their validation (and fragment merge) runs

70 # exactly as before. _map iterates in the same insertion order as the

71 # old self.items().

72 for key, idx in self._map.items():

73 if isinstance(idx, tuple):

74 value: Any = OutOfOrderTableProxy(self, idx)

75 else:

76 value = self._body[idx][1]

77 unwrapped[key.key] = value.unwrap() if hasattr(value, "unwrap") else value

79 return unwrapped

81 @property

82 def value(self) -> dict[str, Any]:

83 """The wrapped dict value"""

84 d: dict[str, Any] = {}

85 for k, v in self._body:

86 if k is None:

87 continue

89 key_str = k.key

90 val: Any = v.value

92 if isinstance(val, Container):

93 val = val.value

95 if key_str in d:

96 merge_dicts(d[key_str], val)

97 else:

98 d[key_str] = val

100 return d

101

102 def parsing(self, parsing: bool) -> None:

103 self._parsed = parsing

104 self._validation_cache.clear()

105

106 for _, v in self._body:

107 if isinstance(v, Table):

108 v.value.parsing(parsing)

109 elif isinstance(v, AoT):

110 for t in v.body:

111 t.value.parsing(parsing)

112

113 def add(self, key: Key | Item | str, item: Any = None) -> Container:

114 """

115 Adds an item to the current Container.

116

117 :Example:

118

119 >>> # add a key-value pair

120 >>> doc.add('key', 'value')

121 >>> # add a comment or whitespace or newline

122 >>> doc.add(comment('# comment'))

123 """

124 if item is None:

125 if not isinstance(key, (Comment, Whitespace)):

126 raise ValueError(

127 "Non comment/whitespace items must have an associated key"

128 )

129

130 return self.append(None, key)

131

132 assert not isinstance(key, Item)

133 return self.append(key, item)

134

135 def _handle_dotted_key(self, key: Key, value: Item) -> None:

136 if isinstance(value, (Table, AoT)):

137 raise TOMLKitError("Can't add a table to a dotted key")

138 name, *mid, last = key

139 name._dotted = True

140 table = current = Table(Container(True), Trivia(), False, is_super_table=True)

141 for _name in mid:

142 _name._dotted = True

143 new_table = Table(Container(True), Trivia(), False, is_super_table=True)

144 current.append(_name, new_table)

145 current = new_table

146

147 last.sep = key.sep

148 current.append(last, value)

149

150 self.append(name, table)

151 return

152

153 def _get_last_index_before_table(self) -> int:

154 last_index = -1

155 for i, (k, v) in enumerate(self._body):

156 if isinstance(v, Null):

157 continue # Null elements are inserted after deletion

158

159 if isinstance(v, Whitespace) and not v.is_fixed():

160 continue

161

162 if isinstance(v, (Table, AoT)) and k is not None and not k.is_dotted():

163 break

164

165 if (

166 isinstance(v, Table)

167 and k is not None

168 and k.is_dotted()

169 and self._renders_table_header(v)

170 ):

171 # A dotted-key super table renders inline (`a.b = 1`) only as

172 # long as none of its children render a `[table]` header; once

173 # one does, anything appended after it would land inside that

174 # table's scope.

175 break

176 last_index = i

177 return last_index + 1

178

179 def _renders_table_header(self, table: Table) -> bool:

180 for k, v in table.value.body:

181 if isinstance(v, AoT):

182 return True

183 if isinstance(v, Table):

184 if k is not None and k.is_dotted() and v.is_super_table():

185 if self._renders_table_header(v):

186 return True

187 else:

188 return True

189 return False

190

191 def _validate_out_of_order_table(self, key: Key | None = None) -> None:

192 if key is None:

193 for k in list(self._out_of_order_keys):

194 assert k is not None

195 self._validate_out_of_order_table(k)

196 return

197 if key not in self._map:

198 return

199 current_idx = self._map[key]

200 if not isinstance(current_idx, tuple):

201 return

202 if self._parsed:

203 # while parsing, every fragment appended to an out-of-order key

204 # triggers a validation pass; resume from the cached temp

205 # container so each fragment is merged (and deep-copied) once

206 # instead of on every later pass. Fragments are only ever

207 # appended during parsing, so a count prefix stays valid; any

208 # other mutation clears the cache.

209 validated, temp = self._validation_cache.get(key, (0, None))

210 if validated > len(current_idx):

211 validated, temp = 0, None

212 try:

213 temp = OutOfOrderTableProxy.validate(

214 self, current_idx[validated:], temp

215 )

216 except Exception:

217 # the temp container may be partially mutated; don't let a

218 # caught-and-retried failure resume from a poisoned cache

219 self._validation_cache.pop(key, None)

220 raise

221 self._validation_cache[key] = (len(current_idx), temp)

222 return

223 OutOfOrderTableProxy.validate(self, current_idx)

224

225 def append(

226 self, key: Key | str | None, item: Any, validate: bool = True

227 ) -> Container:

228 """Similar to :meth:`add` but both key and value must be given."""

229 if not isinstance(key, Key) and key is not None:

230 key = SingleKey(key)

231

232 if not isinstance(item, Item):

233 item = _item(item)

234

235 if key is not None and key.is_multi():

236 self._handle_dotted_key(key, item)

237 return self

238

239 if isinstance(item, (AoT, Table)) and item.name is None:

240 assert isinstance(key, Key)

241 item.name = key.key

242

243 prev = self._previous_item()

244 prev_ws = isinstance(prev, Whitespace) or ends_with_whitespace(prev)

245 if isinstance(item, Table):

246 if not self._parsed:

247 item.invalidate_display_name()

248 if (

249 self._body

250 and not (self._parsed or item.trivia.indent or prev_ws)

251 and key is not None

252 and not key.is_dotted()

253 ):

254 item.trivia.indent = "\n"

255

256 if isinstance(item, AoT) and self._body and not self._parsed:

257 item.invalidate_display_name()

258 if item and not ("\n" in item[0].trivia.indent or prev_ws):

259 item[0].trivia.indent = "\n" + item[0].trivia.indent

260

261 if key is not None and key in self:

262 current_idx = self._map[key]

263 if isinstance(current_idx, tuple):

264 current_body_element = self._body[current_idx[-1]]

265 else:

266 current_body_element = self._body[current_idx]

267

268 current = current_body_element[1]

269

270 if isinstance(item, Table):

271 if not isinstance(current, (Table, AoT)):

272 raise KeyAlreadyPresent(key)

273

274 if item.is_aot_element():

275 # New AoT element found later on

276 # Adding it to the current AoT

277 if not isinstance(current, AoT):

278 current = AoT([current, item], parsed=self._parsed)

279

280 self._replace(key, key, current)

281 else:

282 current.append(item)

283

284 return self

285 elif isinstance(current, AoT):

286 if not item.is_aot_element():

287 if item.is_super_table() and len(current.body):

288 # A sub-table header such as `[fruit.apple.texture]`

289 # appearing after the array `[[fruit]]` (possibly with

290 # unrelated tables in between) extends the last element

291 # of the array, per the TOML spec.

292 last = current[-1]

293 for k, v in item.value.body:

294 last.value.append(k, v)

295

296 return self

297 # Tried to define a table after an AoT with the same name.

298 raise KeyAlreadyPresent(key)

299

300 current.append(item)

301

302 return self

303 elif current.is_super_table():

304 if item.is_super_table():

305 # We need to merge both super tables

306 if (

307 key.is_dotted()

308 or (

309 current_body_element[0] is not None

310 and current_body_element[0].is_dotted()

311 )

312 or self._table_keys[-1] != current_body_element[0]

313 ):

314 if key.is_dotted() and not self._parsed:

315 idx = self._get_last_index_before_table()

316 else:

317 idx = len(self._body)

318

319 if idx < len(self._body):

320 self._insert_at(idx, key, item)

321 else:

322 self._raw_append(key, item)

323

324 if validate:

325 self._validate_out_of_order_table(key)

326

327 return self

328

329 # Merge the new super table's body into the existing one

330 # in place. Previously this deep-copied `current` before

331 # appending, which is O(size of current) on every merge

332 # and therefore O(n^2) when many subtables share a super

333 # table (e.g. consecutive `[a.b.c]` / `[a.b.d]` headers).

334 # Mutating in place is O(1) per merge. The defensive copy

335 # that protected the out-of-order validation pass has been

336 # moved into OutOfOrderTableProxy (its only consumer).

337 for k, v in item.value.body:

338 current.append(k, v)

339

340 return self

341 elif (

342 current_body_element[0] is not None

343 and current_body_element[0].is_dotted()

344 ):

345 raise TOMLKitError("Redefinition of an existing table")

346 else:

347 # Merging a concrete table into an existing implicit/super

348 # table is only valid if it does not redefine existing

349 # subtrees via dotted keys and does not change prior types.

350 assert isinstance(current, Table)

351 self._validate_table_candidate(current, item)

352 elif not item.is_super_table():

353 raise KeyAlreadyPresent(key)

354 else:

355 # An existing concrete table (current) is being extended by

356 # a super-table (item) — e.g. [a] b=1 then [a.b] c=2 out of

357 # order, or [a] b.c=1 then [a.b] d=2. Validate that the

358 # super-table does not redefine any existing key, raising

359 # early at parse time. When validation passes, fall through

360 # — _raw_append below will create an out-of-order entry and

361 # preserve table ordering in the document.

362 assert isinstance(current, Table)

363 self._validate_table_candidate(current, item)

364 elif isinstance(item, AoT):

365 if not isinstance(current, AoT):

366 # Tried to define an AoT after a table with the same name.

367 raise KeyAlreadyPresent(key)

368

369 for table in item.body:

370 current.append(table)

371

372 return self

373 else:

374 raise KeyAlreadyPresent(key)

375

376 is_table = isinstance(item, (Table, AoT))

377 if (

378 key is not None

379 and self._body

380 and not self._parsed

381 and (not is_table or key.is_dotted())

382 ):

383 # If there is already at least one table in the current container

384 # and the given item is not a table, we need to find the last

385 # item that is not a table and insert after it

386 # If no such item exists, insert at the top of the table

387 last_index = self._get_last_index_before_table()

388

389 if last_index < len(self._body):

390 after_item = self._body[last_index][1]

391 if not (

392 isinstance(after_item, Whitespace)

393 or "\n" in after_item.trivia.indent

394 ):

395 after_item.trivia.indent = "\n" + after_item.trivia.indent

396 return self._insert_at(last_index, key, item)

397 else:

398 previous_item = self._body[-1][1]

399 if isinstance(previous_item, Table) and previous_item.is_super_table():

400 previous_child = previous_item.value._previous_item()

401 if (

402 previous_child is not None

403 and not isinstance(previous_child, Whitespace)

404 and "\n" in previous_item.trivia.trail

405 and "\n" not in previous_child.trivia.trail

406 ):

407 previous_child.trivia.trail += previous_item.trivia.trail

408 if not (

409 isinstance(previous_item, Whitespace)

410 or ends_with_whitespace(previous_item)

411 or "\n" in previous_item.trivia.trail

412 ):

413 previous_item.trivia.trail += "\n"

414

415 self._raw_append(key, item)

416 if validate and key is not None:

417 self._validate_out_of_order_table(key)

418 return self

419

420 def _validate_table_candidate(self, current: Table, candidate: Table) -> None:

421 for k, v in candidate.value.body:

422 if k is None:

423 continue

424

425 if k in current.value._map:

426 existing = current.value.item(k)

427 if isinstance(existing, (Table, AoT)) != isinstance(v, (Table, AoT)):

428 raise KeyAlreadyPresent(k)

429 if k.is_dotted():

430 raise TOMLKitError("Redefinition of an existing table")

431 if isinstance(existing, Table) and isinstance(v, Table):

432 if not existing.is_super_table() and not v.is_super_table():

433 # Both sides are concrete `[table]` definitions of the

434 # same name; the table is declared twice.

435 raise KeyAlreadyPresent(k)

436 # One side is still an implicit/super table, so a duplicate

437 # (if any) is nested deeper - keep checking the subtree.

438 self._validate_table_candidate(existing, v)

439 continue

440

441 if not k.is_dotted():

442 # Even when the candidate key itself is not dotted, an

443 # existing dotted key may already use it as a prefix —

444 # e.g. [a] b.c=1 then [a.b] d=2 (b prefixes b.c).

445 for existing_key in current.value._map:

446 if existing_key.is_dotted() and next(iter(existing_key)) == k:

447 raise TOMLKitError("Redefinition of an existing table")

448 continue

449

450 head = next(iter(k))

451 if head in current.value._map:

452 raise TOMLKitError("Redefinition of an existing table")

453

454 def _raw_append(self, key: Key | None, item: Item) -> None:

455 if key is not None and key in self._map:

456 current_idx = self._map[key]

457 if not isinstance(current_idx, tuple):

458 current_idx = (current_idx,)

459

460 current = self._body[current_idx[-1]][1]

461 if not isinstance(current, Table):

462 raise KeyAlreadyPresent(key)

463

464 self._map[key] = (*current_idx, len(self._body))

465 self._out_of_order_keys.add(key)

466 elif key is not None:

467 self._map[key] = len(self._body)

468

469 self._body.append((key, item))

470 if item.is_table() and key is not None:

471 self._table_keys.append(key)

472

473 if key is not None:

474 dict.__setitem__(self, key.key, item.value)

475

476 def _remove_at(self, idx: int) -> None:

477 key = self._body[idx][0]

478 assert key is not None

479 index = self._map.get(key)

480 if index is None:

481 raise NonExistentKey(key)

482 self._validation_cache.clear()

483 self._body[idx] = (None, Null())

484

485 if isinstance(index, tuple):

486 index_list = list(index)

487 index_list.remove(idx)

488 if len(index_list) == 1:

489 self._map[key] = index_list.pop()

490 else:

491 self._map[key] = tuple(index_list)

492 else:

493 dict.__delitem__(self, key.key)

494 self._map.pop(key)

495

496 def remove(self, key: Key | str) -> Container:

497 """Remove a key from the container."""

498 if not isinstance(key, Key):

499 key = SingleKey(key)

500

501 idx = self._map.pop(key, None)

502 if idx is None:

503 raise NonExistentKey(key)

504

505 self._validation_cache.clear()

506 if isinstance(idx, tuple):

507 for i in idx:

508 self._body[i] = (None, Null())

509 else:

510 self._body[idx] = (None, Null())

511

512 dict.__delitem__(self, key.key)

513

514 return self

515

516 def _insert_after(

517 self, key: Key | str, other_key: Key | str, item: Any

518 ) -> Container:

519 if key is None:

520 raise ValueError("Key cannot be null in insert_after()")

521

522 if key not in self:

523 raise NonExistentKey(key)

524

525 if not isinstance(key, Key):

526 key = SingleKey(key)

527

528 if not isinstance(other_key, Key):

529 other_key = SingleKey(other_key)

530

531 item = _item(item)

532

533 idx = self._map[key]

534 # Insert after the max index if there are many.

535 if isinstance(idx, tuple):

536 idx = max(idx)

537 current_item = self._body[idx][1]

538 if "\n" not in current_item.trivia.trail:

539 current_item.trivia.trail += "\n"

540

541 # Increment indices after the current index

542 for k, v in self._map.items():

543 if isinstance(v, tuple):

544 new_indices = []

545 for v_ in v:

546 if v_ > idx:

547 v_ = v_ + 1

548

549 new_indices.append(v_)

550

551 self._map[k] = tuple(new_indices)

552 elif v > idx:

553 self._map[k] = v + 1

554

555 self._map[other_key] = idx + 1

556 self._body.insert(idx + 1, (other_key, item))

557

558 if key is not None:

559 dict.__setitem__(self, other_key.key, item.value)

560

561 return self

562

563 def _insert_at(self, idx: int, key: Key | str, item: Any) -> Container:

564 if idx > len(self._body) - 1:

565 raise ValueError(f"Unable to insert at position {idx}")

566

567 if not isinstance(key, Key):

568 key = SingleKey(key)

569

570 item = _item(item)

571

572 if idx > 0:

573 previous_item = self._body[idx - 1][1]

574 if not (

575 isinstance(previous_item, Whitespace)

576 or ends_with_whitespace(previous_item)

577 or isinstance(item, (AoT, Table))

578 or "\n" in previous_item.trivia.trail

579 ):

580 previous_item.trivia.trail += "\n"

581

582 # Increment indices after the current index

583 for k, v in self._map.items():

584 if isinstance(v, tuple):

585 new_indices = []

586 for v_ in v:

587 if v_ >= idx:

588 v_ = v_ + 1

589

590 new_indices.append(v_)

591

592 self._map[k] = tuple(new_indices)

593 elif v >= idx:

594 self._map[k] = v + 1

595

596 if key in self._map:

597 current_idx = self._map[key]

598 if not isinstance(current_idx, tuple):

599 current_idx = (current_idx,)

600 self._map[key] = (*current_idx, idx)

601 self._out_of_order_keys.add(key)

602 else:

603 self._map[key] = idx

604 self._body.insert(idx, (key, item))

605

606 dict.__setitem__(self, key.key, item.value)

607

608 return self

609

610 def item(self, key: Key | str) -> Item | OutOfOrderTableProxy:

611 """Get an item for the given key."""

612 if not isinstance(key, Key):

613 key = SingleKey(key)

614

615 idx = self._map.get(key)

616 if idx is None:

617 raise NonExistentKey(key)

618

619 if isinstance(idx, tuple):

620 # The item we are getting is an out of order table

621 # so we need a proxy to retrieve the proper objects

622 # from the parent container

623 return OutOfOrderTableProxy(self, idx)

624

625 return self._body[idx][1]

626

627 def last_item(self) -> Item | None:

628 """Get the last item."""

629 if self._body:

630 return self._body[-1][1]

631 return None

632

633 def as_string(self) -> str:

634 """Render as TOML string."""

635 s = ""

636 for k, v in self._body:

637 if k is not None:

638 if isinstance(v, Table):

639 if (

640 s.strip(" ")

641 and not s.strip(" ").endswith("\n")

642 and "\n" not in v.trivia.indent

643 ):

644 s += "\n"

645 s += self._render_table(k, v)

646 elif isinstance(v, AoT):

647 if (

648 s.strip(" ")

649 and not s.strip(" ").endswith("\n")

650 and "\n" not in v.trivia.indent

651 ):

652 s += "\n"

653 s += self._render_aot(k, v)

654 else:

655 s += self._render_simple_item(k, v)

656 else:

657 s += self._render_simple_item(k, v)

658

659 return s

660

661 def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> str:

662 cur = ""

663

664 if table.display_name is not None:

665 _key = table.display_name

666 else:

667 _key = key.as_string()

668

669 if prefix is not None:

670 _key = prefix + "." + _key

671

672 if (

673 not table.is_super_table()

674 or (

675 any(

676 not isinstance(v, (Table, AoT, Whitespace, Null))

677 for _, v in table.value.body

678 )

679 and not key.is_dotted()

680 )

681 or (

682 any(

683 k is not None and k.is_dotted()

684 for k, v in table.value.body

685 if isinstance(v, Table)

686 )

687 and not key.is_dotted()

688 )

689 ):

690 open_, close = "[", "]"

691 if table.is_aot_element():

692 open_, close = "[[", "]]"

693

694 newline_in_table_trivia = (

695 "\n" if "\n" not in table.trivia.trail and len(table.value) > 0 else ""

696 )

697 cur += (

698 f"{table.trivia.indent}"

699 f"{open_}"

700 f"{decode(_key)}"

701 f"{close}"

702 f"{table.trivia.comment_ws}"

703 f"{decode(table.trivia.comment)}"

704 f"{table.trivia.trail}"

705 f"{newline_in_table_trivia}"

706 )

707 elif table.trivia.indent == "\n":

708 cur += table.trivia.indent

709

710 for k, v in table.value.body:

711 if isinstance(v, Table):

712 if (

713 cur.strip(" ")

714 and not cur.strip(" ").endswith("\n")

715 and "\n" not in v.trivia.indent

716 ):

717 cur += "\n"

718 assert k is not None

719 if v.is_super_table():

720 if k.is_dotted() and not key.is_dotted():

721 # Dotted key inside table

722 cur += self._render_table(k, v)

723 else:

724 cur += self._render_table(k, v, prefix=_key)

725 else:

726 cur += self._render_table(k, v, prefix=_key)

727 elif isinstance(v, AoT):

728 if (

729 cur.strip(" ")

730 and not cur.strip(" ").endswith("\n")

731 and "\n" not in v.trivia.indent

732 ):

733 cur += "\n"

734 assert k is not None

735 cur += self._render_aot(k, v, prefix=_key)

736 else:

737 cur += self._render_simple_item(

738 k, v, prefix=_key if key.is_dotted() else None

739 )

740

741 return cur

742

743 def _render_aot(self, key: Key, aot: AoT, prefix: str | None = None) -> str:

744 _key = key.as_string()

745 if prefix is not None:

746 _key = prefix + "." + _key

747

748 cur = ""

749 _key = decode(_key)

750 for table in aot.body:

751 cur += self._render_aot_table(table, prefix=_key)

752

753 return cur

754

755 def _render_aot_table(self, table: Table, prefix: str | None = None) -> str:

756 cur = ""

757 _key = prefix or ""

758 open_, close = "[[", "]]"

759

760 cur += (

761 f"{table.trivia.indent}"

762 f"{open_}"

763 f"{decode(_key)}"

764 f"{close}"

765 f"{table.trivia.comment_ws}"

766 f"{decode(table.trivia.comment)}"

767 f"{table.trivia.trail}"

768 )

769

770 for k, v in table.value.body:

771 if isinstance(v, Table):

772 assert k is not None

773 if v.is_super_table():

774 if k.is_dotted():

775 # Dotted key inside table

776 cur += self._render_table(k, v)

777 else:

778 cur += self._render_table(k, v, prefix=_key)

779 else:

780 cur += self._render_table(k, v, prefix=_key)

781 elif isinstance(v, AoT):

782 assert k is not None

783 cur += self._render_aot(k, v, prefix=_key)

784 else:

785 cur += self._render_simple_item(k, v)

786

787 return cur

788

789 def _render_simple_item(

790 self, key: Key | None, item: Item, prefix: str | None = None

791 ) -> str:

792 if key is None:

793 return item.as_string()

794

795 _key = key.as_string()

796 if prefix is not None:

797 _key = prefix + "." + _key

798

799 return (

800 f"{item.trivia.indent}"

801 f"{decode(_key)}"

802 f"{key.sep}"

803 f"{decode(item.as_string())}"

804 f"{item.trivia.comment_ws}"

805 f"{decode(item.trivia.comment)}"

806 f"{item.trivia.trail}"

807 )

808

809 def __len__(self) -> int:

810 return dict.__len__(self)

811

812 def __iter__(self) -> Iterator[str]:

813 return iter(dict.keys(self))

814

815 # Dictionary methods

816 def __getitem__(self, key: Key | str) -> Any:

817 item = self.item(key)

818 if isinstance(item, Item) and item.is_boolean():

819 return item.value

820

821 return item

822

823 def __contains__(self, key: object) -> bool:

824 # Native membership test. The inherited ``MutableMapping.__contains__``

825 # resolves the value via ``__getitem__``/``item()`` (and builds a

826 # ``NonExistentKey`` on every absent key) only to discard it. Resolve the

827 # key the same way ``item()`` does -- ``str`` becomes a ``SingleKey``

828 # (a non-str/non-``Key`` argument still raises ``TypeError``) -- then

829 # probe ``_map`` directly. For an out-of-order table the proxy is still

830 # built so its validation runs exactly as before.

831 if not isinstance(key, Key):

832 key = SingleKey(key) # type: ignore[arg-type]

833 idx = self._map.get(key)

834 if idx is None:

835 return False

836 if isinstance(idx, tuple):

837 # during parsing every fragment append re-probes membership;

838 # if the validation cache covers exactly these fragments they

839 # are already known to merge cleanly, so skip rebuilding the

840 # proxy (which is linear in the number of fragments)

841 validated, _ = self._validation_cache.get(key, (0, None))

842 if not (self._parsed and validated == len(idx)):

843 OutOfOrderTableProxy(self, idx)

844 return True

845

846 def __setitem__(self, key: Key | str, value: Any) -> None:

847 if key in self:

848 old_key = next(filter(lambda k: k == key, self._map))

849 self._replace(old_key, key, value)

850 else:

851 self.append(key, value)

852

853 def __delitem__(self, key: Key | str) -> None:

854 self.remove(key)

855

856 def setdefault(self, key: Key | str, default: Any = None) -> Any:

857 if key not in self:

858 self[key] = default

859 return self[key]

860

861 def _replace(self, key: Key | str, new_key: Key | str, value: Item) -> None:

862 if not isinstance(key, Key):

863 key = SingleKey(key)

864

865 idx = self._map.get(key)

866 if idx is None:

867 raise NonExistentKey(key)

868

869 self._replace_at(idx, new_key, value)

870

871 def _replace_at(

872 self, idx: int | tuple[int, ...], new_key: Key | str, value: Item

873 ) -> None:

874 value = _item(value)

875 self._validation_cache.clear()

876

877 if isinstance(idx, tuple):

878 for i in idx[1:]:

879 self._body[i] = (None, Null())

880

881 idx = idx[0]

882

883 k, v = self._body[idx]

884 assert k is not None

885 # A dotted key renders its value inline (e.g. ``a.b = 1``), which is only

886 # consistent with a super table. When the replacement value renders with

887 # its own ``[header]`` instead (a non-super table, or an array of tables),

888 # keeping the dotted key duplicates the prefix onto the header (#524) and

889 # lets the header swallow following dotted siblings (#542). Drop the dotted

890 # key so the replacement renders as a plain table/array of tables.

891 dotted_to_header = k.is_dotted() and (

892 isinstance(value, AoT)

893 or (isinstance(value, Table) and not value.is_super_table())

894 )

895 # That new header also captures every sibling that renders inline -- plain

896 # values and dotted keys -- if any still follow it (#513), so it must be

897 # moved past them, exactly as a value-to-table change already is.

898 reposition_dotted = dotted_to_header and any(

899 not isinstance(cur_val, (Null, Whitespace))

900 and not (

901 isinstance(cur_val, (Table, AoT))

902 and (cur_key is None or not cur_key.is_dotted())

903 )

904 for cur_key, cur_val in self._body[idx + 1 :]

905 )

906 if not isinstance(new_key, Key):

907 if (

908 isinstance(value, (AoT, Table)) != isinstance(v, (AoT, Table))

909 or new_key != k.key

910 or dotted_to_header

911 ):

912 new_key = SingleKey(new_key)

913 else: # Inherit the sep of the old key

914 new_key = k

915

916 del self._map[k]

917 self._map[new_key] = idx

918 if new_key != k:

919 dict.__delitem__(self, k.key)

920

921 if (

922 isinstance(value, (AoT, Table)) != isinstance(v, (AoT, Table))

923 or reposition_dotted

924 ):

925 self.remove(k)

926 if isinstance(value, (AoT, Table)):

927 # New tables must appear after all entries that render inline:

928 # plain values and dotted keys (which are super tables). Skip

929 # those and insert before the first real ``[header]`` so the new

930 # table cannot swallow a following sibling on round-trip.

931 for i in range(idx, len(self._body)):

932 cur_key, cur_val = self._body[i]

933 if isinstance(cur_val, (AoT, Table)) and not (

934 cur_key is not None and cur_key.is_dotted()

935 ):

936 self._insert_at(i, new_key, value)

937 idx = i

938 break

939 else:

940 idx = -1

941 self.append(new_key, value)

942 else:

943 # the replaced table's slot lies in the table region, where a

944 # plain value would be captured by the preceding table on

945 # round-trip; append() puts it with the other root-level values

946 idx = -1

947 self.append(new_key, value)

948 else:

949 # Copying trivia

950 if not isinstance(value, (Whitespace, AoT)):

951 value.trivia.indent = v.trivia.indent

952 value.trivia.comment_ws = value.trivia.comment_ws or v.trivia.comment_ws

953 value.trivia.comment = value.trivia.comment or v.trivia.comment

954 value.trivia.trail = v.trivia.trail

955 self._body[idx] = (new_key, value)

956

957 if value is not v and hasattr(value, "invalidate_display_name"):

958 value.invalidate_display_name()

959

960 if isinstance(value, Table):

961 # Insert a cosmetic new line for tables if:

962 # - it does not have it yet OR is not followed by one

963 # - it is not the last item, or

964 # - The table being replaced has a newline

965 result = self._previous_item_with_index()

966 assert result is not None

967 last, _ = result

968 idx = last if idx < 0 else idx

969 has_ws = ends_with_whitespace(value)

970 replace_has_ws = (

971 isinstance(v, Table)

972 and v.value.body

973 and isinstance(v.value.body[-1][1], Whitespace)

974 )

975 next_ws = idx < last and isinstance(self._body[idx + 1][1], Whitespace)

976 if (idx < last or replace_has_ws) and not (next_ws or has_ws):

977 value.append(None, Whitespace("\n"))

978

979 assert isinstance(new_key, Key)

980 dict.__setitem__(self, new_key.key, value.value)

981

982 def __str__(self) -> str:

983 return str(self.value)

984

985 def __repr__(self) -> str:

986 return repr(self.value)

987

988 def __eq__(self, other: object) -> bool:

989 if not isinstance(other, dict):

990 return NotImplemented

991

992 return bool(_equal_with_nan(self.value, other))

993

994 def _getstate(self, protocol: int) -> tuple[bool]:

995 return (self._parsed,)

996

997 def __reduce__(self) -> tuple[type, tuple[bool], tuple[Any, ...]]:

998 return self.__reduce_ex__(2)

999

1000 def __reduce_ex__(self, protocol: int) -> tuple[type, tuple[bool], tuple[Any, ...]]: # type: ignore[override]

1001 return (

1002 self.__class__,

1003 self._getstate(protocol),

1004 (self._map, self._body, self._parsed, self._table_keys),

1005 )

1006

1007 def __setstate__(self, state: tuple[Any, ...]) -> None:

1008 self._map = state[0]

1009 self._body = state[1]

1010 self._parsed = state[2]

1011 self._table_keys = state[3]

1012 self._out_of_order_keys = {

1013 k for k, v in self._map.items() if isinstance(v, tuple)

1014 }

1015

1016 for key, item in self._body:

1017 if key is not None:

1018 dict.__setitem__(self, key.key, item.value)

1019

1020 def copy(self) -> Self:

1021 return copy.copy(self)

1022

1023 def __copy__(self) -> Self:

1024 c = self.__class__(self._parsed)

1025 for k, v in dict.items(self):

1026 dict.__setitem__(c, k, v)

1027

1028 c._body += self.body

1029 c._map.update(self._map)

1030 c._out_of_order_keys |= self._out_of_order_keys

1031

1032 return c

1033

1034 def _previous_item_with_index(

1035 self, idx: int | None = None, ignore: tuple[type, ...] = (Null,)

1036 ) -> tuple[int, Item] | None:

1037 """Find the immediate previous item before index ``idx``"""

1038 if idx is None or idx > len(self._body):

1039 idx = len(self._body)

1040 for i in range(idx - 1, -1, -1):

1041 v = self._body[i][-1]

1042 if not isinstance(v, ignore):

1043 return i, v

1044 return None

1045

1046 def _previous_item(

1047 self, idx: int | None = None, ignore: tuple[type, ...] = (Null,)

1048 ) -> Item | None:

1049 """Find the immediate previous item before index ``idx``.

1050 If ``idx`` is not given, the last item is returned.

1051 """

1052 prev = self._previous_item_with_index(idx, ignore)

1053 return prev[-1] if prev else None

1054

1055

1056class OutOfOrderTableProxy(_CustomDict): # type: ignore[type-arg]

1057 @staticmethod

1058 def validate(

1059 container: Container,

1060 indices: tuple[int, ...],

1061 temp_container: Container | None = None,

1062 ) -> Container:

1063 """Validate out of order tables in the given container"""

1064 # Append all items to a temp container to see if there is any error.

1065 # We deep-copy each value before appending: appending a super table

1066 # merges it in place into a matching one already in temp_container, and

1067 # those values are the live subtables of `container`. Without the copy

1068 # the merge would mutate the caller's tables (and corrupt a later

1069 # validation pass). The container merge itself is now copy-free for

1070 # speed, so this is where the isolation lives.

1071 # Passing a previous pass's temp container back in (with the already

1072 # validated indices stripped from `indices`) resumes the validation

1073 # incrementally.

1074 if temp_container is None:

1075 temp_container = Container(True)

1076 for i in indices:

1077 _, item = container._body[i]

1078

1079 if isinstance(item, Table):

1080 for k, v in item.value.body:

1081 temp_container.append(k, copy.deepcopy(v), validate=True)

1082

1083 temp_container._validate_out_of_order_table()

1084 return temp_container

1085

1086 def __init__(self, container: Container, indices: tuple[int, ...]) -> None:

1087 self._container = container

1088 self._internal_container = Container(True)

1089 self._tables: list[Table] = []

1090 self._tables_map: dict[Key, list[int]] = {}

1091

1092 for i in indices:

1093 _, _item = self._container._body[i]

1094

1095 if isinstance(_item, Table):

1096 self._tables.append(_item)

1097 table_idx = len(self._tables) - 1

1098 for k, v in _item.value.body:

1099 merged = self._merge_aot_fragment(k, v)

1100 if merged is None:

1101 self._internal_container._raw_append(k, v)

1102 else:

1103 v = merged

1104 key_indices = self._tables_map.setdefault(k, []) # type: ignore[arg-type]

1105 if table_idx not in key_indices:

1106 key_indices.append(table_idx)

1107 if k is not None:

1108 dict.__setitem__(self, k.key, v)

1109

1110 self._internal_container._validate_out_of_order_table()

1111

1112 def _merge_aot_fragment(self, key: Key | None, item: Item) -> AoT | None:

1113 """

1114 Merge an array-of-tables fragment from a later out-of-order table part.

1115

1116 An AoT whose elements are split across out-of-order parts of the same

1117 table arrives here once per part; ``_raw_append`` only knows how to

1118 chain duplicate ``Table`` parts and would raise ``KeyAlreadyPresent``.

1119 The fragments are presented as a new merged ``AoT`` referencing the

1120 live element tables, without mutating either fragment (the parts keep

1121 rendering their own elements).

1122

1123 Returns the merged ``AoT``, or ``None`` if this is not such a fragment.

1124 """

1125 internal = self._internal_container

1126 if key is None or not isinstance(item, AoT) or key not in internal._map:

1127 return None

1128 idx = internal._map[key]

1129 if isinstance(idx, tuple):

1130 # A tuple index means the key already resolved to several body

1131 # positions, which here only happens for a degenerate collision

1132 # (e.g. a non-AoT part sharing the key). Three or more genuine AoT

1133 # fragments do not reach this branch: each later fragment merges

1134 # into the single growing AoT below, so ``idx`` stays an int.

1135 return None

1136 existing = internal._body[idx][1]

1137 if not isinstance(existing, AoT):

1138 return None

1139

1140 merged = AoT([*existing.body, *item.body], parsed=True)

1141 internal._body[idx] = (internal._body[idx][0], merged)

1142 dict.__setitem__(internal, key.key, merged.value)

1143 return merged

1144

1145 def unwrap(self) -> dict[str, Any]:

1146 return self._internal_container.unwrap()

1147

1148 @property

1149 def value(self) -> dict[str, Any]:

1150 return self._internal_container.value

1151

1152 def __str__(self) -> str:

1153 return str(self.value)

1154

1155 def __repr__(self) -> str:

1156 return repr(self.value)

1157

1158 def __contains__(self, key: object) -> bool:

1159 # Native membership test. The inherited ``MutableMapping.__contains__``

1160 # resolves the value via ``__getitem__`` (and builds a ``NonExistentKey``

1161 # on every absent key) only to discard it. Probe the internal container

1162 # directly -- the same predicate ``__getitem__`` already uses -- which is

1163 # itself a native ``_map`` lookup that still rebuilds the proxy for an

1164 # out-of-order key so its validation runs exactly as before.

1165 return key in self._internal_container

1166

1167 def __getitem__(self, key: Key | str) -> Any:

1168 if key not in self._internal_container:

1169 raise NonExistentKey(key)

1170

1171 return self._internal_container[key]

1172

1173 def __setitem__(self, key: Key | str, value: Any) -> None:

1174 from .items import item as _item_fn

1175

1176 def _is_table_or_aot(it: Any) -> bool:

1177 return isinstance(_item_fn(it), (Table, AoT))

1178

1179 _key: Key = key if isinstance(key, Key) else SingleKey(key)

1180

1181 if _key in self._tables_map:

1182 # Overwrite the first table and remove others

1183 map_indices = self._tables_map[_key]

1184 while len(map_indices) > 1:

1185 table = self._tables[map_indices.pop()]

1186 self._remove_table(table)

1187 old_value = self._tables[map_indices[0]][key]

1188 if _is_table_or_aot(old_value) and not _is_table_or_aot(value):

1189 # Remove the entry from the map and set value again.

1190 del self._tables[map_indices[0]][key]

1191 del self._tables_map[_key]

1192 self[key] = value

1193 return

1194 self._tables[map_indices[0]][key] = value

1195 elif self._tables:

1196 if not _is_table_or_aot(value): # if the value is a plain value

1197 for table in self._tables:

1198 # find the first table that allows plain values

1199 if any(not _is_table_or_aot(v) for _, v in table.items()):

1200 table[key] = value

1201 break

1202 else:

1203 # No part holds a plain value yet, so the chosen part must

1204 # start rendering its own ``[key]`` header. Prefer a part

1205 # that is not a super table (it already renders that header):

1206 # turning a header-less super part concrete here would emit a

1207 # second, duplicate header next to an existing concrete part

1208 # and produce TOML that no longer parses.

1209 for table in self._tables:

1210 if not table.is_super_table():

1211 table[key] = value

1212 break

1213 else:

1214 self._tables[0][key] = value

1215 else:

1216 self._tables[0][key] = value

1217 else:

1218 self._container[key] = value

1219

1220 self._internal_container[key] = value

1221 if key is not None:

1222 dict.__setitem__(self, key, value)

1223

1224 def _remove_table(self, table: Table) -> None:

1225 """Remove table from the parent container"""

1226 self._tables.remove(table)

1227 for idx, body_item in enumerate(self._container._body):

1228 if body_item[1] is table:

1229 self._container._remove_at(idx)

1230 break

1231

1232 def __delitem__(self, key: Key | str) -> None:

1233 _key: Key = key if isinstance(key, Key) else SingleKey(key)

1234 if _key not in self._tables_map:

1235 raise NonExistentKey(key)

1236

1237 for i in reversed(self._tables_map[_key]):

1238 table = self._tables[i]

1239 del table[key]

1240 if not table and len(self._tables) > 1:

1241 self._remove_table(table)

1242

1243 del self._tables_map[_key]

1244 del self._internal_container[key]

1245 if key is not None:

1246 dict.__delitem__(self, key)

1247

1248 def __iter__(self) -> Iterator[str]:

1249 return iter(dict.keys(self))

1250

1251 def __len__(self) -> int:

1252 return dict.__len__(self)

1253

1254 def setdefault(self, key: Key | str, default: Any = None) -> Any:

1255 if key not in self:

1256 self[key] = default

1257 return self[key]

1258

1259

1260def ends_with_whitespace(it: Any) -> bool:

1261 """Returns ``True`` if the given item ``it`` is a ``Table`` or ``AoT`` object

1262 ending with a ``Whitespace``.

1263 """

1264 if isinstance(it, Whitespace):

1265 return True

1266 if isinstance(it, Table):

1267 previous = it.value._previous_item()

1268 return previous is not None and ends_with_whitespace(previous)

1269 return isinstance(it, AoT) and len(it) > 0 and ends_with_whitespace(it[-1])

1270

1271

1272def _equal_with_nan(left: Any, right: Any) -> bool:

1273 if isinstance(left, dict) and isinstance(right, dict):

1274 if left.keys() != right.keys():

1275 return False

1276 return all(_equal_with_nan(left[k], right[k]) for k in left)

1277

1278 if isinstance(left, list) and isinstance(right, list):

1279 if len(left) != len(right):

1280 return False

1281 return all(_equal_with_nan(l, r) for l, r in zip(left, right)) # noqa: B905, E741

1282

1283 if isinstance(left, float) and isinstance(right, float):

1284 if math.isnan(left) and math.isnan(right):

1285 return True

1286

1287 return bool(left == right)