Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tomlkit/container.py: 65%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import copy
4import math
6from collections.abc import Iterator
7from typing import TYPE_CHECKING
8from typing import Any
11if TYPE_CHECKING:
12 from typing import Self
14from tomlkit._compat import decode
15from tomlkit._types import _CustomDict
16from tomlkit._utils import merge_dicts
17from tomlkit.exceptions import KeyAlreadyPresent
18from tomlkit.exceptions import NonExistentKey
19from tomlkit.exceptions import TOMLKitError
20from tomlkit.items import AoT
21from tomlkit.items import Comment
22from tomlkit.items import Item
23from tomlkit.items import Key
24from tomlkit.items import Null
25from tomlkit.items import SingleKey
26from tomlkit.items import Table
27from tomlkit.items import Trivia
28from tomlkit.items import Whitespace
29from tomlkit.items import item as _item
32_NOT_SET = object()
35def _is_empty_aot(item: Item) -> bool:
36 """Whether ``item`` is an array of tables with no elements left."""
37 return isinstance(item, AoT) and not item.body
40def _emits_header(key: Key, table: Table) -> bool:
41 """Whether rendering ``table`` under ``key`` emits a ``[header]`` line.
43 A super table normally renders as nothing but the path prefix of its
44 children, so it opens no scope of its own.
45 """
46 return (
47 not table.is_super_table()
48 or (
49 any(
50 not isinstance(v, (Table, AoT, Whitespace, Null))
51 for _, v in table.value.body
52 )
53 and not key.is_dotted()
54 )
55 or (
56 any(
57 k is not None and k.is_dotted()
58 for k, v in table.value.body
59 if isinstance(v, Table)
60 )
61 and not key.is_dotted()
62 )
63 )
66def _header_less_aots(
67 body: list[tuple[Key | None, Item]], prefix: str = ""
68) -> list[tuple[str, AoT]]:
69 """Every emptied array of tables reachable without crossing a header.
71 An emptied array of tables has no ``[[key]]`` header left to render, so it
72 falls back to the inline ``key = []`` form -- and TOML reads a bare
73 key/value pair into whichever table the closest preceding header opened.
74 So the fallbacks of a scope have to be written at the top of that scope,
75 with a key relative to it, and that includes the ones sitting inside super
76 tables, which emit no header to separate them.
78 Returns ``(dotted key, aot)`` pairs, keyed relative to the scope ``body``
79 belongs to.
80 """
81 found: list[tuple[str, AoT]] = []
82 for k, v in body:
83 if k is None:
84 continue
85 path = f"{prefix}.{k.as_string()}" if prefix else k.as_string()
86 if _is_empty_aot(v):
87 found.append((path, v))
88 elif isinstance(v, Table) and not _emits_header(k, v):
89 found.extend(_header_less_aots(v.value.body, path))
90 return found
93def _render_empty_aot(key: str, aot: AoT) -> str:
94 """Render an emptied array of tables as an inline empty array."""
95 return (
96 f"{aot.trivia.indent}"
97 f"{decode(key)}"
98 f" = []"
99 f"{aot.trivia.comment_ws}"
100 f"{decode(aot.trivia.comment)}"
101 f"{aot.trivia.trail or chr(10)}"
102 )
105class Container(_CustomDict): # type: ignore[type-arg]
106 """
107 A container for items within a TOMLDocument.
109 This class implements the `dict` interface with copy/deepcopy protocol.
110 """
112 def __init__(self, parsed: bool = False) -> None:
113 self._map: dict[Key, int | tuple[int, ...]] = {}
114 self._body: list[tuple[Key | None, Item]] = []
115 self._parsed = parsed
116 self._table_keys: list[Key] = []
117 # number of already-validated fragments and the temp container they
118 # were merged into, per out-of-order key; lets parse-time validation
119 # resume where the previous pass stopped instead of re-merging every
120 # fragment (quadratic) on each append
121 self._validation_cache: dict[Key, tuple[int, Container]] = {}
122 # superset of the keys mapped to an index tuple, so validating all
123 # out-of-order tables doesn't have to scan every key in the map;
124 # stale entries are filtered by the per-key isinstance check
125 self._out_of_order_keys: set[Key] = set()
127 @property
128 def body(self) -> list[tuple[Key | None, Item]]:
129 return self._body
131 def unwrap(self) -> dict[str, Any]:
132 """Returns as pure python object (ppo)"""
133 unwrapped: dict[str, Any] = {}
134 # Resolve each key straight from _map, which already holds the parsed
135 # Key objects and their body index, instead of via self.items(): the
136 # inherited MutableMapping iteration goes through __getitem__, which
137 # rebuilds a SingleKey from the bare string on every key only to throw
138 # it away. Out-of-order keys (a tuple index) still go through
139 # OutOfOrderTableProxy so their validation (and fragment merge) runs
140 # exactly as before. _map is keyed for lookup, not ordered: replacing a
141 # value re-inserts its key and moves it last, so take the order from the
142 # body index, which is what dumps() and items() follow.
143 for key, idx in sorted(
144 self._map.items(),
145 key=lambda item: item[1][0] if isinstance(item[1], tuple) else item[1],
146 ):
147 if isinstance(idx, tuple):
148 value: Any = OutOfOrderTableProxy(self, idx)
149 else:
150 value = self._body[idx][1]
151 unwrapped[key.key] = value.unwrap() if hasattr(value, "unwrap") else value
153 return unwrapped
155 @property
156 def value(self) -> dict[str, Any]:
157 """The wrapped dict value"""
158 d: dict[str, Any] = {}
159 for k, v in self._body:
160 if k is None:
161 continue
163 key_str = k.key
164 val: Any = v.value
166 if isinstance(val, Container):
167 val = val.value
169 if key_str in d:
170 merge_dicts(d[key_str], val)
171 else:
172 d[key_str] = val
174 return d
176 def parsing(self, parsing: bool) -> None:
177 self._parsed = parsing
178 self._validation_cache.clear()
180 for _, v in self._body:
181 if isinstance(v, Table):
182 v.value.parsing(parsing)
183 elif isinstance(v, AoT):
184 for t in v.body:
185 t.value.parsing(parsing)
187 def add(self, key: Key | Item | str, item: Any = None) -> Container:
188 """
189 Adds an item to the current Container.
191 :Example:
193 >>> # add a key-value pair
194 >>> doc.add('key', 'value')
195 >>> # add a comment or whitespace or newline
196 >>> doc.add(comment('# comment'))
197 """
198 if item is None:
199 if not isinstance(key, (Comment, Whitespace)):
200 raise ValueError(
201 "Non comment/whitespace items must have an associated key"
202 )
204 return self.append(None, key)
206 assert not isinstance(key, Item)
207 return self.append(key, item)
209 def _handle_dotted_key(self, key: Key, value: Item) -> None:
210 if isinstance(value, (Table, AoT)):
211 raise TOMLKitError("Can't add a table to a dotted key")
212 name, *mid, last = key
213 name._dotted = True
214 table = current = Table(Container(True), Trivia(), False, is_super_table=True)
215 for _name in mid:
216 _name._dotted = True
217 new_table = Table(Container(True), Trivia(), False, is_super_table=True)
218 current.append(_name, new_table)
219 current = new_table
221 last.sep = key.sep
222 current.append(last, value)
224 self.append(name, table)
225 return
227 def _get_last_index_before_table(self) -> int:
228 last_index = -1
229 for i, (k, v) in enumerate(self._body):
230 if isinstance(v, Null):
231 continue # Null elements are inserted after deletion
233 if isinstance(v, Whitespace) and not v.is_fixed():
234 continue
236 if isinstance(v, (Table, AoT)) and k is not None and not k.is_dotted():
237 break
239 if (
240 isinstance(v, Table)
241 and k is not None
242 and k.is_dotted()
243 and self._renders_table_header(v)
244 ):
245 # A dotted-key super table renders inline (`a.b = 1`) only as
246 # long as none of its children render a `[table]` header; once
247 # one does, anything appended after it would land inside that
248 # table's scope.
249 break
250 last_index = i
251 return last_index + 1
253 def _renders_table_header(self, table: Table) -> bool:
254 for k, v in table.value.body:
255 if isinstance(v, AoT):
256 return True
257 if isinstance(v, Table):
258 if k is not None and k.is_dotted() and v.is_super_table():
259 if self._renders_table_header(v):
260 return True
261 else:
262 return True
263 return False
265 def _validate_out_of_order_table(self, key: Key | None = None) -> None:
266 if key is None:
267 for k in list(self._out_of_order_keys):
268 assert k is not None
269 self._validate_out_of_order_table(k)
270 return
271 if key not in self._map:
272 return
273 current_idx = self._map[key]
274 if not isinstance(current_idx, tuple):
275 return
276 if self._parsed:
277 # while parsing, every fragment appended to an out-of-order key
278 # triggers a validation pass; resume from the cached temp
279 # container so each fragment is merged (and deep-copied) once
280 # instead of on every later pass. Fragments are only ever
281 # appended during parsing, so a count prefix stays valid; any
282 # other mutation clears the cache.
283 validated, temp = self._validation_cache.get(key, (0, None))
284 if validated > len(current_idx):
285 validated, temp = 0, None
286 try:
287 temp = OutOfOrderTableProxy.validate(
288 self, current_idx[validated:], temp
289 )
290 except Exception:
291 # the temp container may be partially mutated; don't let a
292 # caught-and-retried failure resume from a poisoned cache
293 self._validation_cache.pop(key, None)
294 raise
295 self._validation_cache[key] = (len(current_idx), temp)
296 return
297 OutOfOrderTableProxy.validate(self, current_idx)
299 def append(
300 self, key: Key | str | None, item: Any, validate: bool = True
301 ) -> Container:
302 """Similar to :meth:`add` but both key and value must be given."""
303 if not isinstance(key, Key) and key is not None:
304 key = SingleKey(key)
306 if not isinstance(item, Item):
307 item = _item(item)
309 if key is not None and key.is_multi():
310 self._handle_dotted_key(key, item)
311 return self
313 if isinstance(item, (AoT, Table)) and item.name is None:
314 assert isinstance(key, Key)
315 item.name = key.key
317 prev = self._previous_item()
318 prev_ws = isinstance(prev, Whitespace) or ends_with_whitespace(prev)
319 if isinstance(item, Table):
320 if not self._parsed:
321 item.invalidate_display_name()
322 if (
323 self._body
324 and not (self._parsed or item.trivia.indent or prev_ws)
325 and key is not None
326 and not key.is_dotted()
327 ):
328 item.trivia.indent = "\n"
330 if isinstance(item, AoT) and self._body and not self._parsed:
331 item.invalidate_display_name()
332 if item and not ("\n" in item[0].trivia.indent or prev_ws):
333 item[0].trivia.indent = "\n" + item[0].trivia.indent
335 if key is not None and key in self:
336 current_idx = self._map[key]
337 if isinstance(current_idx, tuple):
338 current_body_element = self._body[current_idx[-1]]
339 else:
340 current_body_element = self._body[current_idx]
342 current = current_body_element[1]
344 if isinstance(item, Table):
345 if not isinstance(current, (Table, AoT)):
346 raise KeyAlreadyPresent(key)
348 if item.is_aot_element():
349 # New AoT element found later on
350 # Adding it to the current AoT
351 if not isinstance(current, AoT):
352 current = AoT([current, item], parsed=self._parsed)
354 self._replace(key, key, current)
355 else:
356 current.append(item)
358 return self
359 elif isinstance(current, AoT):
360 if not item.is_aot_element():
361 if item.is_super_table() and len(current.body):
362 # A sub-table header such as `[fruit.apple.texture]`
363 # appearing after the array `[[fruit]]` (possibly with
364 # unrelated tables in between) extends the last element
365 # of the array, per the TOML spec.
366 last = current[-1]
367 for k, v in item.value.body:
368 last.value.append(k, v)
370 return self
371 # Tried to define a table after an AoT with the same name.
372 raise KeyAlreadyPresent(key)
374 current.append(item)
376 return self
377 elif current.is_super_table():
378 if item.is_super_table():
379 # We need to merge both super tables
380 if (
381 key.is_dotted()
382 or (
383 current_body_element[0] is not None
384 and current_body_element[0].is_dotted()
385 )
386 or self._table_keys[-1] != current_body_element[0]
387 ):
388 if key.is_dotted() and not self._parsed:
389 idx = self._get_last_index_before_table()
390 else:
391 idx = len(self._body)
393 if idx < len(self._body):
394 self._insert_at(idx, key, item)
395 else:
396 self._raw_append(key, item)
398 if validate:
399 self._validate_out_of_order_table(key)
401 return self
403 # Merge the new super table's body into the existing one
404 # in place. Previously this deep-copied `current` before
405 # appending, which is O(size of current) on every merge
406 # and therefore O(n^2) when many subtables share a super
407 # table (e.g. consecutive `[a.b.c]` / `[a.b.d]` headers).
408 # Mutating in place is O(1) per merge. The defensive copy
409 # that protected the out-of-order validation pass has been
410 # moved into OutOfOrderTableProxy (its only consumer).
411 for k, v in item.value.body:
412 current.append(k, v)
414 return self
415 elif (
416 current_body_element[0] is not None
417 and current_body_element[0].is_dotted()
418 ):
419 raise TOMLKitError("Redefinition of an existing table")
420 else:
421 # Merging a concrete table into an existing implicit/super
422 # table is only valid if it does not redefine existing
423 # subtrees via dotted keys and does not change prior types.
424 assert isinstance(current, Table)
425 self._validate_table_candidate(current, item)
426 elif not item.is_super_table():
427 raise KeyAlreadyPresent(key)
428 else:
429 # An existing concrete table (current) is being extended by
430 # a super-table (item) — e.g. [a] b=1 then [a.b] c=2 out of
431 # order, or [a] b.c=1 then [a.b] d=2. Validate that the
432 # super-table does not redefine any existing key, raising
433 # early at parse time. When validation passes, fall through
434 # — _raw_append below will create an out-of-order entry and
435 # preserve table ordering in the document.
436 assert isinstance(current, Table)
437 self._validate_table_candidate(current, item)
438 elif isinstance(item, AoT):
439 if not isinstance(current, AoT):
440 # Tried to define an AoT after a table with the same name.
441 raise KeyAlreadyPresent(key)
443 for table in item.body:
444 current.append(table)
446 return self
447 else:
448 raise KeyAlreadyPresent(key)
450 is_table = isinstance(item, (Table, AoT))
451 if (
452 key is not None
453 and self._body
454 and not self._parsed
455 and (not is_table or key.is_dotted())
456 ):
457 # If there is already at least one table in the current container
458 # and the given item is not a table, we need to find the last
459 # item that is not a table and insert after it
460 # If no such item exists, insert at the top of the table
461 last_index = self._get_last_index_before_table()
463 if last_index < len(self._body):
464 after_item = self._body[last_index][1]
465 if not (
466 isinstance(after_item, Whitespace)
467 or "\n" in after_item.trivia.indent
468 ):
469 after_item.trivia.indent = "\n" + after_item.trivia.indent
470 return self._insert_at(last_index, key, item)
471 else:
472 previous_item = self._body[-1][1]
473 if isinstance(previous_item, Table) and previous_item.is_super_table():
474 previous_child = previous_item.value._previous_item()
475 if (
476 previous_child is not None
477 and not isinstance(previous_child, Whitespace)
478 and "\n" in previous_item.trivia.trail
479 and "\n" not in previous_child.trivia.trail
480 ):
481 previous_child.trivia.trail += previous_item.trivia.trail
482 if not (
483 isinstance(previous_item, Whitespace)
484 or ends_with_whitespace(previous_item)
485 or "\n" in previous_item.trivia.trail
486 ):
487 previous_item.trivia.trail += "\n"
489 self._raw_append(key, item)
490 if validate and key is not None:
491 self._validate_out_of_order_table(key)
492 return self
494 def _validate_table_candidate(self, current: Table, candidate: Table) -> None:
495 for k, v in candidate.value.body:
496 if k is None:
497 continue
499 if k in current.value._map:
500 existing = current.value.item(k)
501 if isinstance(existing, (Table, AoT)) != isinstance(v, (Table, AoT)):
502 raise KeyAlreadyPresent(k)
503 if k.is_dotted() and not (
504 isinstance(existing, Table) and existing.is_super_table()
505 ):
506 raise TOMLKitError("Redefinition of an existing table")
507 if isinstance(existing, Table) and isinstance(v, Table):
508 if not existing.is_super_table() and not v.is_super_table():
509 # Both sides are concrete `[table]` definitions of the
510 # same name; the table is declared twice.
511 raise KeyAlreadyPresent(k)
512 # One side is still an implicit/super table, so a duplicate
513 # (if any) is nested deeper - keep checking the subtree.
514 self._validate_table_candidate(existing, v)
515 continue
517 if not k.is_dotted():
518 # Even when the candidate key itself is not dotted, an
519 # existing dotted key may already use it as a prefix —
520 # e.g. [a] b.c=1 then [a.b] d=2 (b prefixes b.c).
521 for existing_key in current.value._map:
522 if existing_key.is_dotted() and next(iter(existing_key)) == k:
523 raise TOMLKitError("Redefinition of an existing table")
524 continue
526 head = next(iter(k))
527 if head in current.value._map:
528 raise TOMLKitError("Redefinition of an existing table")
530 def _raw_append(self, key: Key | None, item: Item) -> None:
531 if key is not None and key in self._map:
532 current_idx = self._map[key]
533 if not isinstance(current_idx, tuple):
534 current_idx = (current_idx,)
536 current = self._body[current_idx[-1]][1]
537 if not isinstance(current, Table):
538 raise KeyAlreadyPresent(key)
540 self._map[key] = (*current_idx, len(self._body))
541 self._out_of_order_keys.add(key)
542 elif key is not None:
543 self._map[key] = len(self._body)
545 self._body.append((key, item))
546 if item.is_table() and key is not None:
547 self._table_keys.append(key)
549 if key is not None:
550 dict.__setitem__(self, key.key, item.value)
552 def _remove_at(self, idx: int) -> None:
553 key = self._body[idx][0]
554 assert key is not None
555 index = self._map.get(key)
556 if index is None:
557 raise NonExistentKey(key)
558 self._validation_cache.clear()
559 self._body[idx] = (None, Null())
561 if isinstance(index, tuple):
562 index_list = list(index)
563 index_list.remove(idx)
564 if len(index_list) == 1:
565 self._map[key] = index_list.pop()
566 else:
567 self._map[key] = tuple(index_list)
568 else:
569 dict.__delitem__(self, key.key)
570 self._map.pop(key)
572 def remove(self, key: Key | str) -> Container:
573 """Remove a key from the container."""
574 if not isinstance(key, Key):
575 key = SingleKey(key)
577 idx = self._map.pop(key, None)
578 if idx is None:
579 raise NonExistentKey(key)
581 self._validation_cache.clear()
582 if isinstance(idx, tuple):
583 for i in idx:
584 self._body[i] = (None, Null())
585 else:
586 self._body[idx] = (None, Null())
588 dict.__delitem__(self, key.key)
590 return self
592 def _insert_after(
593 self, key: Key | str, other_key: Key | str, item: Any
594 ) -> Container:
595 if key is None:
596 raise ValueError("Key cannot be null in insert_after()")
598 if key not in self:
599 raise NonExistentKey(key)
601 if not isinstance(key, Key):
602 key = SingleKey(key)
604 if not isinstance(other_key, Key):
605 other_key = SingleKey(other_key)
607 item = _item(item)
609 idx = self._map[key]
610 # Insert after the max index if there are many.
611 if isinstance(idx, tuple):
612 idx = max(idx)
613 current_item = self._body[idx][1]
614 if "\n" not in current_item.trivia.trail:
615 current_item.trivia.trail += "\n"
617 # Increment indices after the current index
618 for k, v in self._map.items():
619 if isinstance(v, tuple):
620 new_indices = []
621 for v_ in v:
622 if v_ > idx:
623 v_ = v_ + 1
625 new_indices.append(v_)
627 self._map[k] = tuple(new_indices)
628 elif v > idx:
629 self._map[k] = v + 1
631 self._map[other_key] = idx + 1
632 self._body.insert(idx + 1, (other_key, item))
634 if key is not None:
635 dict.__setitem__(self, other_key.key, item.value)
637 return self
639 def _insert_at(self, idx: int, key: Key | str, item: Any) -> Container:
640 if idx > len(self._body) - 1:
641 raise ValueError(f"Unable to insert at position {idx}")
643 if not isinstance(key, Key):
644 key = SingleKey(key)
646 item = _item(item)
648 if idx > 0:
649 previous_item = self._body[idx - 1][1]
650 if not (
651 isinstance(previous_item, Whitespace)
652 or ends_with_whitespace(previous_item)
653 or isinstance(item, (AoT, Table))
654 or "\n" in previous_item.trivia.trail
655 ):
656 previous_item.trivia.trail += "\n"
658 # Increment indices after the current index
659 for k, v in self._map.items():
660 if isinstance(v, tuple):
661 new_indices = []
662 for v_ in v:
663 if v_ >= idx:
664 v_ = v_ + 1
666 new_indices.append(v_)
668 self._map[k] = tuple(new_indices)
669 elif v >= idx:
670 self._map[k] = v + 1
672 if key in self._map:
673 current_idx = self._map[key]
674 if not isinstance(current_idx, tuple):
675 current_idx = (current_idx,)
676 self._map[key] = (*current_idx, idx)
677 self._out_of_order_keys.add(key)
678 else:
679 self._map[key] = idx
680 self._body.insert(idx, (key, item))
682 dict.__setitem__(self, key.key, item.value)
684 return self
686 def item(self, key: Key | str) -> Item | OutOfOrderTableProxy:
687 """Get an item for the given key."""
688 if not isinstance(key, Key):
689 key = SingleKey(key)
691 idx = self._map.get(key)
692 if idx is None:
693 raise NonExistentKey(key)
695 if isinstance(idx, tuple):
696 # The item we are getting is an out of order table
697 # so we need a proxy to retrieve the proper objects
698 # from the parent container
699 return OutOfOrderTableProxy(self, idx)
701 return self._body[idx][1]
703 def last_item(self) -> Item | None:
704 """Get the last item."""
705 if self._body:
706 return self._body[-1][1]
707 return None
709 def as_string(self) -> str:
710 """Render as TOML string."""
711 s = ""
712 for path, aot in _header_less_aots(self._body):
713 s += _render_empty_aot(path, aot)
714 for k, v in self._body:
715 if k is not None:
716 if _is_empty_aot(v):
717 continue
718 if isinstance(v, Table):
719 if (
720 s.strip(" ")
721 and not s.strip(" ").endswith("\n")
722 and "\n" not in v.trivia.indent
723 ):
724 s += "\n"
725 s += self._render_table(k, v)
726 elif isinstance(v, AoT):
727 if (
728 s.strip(" ")
729 and not s.strip(" ").endswith("\n")
730 and "\n" not in v.trivia.indent
731 ):
732 s += "\n"
733 s += self._render_aot(k, v)
734 else:
735 s += self._render_simple_item(k, v)
736 else:
737 s += self._render_simple_item(k, v)
739 return s
741 def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> str:
742 cur = ""
744 if table.display_name is not None:
745 _key = table.display_name
746 else:
747 _key = key.as_string()
749 if prefix is not None:
750 _key = prefix + "." + _key
752 header_emitted = _emits_header(key, table)
753 if header_emitted:
754 open_, close = "[", "]"
755 if table.is_aot_element():
756 open_, close = "[[", "]]"
758 newline_in_table_trivia = (
759 "\n" if "\n" not in table.trivia.trail and len(table.value) > 0 else ""
760 )
761 cur += (
762 f"{table.trivia.indent}"
763 f"{open_}"
764 f"{decode(_key)}"
765 f"{close}"
766 f"{table.trivia.comment_ws}"
767 f"{decode(table.trivia.comment)}"
768 f"{table.trivia.trail}"
769 f"{newline_in_table_trivia}"
770 )
771 elif table.trivia.indent == "\n":
772 cur += table.trivia.indent
774 if header_emitted:
775 # A super table opens no scope, so its fallbacks belong to -- and
776 # were already written by -- the nearest enclosing header instead.
777 for path, aot in _header_less_aots(table.value.body):
778 cur += _render_empty_aot(path, aot)
779 for k, v in table.value.body:
780 if k is not None and _is_empty_aot(v):
781 continue
782 elif isinstance(v, Table):
783 if (
784 cur.strip(" ")
785 and not cur.strip(" ").endswith("\n")
786 and "\n" not in v.trivia.indent
787 ):
788 cur += "\n"
789 assert k is not None
790 if v.is_super_table():
791 if k.is_dotted() and not key.is_dotted():
792 # Dotted key inside table
793 cur += self._render_table(k, v)
794 else:
795 cur += self._render_table(k, v, prefix=_key)
796 else:
797 cur += self._render_table(k, v, prefix=_key)
798 elif isinstance(v, AoT):
799 if (
800 cur.strip(" ")
801 and not cur.strip(" ").endswith("\n")
802 and "\n" not in v.trivia.indent
803 ):
804 cur += "\n"
805 assert k is not None
806 cur += self._render_aot(k, v, prefix=_key)
807 else:
808 cur += self._render_simple_item(
809 k, v, prefix=_key if key.is_dotted() else None
810 )
812 return cur
814 def _render_aot(self, key: Key, aot: AoT, prefix: str | None = None) -> str:
815 _key = key.as_string()
816 if prefix is not None:
817 _key = prefix + "." + _key
819 cur = ""
820 _key = decode(_key)
821 for table in aot.body:
822 cur += self._render_aot_table(table, prefix=_key)
824 return cur
826 def _render_aot_table(self, table: Table, prefix: str | None = None) -> str:
827 cur = ""
828 _key = prefix or ""
829 open_, close = "[[", "]]"
831 cur += (
832 f"{table.trivia.indent}"
833 f"{open_}"
834 f"{decode(_key)}"
835 f"{close}"
836 f"{table.trivia.comment_ws}"
837 f"{decode(table.trivia.comment)}"
838 f"{table.trivia.trail}"
839 )
841 for path, aot in _header_less_aots(table.value.body):
842 cur += _render_empty_aot(path, aot)
843 for k, v in table.value.body:
844 if k is not None and _is_empty_aot(v):
845 continue
846 elif isinstance(v, Table):
847 assert k is not None
848 if v.is_super_table():
849 if k.is_dotted():
850 # Dotted key inside table
851 cur += self._render_table(k, v)
852 else:
853 cur += self._render_table(k, v, prefix=_key)
854 else:
855 cur += self._render_table(k, v, prefix=_key)
856 elif isinstance(v, AoT):
857 assert k is not None
858 cur += self._render_aot(k, v, prefix=_key)
859 else:
860 cur += self._render_simple_item(k, v)
862 return cur
864 def _render_simple_item(
865 self, key: Key | None, item: Item, prefix: str | None = None
866 ) -> str:
867 if key is None:
868 return item.as_string()
870 _key = key.as_string()
871 if prefix is not None:
872 _key = prefix + "." + _key
874 return (
875 f"{item.trivia.indent}"
876 f"{decode(_key)}"
877 f"{key.sep}"
878 f"{decode(item.as_string())}"
879 f"{item.trivia.comment_ws}"
880 f"{decode(item.trivia.comment)}"
881 f"{item.trivia.trail}"
882 )
884 def __len__(self) -> int:
885 return dict.__len__(self)
887 def __iter__(self) -> Iterator[str]:
888 return iter(dict.keys(self))
890 # Dictionary methods
891 def __getitem__(self, key: Key | str) -> Any:
892 item = self.item(key)
893 if isinstance(item, Item) and item.is_boolean():
894 return item.value
896 return item
898 def __contains__(self, key: object) -> bool:
899 # Native membership test. The inherited ``MutableMapping.__contains__``
900 # resolves the value via ``__getitem__``/``item()`` (and builds a
901 # ``NonExistentKey`` on every absent key) only to discard it. Resolve the
902 # key the same way ``item()`` does -- ``str`` becomes a ``SingleKey``
903 # (a non-str/non-``Key`` argument still raises ``TypeError``) -- then
904 # probe ``_map`` directly. For an out-of-order table the proxy is still
905 # built so its validation runs exactly as before.
906 if not isinstance(key, Key):
907 key = SingleKey(key) # type: ignore[arg-type]
908 idx = self._map.get(key)
909 if idx is None:
910 return False
911 if isinstance(idx, tuple):
912 # during parsing every fragment append re-probes membership;
913 # if the validation cache covers exactly these fragments they
914 # are already known to merge cleanly, so skip rebuilding the
915 # proxy (which is linear in the number of fragments)
916 validated, _ = self._validation_cache.get(key, (0, None))
917 if not (self._parsed and validated == len(idx)):
918 OutOfOrderTableProxy(self, idx)
919 return True
921 def __setitem__(self, key: Key | str, value: Any) -> None:
922 if key in self:
923 old_key = next(filter(lambda k: k == key, self._map))
924 self._replace(old_key, key, value)
925 else:
926 self.append(key, value)
928 def __delitem__(self, key: Key | str) -> None:
929 self.remove(key)
931 def setdefault(self, key: Key | str, default: Any = None) -> Any:
932 if key not in self:
933 self[key] = default
934 return self[key]
936 def _replace(self, key: Key | str, new_key: Key | str, value: Item) -> None:
937 if not isinstance(key, Key):
938 key = SingleKey(key)
940 idx = self._map.get(key)
941 if idx is None:
942 raise NonExistentKey(key)
944 self._replace_at(idx, new_key, value)
946 def _replace_at(
947 self, idx: int | tuple[int, ...], new_key: Key | str, value: Item
948 ) -> None:
949 value = _item(value)
950 self._validation_cache.clear()
952 if isinstance(idx, tuple):
953 for i in idx[1:]:
954 self._body[i] = (None, Null())
956 idx = idx[0]
958 k, v = self._body[idx]
959 assert k is not None
960 # A dotted key renders its value inline (e.g. ``a.b = 1``), which is only
961 # consistent with a super table. When the replacement value renders with
962 # its own ``[header]`` instead (a non-super table, or an array of tables),
963 # keeping the dotted key duplicates the prefix onto the header (#524) and
964 # lets the header swallow following dotted siblings (#542). Drop the dotted
965 # key so the replacement renders as a plain table/array of tables.
966 dotted_to_header = k.is_dotted() and (
967 isinstance(value, AoT)
968 or (isinstance(value, Table) and not value.is_super_table())
969 )
970 # That new header also captures every sibling that renders inline -- plain
971 # values and dotted keys -- if any still follow it (#513), so it must be
972 # moved past them, exactly as a value-to-table change already is.
973 reposition_dotted = dotted_to_header and any(
974 not isinstance(cur_val, (Null, Whitespace))
975 and not (
976 isinstance(cur_val, (Table, AoT))
977 and (cur_key is None or not cur_key.is_dotted())
978 )
979 for cur_key, cur_val in self._body[idx + 1 :]
980 )
981 if not isinstance(new_key, Key):
982 if (
983 isinstance(value, (AoT, Table)) != isinstance(v, (AoT, Table))
984 or new_key != k.key
985 or dotted_to_header
986 ):
987 new_key = SingleKey(new_key)
988 else: # Inherit the sep of the old key
989 new_key = k
991 del self._map[k]
992 self._map[new_key] = idx
993 if new_key != k:
994 dict.__delitem__(self, k.key)
996 if (
997 isinstance(value, (AoT, Table)) != isinstance(v, (AoT, Table))
998 or reposition_dotted
999 ):
1000 self.remove(k)
1001 if isinstance(value, (AoT, Table)):
1002 # New tables must appear after all entries that render inline:
1003 # plain values and dotted keys (which are super tables). Skip
1004 # those and insert before the first real ``[header]`` so the new
1005 # table cannot swallow a following sibling on round-trip.
1006 for i in range(idx, len(self._body)):
1007 cur_key, cur_val = self._body[i]
1008 if isinstance(cur_val, (AoT, Table)) and not (
1009 cur_key is not None and cur_key.is_dotted()
1010 ):
1011 self._insert_at(i, new_key, value)
1012 idx = i
1013 break
1014 else:
1015 idx = -1
1016 self.append(new_key, value)
1017 else:
1018 # the replaced table's slot lies in the table region, where a
1019 # plain value would be captured by the preceding table on
1020 # round-trip; append() puts it with the other root-level values
1021 idx = -1
1022 self.append(new_key, value)
1023 else:
1024 # Copying trivia
1025 if not isinstance(value, (Whitespace, AoT)):
1026 value.trivia.indent = v.trivia.indent
1027 value.trivia.comment_ws = value.trivia.comment_ws or v.trivia.comment_ws
1028 value.trivia.comment = value.trivia.comment or v.trivia.comment
1029 value.trivia.trail = v.trivia.trail
1030 self._body[idx] = (new_key, value)
1032 if value is not v and hasattr(value, "invalidate_display_name"):
1033 value.invalidate_display_name()
1035 if isinstance(value, Table):
1036 # Insert a cosmetic new line for tables if:
1037 # - it does not have it yet OR is not followed by one
1038 # - it is not the last item, or
1039 # - The table being replaced has a newline
1040 result = self._previous_item_with_index()
1041 assert result is not None
1042 last, _ = result
1043 idx = last if idx < 0 else idx
1044 has_ws = ends_with_whitespace(value)
1045 replace_has_ws = (
1046 isinstance(v, Table)
1047 and v.value.body
1048 and isinstance(v.value.body[-1][1], Whitespace)
1049 )
1050 next_ws = idx < last and isinstance(self._body[idx + 1][1], Whitespace)
1051 if (idx < last or replace_has_ws) and not (next_ws or has_ws):
1052 value.append(None, Whitespace("\n"))
1054 assert isinstance(new_key, Key)
1055 dict.__setitem__(self, new_key.key, value.value)
1057 def __str__(self) -> str:
1058 return str(self.value)
1060 def __repr__(self) -> str:
1061 return repr(self.value)
1063 def __eq__(self, other: object) -> bool:
1064 if not isinstance(other, dict):
1065 return NotImplemented
1067 return bool(_equal_with_nan(self.value, other))
1069 def _getstate(self, protocol: int) -> tuple[bool]:
1070 return (self._parsed,)
1072 def __reduce__(self) -> tuple[type, tuple[bool], tuple[Any, ...]]:
1073 return self.__reduce_ex__(2)
1075 def __reduce_ex__(self, protocol: int) -> tuple[type, tuple[bool], tuple[Any, ...]]: # type: ignore[override]
1076 return (
1077 self.__class__,
1078 self._getstate(protocol),
1079 (self._map, self._body, self._parsed, self._table_keys),
1080 )
1082 def __setstate__(self, state: tuple[Any, ...]) -> None:
1083 self._map = state[0]
1084 self._body = state[1]
1085 self._parsed = state[2]
1086 self._table_keys = state[3]
1087 self._out_of_order_keys = {
1088 k for k, v in self._map.items() if isinstance(v, tuple)
1089 }
1091 for key, item in self._body:
1092 if key is not None:
1093 dict.__setitem__(self, key.key, item.value)
1095 def copy(self) -> Self:
1096 return copy.copy(self)
1098 def __copy__(self) -> Self:
1099 c = self.__class__(self._parsed)
1100 for k, v in dict.items(self):
1101 dict.__setitem__(c, k, v)
1103 c._body += self.body
1104 c._map.update(self._map)
1105 c._out_of_order_keys |= self._out_of_order_keys
1107 return c
1109 def _previous_item_with_index(
1110 self, idx: int | None = None, ignore: tuple[type, ...] = (Null,)
1111 ) -> tuple[int, Item] | None:
1112 """Find the immediate previous item before index ``idx``"""
1113 if idx is None or idx > len(self._body):
1114 idx = len(self._body)
1115 for i in range(idx - 1, -1, -1):
1116 v = self._body[i][-1]
1117 if not isinstance(v, ignore):
1118 return i, v
1119 return None
1121 def _previous_item(
1122 self, idx: int | None = None, ignore: tuple[type, ...] = (Null,)
1123 ) -> Item | None:
1124 """Find the immediate previous item before index ``idx``.
1125 If ``idx`` is not given, the last item is returned.
1126 """
1127 prev = self._previous_item_with_index(idx, ignore)
1128 return prev[-1] if prev else None
1131class OutOfOrderTableProxy(_CustomDict): # type: ignore[type-arg]
1132 @staticmethod
1133 def validate(
1134 container: Container,
1135 indices: tuple[int, ...],
1136 temp_container: Container | None = None,
1137 ) -> Container:
1138 """Validate out of order tables in the given container"""
1139 # Append all items to a temp container to see if there is any error.
1140 # We deep-copy each value before appending: appending a super table
1141 # merges it in place into a matching one already in temp_container, and
1142 # those values are the live subtables of `container`. Without the copy
1143 # the merge would mutate the caller's tables (and corrupt a later
1144 # validation pass). The container merge itself is now copy-free for
1145 # speed, so this is where the isolation lives.
1146 # Passing a previous pass's temp container back in (with the already
1147 # validated indices stripped from `indices`) resumes the validation
1148 # incrementally.
1149 if temp_container is None:
1150 temp_container = Container(True)
1151 for i in indices:
1152 _, item = container._body[i]
1154 if isinstance(item, Table):
1155 for k, v in item.value.body:
1156 temp_container.append(k, copy.deepcopy(v), validate=True)
1158 temp_container._validate_out_of_order_table()
1159 return temp_container
1161 def __init__(self, container: Container, indices: tuple[int, ...]) -> None:
1162 self._container = container
1163 self._internal_container = Container(True)
1164 self._tables: list[Table] = []
1165 self._tables_map: dict[Key, list[int]] = {}
1167 for i in indices:
1168 _, _item = self._container._body[i]
1170 if isinstance(_item, Table):
1171 self._tables.append(_item)
1172 table_idx = len(self._tables) - 1
1173 for k, v in _item.value.body:
1174 merged = self._merge_aot_fragment(k, v)
1175 if merged is None:
1176 self._internal_container._raw_append(k, v)
1177 else:
1178 v = merged
1179 key_indices = self._tables_map.setdefault(k, []) # type: ignore[arg-type]
1180 if table_idx not in key_indices:
1181 key_indices.append(table_idx)
1182 if k is not None:
1183 dict.__setitem__(self, k.key, v)
1185 self._internal_container._validate_out_of_order_table()
1187 def _merge_aot_fragment(self, key: Key | None, item: Item) -> AoT | None:
1188 """
1189 Merge an array-of-tables fragment from a later out-of-order table part.
1191 An AoT whose elements are split across out-of-order parts of the same
1192 table arrives here once per part; ``_raw_append`` only knows how to
1193 chain duplicate ``Table`` parts and would raise ``KeyAlreadyPresent``.
1194 The fragments are presented as a new merged ``AoT`` referencing the
1195 live element tables, without mutating either fragment (the parts keep
1196 rendering their own elements).
1198 Returns the merged ``AoT``, or ``None`` if this is not such a fragment.
1199 """
1200 internal = self._internal_container
1201 if key is None or not isinstance(item, AoT) or key not in internal._map:
1202 return None
1203 idx = internal._map[key]
1204 if isinstance(idx, tuple):
1205 # A tuple index means the key already resolved to several body
1206 # positions, which here only happens for a degenerate collision
1207 # (e.g. a non-AoT part sharing the key). Three or more genuine AoT
1208 # fragments do not reach this branch: each later fragment merges
1209 # into the single growing AoT below, so ``idx`` stays an int.
1210 return None
1211 existing = internal._body[idx][1]
1212 if not isinstance(existing, AoT):
1213 return None
1215 merged = AoT([*existing.body, *item.body], parsed=True)
1216 internal._body[idx] = (internal._body[idx][0], merged)
1217 dict.__setitem__(internal, key.key, merged.value)
1218 return merged
1220 def unwrap(self) -> dict[str, Any]:
1221 return self._internal_container.unwrap()
1223 @property
1224 def value(self) -> dict[str, Any]:
1225 return self._internal_container.value
1227 def __str__(self) -> str:
1228 return str(self.value)
1230 def __repr__(self) -> str:
1231 return repr(self.value)
1233 def __contains__(self, key: object) -> bool:
1234 # Native membership test. The inherited ``MutableMapping.__contains__``
1235 # resolves the value via ``__getitem__`` (and builds a ``NonExistentKey``
1236 # on every absent key) only to discard it. Probe the internal container
1237 # directly -- the same predicate ``__getitem__`` already uses -- which is
1238 # itself a native ``_map`` lookup that still rebuilds the proxy for an
1239 # out-of-order key so its validation runs exactly as before.
1240 return key in self._internal_container
1242 def __getitem__(self, key: Key | str) -> Any:
1243 if key not in self._internal_container:
1244 raise NonExistentKey(key)
1246 return self._internal_container[key]
1248 def __setitem__(self, key: Key | str, value: Any) -> None:
1249 from .items import item as _item_fn
1251 def _is_table_or_aot(it: Any) -> bool:
1252 return isinstance(_item_fn(it), (Table, AoT))
1254 _key: Key = key if isinstance(key, Key) else SingleKey(key)
1256 if _key in self._tables_map:
1257 # Overwrite the first table and remove others
1258 map_indices = self._tables_map[_key]
1259 while len(map_indices) > 1:
1260 table = self._tables[map_indices.pop()]
1261 self._remove_table(table)
1262 old_value = self._tables[map_indices[0]][key]
1263 if _is_table_or_aot(old_value) and not _is_table_or_aot(value):
1264 # Remove the entry from the map and set value again.
1265 del self._tables[map_indices[0]][key]
1266 del self._tables_map[_key]
1267 self[key] = value
1268 return
1269 self._tables[map_indices[0]][key] = value
1270 elif self._tables:
1271 if not _is_table_or_aot(value): # if the value is a plain value
1272 for table in self._tables:
1273 # find the first table that allows plain values
1274 if any(not _is_table_or_aot(v) for _, v in table.items()):
1275 table[key] = value
1276 break
1277 else:
1278 # No part holds a plain value yet, so the chosen part must
1279 # start rendering its own ``[key]`` header. Prefer a part
1280 # that is not a super table (it already renders that header):
1281 # turning a header-less super part concrete here would emit a
1282 # second, duplicate header next to an existing concrete part
1283 # and produce TOML that no longer parses.
1284 for table in self._tables:
1285 if not table.is_super_table():
1286 table[key] = value
1287 break
1288 else:
1289 self._tables[0][key] = value
1290 else:
1291 self._tables[0][key] = value
1292 else:
1293 self._container[key] = value
1295 self._internal_container[key] = value
1296 if key is not None:
1297 dict.__setitem__(self, key, value)
1299 def _remove_table(self, table: Table) -> None:
1300 """Remove table from the parent container"""
1301 self._tables.remove(table)
1302 for idx, body_item in enumerate(self._container._body):
1303 if body_item[1] is table:
1304 self._container._remove_at(idx)
1305 break
1307 def __delitem__(self, key: Key | str) -> None:
1308 _key: Key = key if isinstance(key, Key) else SingleKey(key)
1309 if _key not in self._tables_map:
1310 raise NonExistentKey(key)
1312 for i in reversed(self._tables_map[_key]):
1313 table = self._tables[i]
1314 del table[key]
1315 if not table and len(self._tables) > 1:
1316 self._remove_table(table)
1318 del self._tables_map[_key]
1319 del self._internal_container[key]
1320 if key is not None:
1321 dict.__delitem__(self, key)
1323 def __iter__(self) -> Iterator[str]:
1324 return iter(dict.keys(self))
1326 def __len__(self) -> int:
1327 return dict.__len__(self)
1329 def setdefault(self, key: Key | str, default: Any = None) -> Any:
1330 if key not in self:
1331 self[key] = default
1332 return self[key]
1335def ends_with_whitespace(it: Any) -> bool:
1336 """Returns ``True`` if the given item ``it`` is a ``Table`` or ``AoT`` object
1337 ending with a ``Whitespace``.
1338 """
1339 if isinstance(it, Whitespace):
1340 return True
1341 if isinstance(it, Table):
1342 previous = it.value._previous_item()
1343 return previous is not None and ends_with_whitespace(previous)
1344 return isinstance(it, AoT) and len(it) > 0 and ends_with_whitespace(it[-1])
1347def _equal_with_nan(left: Any, right: Any) -> bool:
1348 if isinstance(left, dict) and isinstance(right, dict):
1349 if left.keys() != right.keys():
1350 return False
1351 return all(_equal_with_nan(left[k], right[k]) for k in left)
1353 if isinstance(left, list) and isinstance(right, list):
1354 if len(left) != len(right):
1355 return False
1356 return all(_equal_with_nan(l, r) for l, r in zip(left, right)) # noqa: E741
1358 if isinstance(left, float) and isinstance(right, float):
1359 if math.isnan(left) and math.isnan(right):
1360 return True
1362 return bool(left == right)