Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tomlkit/parser.py: 98%

1from __future__ import annotations

3import datetime

4import re

5import string

7from typing import Any

8from typing import Callable

10from tomlkit._compat import decode

11from tomlkit._utils import RFC_3339_LOOSE

12from tomlkit._utils import _escaped

13from tomlkit._utils import parse_rfc3339

14from tomlkit.container import Container

15from tomlkit.exceptions import EmptyKeyError

16from tomlkit.exceptions import EmptyTableNameError

17from tomlkit.exceptions import InternalParserError

18from tomlkit.exceptions import InvalidCharInStringError

19from tomlkit.exceptions import InvalidControlChar

20from tomlkit.exceptions import InvalidDateError

21from tomlkit.exceptions import InvalidDateTimeError

22from tomlkit.exceptions import InvalidNumberError

23from tomlkit.exceptions import InvalidTimeError

24from tomlkit.exceptions import InvalidUnicodeValueError

25from tomlkit.exceptions import ParseError

26from tomlkit.exceptions import UnexpectedCharError

27from tomlkit.exceptions import UnexpectedEofError

28from tomlkit.items import AoT

29from tomlkit.items import Array

30from tomlkit.items import Bool

31from tomlkit.items import BoolType

32from tomlkit.items import Comment

33from tomlkit.items import Date

34from tomlkit.items import DateTime

35from tomlkit.items import Float

36from tomlkit.items import InlineTable

37from tomlkit.items import Integer

38from tomlkit.items import Item

39from tomlkit.items import Key

40from tomlkit.items import KeyType

41from tomlkit.items import Null

42from tomlkit.items import SingleKey

43from tomlkit.items import String

44from tomlkit.items import StringType

45from tomlkit.items import Table

46from tomlkit.items import Time

47from tomlkit.items import Trivia

48from tomlkit.items import Whitespace

49from tomlkit.source import Source

50from tomlkit.source import _StateHandler

51from tomlkit.toml_document import TOMLDocument

54CTRL_I = 0x09 # Tab

55CTRL_J = 0x0A # Line feed

56CTRL_M = 0x0D # Carriage return

57CTRL_CHAR_LIMIT = 0x1F

58CHR_DEL = 0x7F

60# TOML character classes (formerly the `TOMLChar` constants), as frozensets for

61# O(1) membership tests; also the stop-sets for the Source.advance_while /

62# advance_until bulk run scans that replace per-character

63# `while self._current in <set> and self.inc()` loops with a single scan.

64_SPACES = frozenset(" \t")

65_NL = frozenset("\n\r")

66_WS = _SPACES | _NL

67_KV = frozenset("= \t")

68_BARE_KEY_OR_SPACE = frozenset(string.ascii_letters + string.digits + "-_ \t")

69_NUM_STOP = frozenset(" \t\n\r#,]}")

70_DATE_TAIL_STOP = frozenset("\t\n\r#,]}")

71# Control chars invalid inside a single-line string (DEL + everything <= 0x1F

72# except tab) — exactly the set that raises InvalidControlChar in the per-char

73# string loop. The single-line string-body fast-path stops its bulk scan at the

74# first delimiter / backslash / control char, then the main loop handles that

75# char with its existing branch (raising InvalidControlChar where needed).

76_CTRL_SINGLE = frozenset(chr(c) for c in range(0x20) if c != CTRL_I) | {chr(CHR_DEL)}

77_SINGLE_LITERAL_STOP = _CTRL_SINGLE | {"'"} # literal: only the closing quote

78_SINGLE_BASIC_STOP = _CTRL_SINGLE | {'"', "\\"} # basic: quote or escape

80# Same idea for multiline string bodies. A multiline string may contain raw tab,

81# line feed and carriage return, so those are NOT in the control-reject set; but

82# the bulk scan must still stop at CR (the per-char loop validates the \r\n pair

83# and rejects a lone \r) and at the control chars that DO raise. LF and tab are

84# left out of the stop-set entirely, so a multiline body is scanned in one slice

85# across newlines up to the next delimiter / backslash / CR / invalid control.

86_CTRL_MULTI = frozenset(

87 chr(c) for c in range(0x20) if c not in (CTRL_I, CTRL_J, CTRL_M)

88) | {chr(CHR_DEL)}

89_MULTI_LITERAL_STOP = _CTRL_MULTI | {"'", "\r"} # literal: closing quote or CR

90_MULTI_BASIC_STOP = _CTRL_MULTI | {'"', "\\", "\r"} # basic: quote, escape or CR

93class Parser:

94 """

95 Parser for TOML documents.

96 """

98 # Deeply nested documents would overflow the interpreter stack: arrays and

99 # inline tables are parsed recursively, and every fragment of a dotted key

100 # adds a level of nested containers. Refuse documents beyond this depth.

101 MAX_NESTING_DEPTH = 100

102

103 def __init__(self, string: str | bytes) -> None:

104 # Input to parse

105 self._src = Source(decode(string))

106

107 self._aot_stack: list[Key] = []

108 self._nesting_depth = 0

109

110 @property

111 def _state(self) -> _StateHandler:

112 return self._src.state

113

114 @property

115 def _idx(self) -> int:

116 return self._src.idx

117

118 @property

119 def _current(self) -> str:

120 return self._src.current

121

122 @property

123 def _marker(self) -> int:

124 return self._src.marker

125

126 def extract(self) -> str:

127 """

128 Extracts the value between marker and index

129 """

130 return self._src.extract()

131

132 def inc(self, exception: type[ParseError] | None = None) -> bool:

133 """

134 Increments the parser if the end of the input has not been reached.

135 Returns whether or not it was able to advance.

136 """

137 return self._src.inc(exception=exception)

138

139 def inc_n(self, n: int, exception: type[ParseError] | None = None) -> bool:

140 """

141 Increments the parser by n characters

142 if the end of the input has not been reached.

143 """

144 return self._src.inc_n(n=n, exception=exception)

145

146 def consume(self, chars: str, min: int = 0, max: int = -1) -> None:

147 """

148 Consume chars until min/max is satisfied is valid.

149 """

150 return self._src.consume(chars=chars, min=min, max=max)

151

152 def end(self) -> bool:

153 """

154 Returns True if the parser has reached the end of the input.

155 """

156 return self._src.end()

157

158 def mark(self) -> None:

159 """

160 Sets the marker to the index's current position

161 """

162 self._src.mark()

163

164 def parse_error(

165 self,

166 exception: type[ParseError] = ParseError,

167 *args: Any,

168 **kwargs: Any,

169 ) -> ParseError:

170 """

171 Creates a generic "parse error" at the current position.

172 """

173 return self._src.parse_error(exception, *args, **kwargs)

174

175 def parse(self) -> TOMLDocument:

176 body = TOMLDocument(True)

177

178 # Take all keyvals outside of tables/AoT's.

179 while not self.end():

180 # Break out if a table is found

181 if self._current == "[":

182 break

183

184 # Otherwise, take and append one KV

185 item = self._parse_item()

186 if not item:

187 break

188

189 key, value = item

190 if (key is not None and key.is_multi()) or not self._merge_ws(value, body):

191 # We actually have a table

192 try:

193 body.append(key, value)

194 except Exception as e:

195 raise self.parse_error(ParseError, str(e)) from e

196

197 self.mark()

198

199 while not self.end():

200 key, value = self._parse_table()

201 if isinstance(value, Table) and value.is_aot_element():

202 # This is just the first table in an AoT. Parse the rest of the array

203 # along with it.

204 value = self._parse_aot(value, key)

205

206 try:

207 body.append(key, value)

208 except Exception as e:

209 raise self.parse_error(ParseError, str(e)) from e

210

211 body.parsing(False)

212

213 return body

214

215 def _merge_ws(self, item: Item, container: Container) -> bool:

216 """

217 Merges the given Item with the last one currently in the given Container if

218 both are whitespace items.

219

220 Returns True if the items were merged.

221 """

222 last = container.last_item()

223 if not last:

224 return False

225

226 if not isinstance(item, Whitespace) or not isinstance(last, Whitespace):

227 return False

228

229 start = self._idx - (len(last.s) + len(item.s))

230 container.body[-1] = (

231 container.body[-1][0],

232 Whitespace(self._src[start : self._idx]),

233 )

234

235 return True

236

237 def _is_child(self, parent: Key, child: Key) -> bool:

238 """

239 Returns whether a key is strictly a child of another key.

240 AoT siblings are not considered children of one another.

241 """

242 parent_parts = tuple(parent)

243 child_parts = tuple(child)

244

245 if parent_parts == child_parts:

246 return False

247

248 return parent_parts == child_parts[: len(parent_parts)]

249

250 def _parse_item(self) -> tuple[Key | None, Item] | None:

251 """

252 Attempts to parse the next item and returns it, along with its key

253 if the item is value-like.

254 """

255 self.mark()

256 with self._state as state:

257 while True:

258 c = self._current

259 if c == "\n":

260 # Found a newline; Return all whitespace found up to this point.

261 self.inc()

262

263 return None, Whitespace(self.extract())

264 elif c in " \t\r":

265 if c == "\r":

266 with self._state(restore=True):

267 if not self.inc() or self._current != "\n":

268 raise self.parse_error(

269 InvalidControlChar, CTRL_M, "documents"

270 )

271 # Skip whitespace.

272 if not self.inc():

273 return None, Whitespace(self.extract())

274 elif c == "#":

275 # Found a comment, parse it

276 indent = self.extract()

277 cws, comment, trail = self._parse_comment_trail()

278

279 return None, Comment(Trivia(indent, cws, comment, trail))

280 elif c == "[":

281 # Found a table, delegate to the calling function.

282 return None

283 else:

284 # Beginning of a KV pair.

285 # Return to beginning of whitespace so it gets included

286 # as indentation for the KV about to be parsed.

287 state.restore = True

288 break

289

290 return self._parse_key_value(True)

291

292 def _parse_comment_trail(self, parse_trail: bool = True) -> tuple[str, str, str]:

293 """

294 Returns (comment_ws, comment, trail)

295 If there is no comment, comment_ws and comment will

296 simply be empty.

297 """

298 if self.end():

299 return "", "", ""

300

301 comment = ""

302 comment_ws = ""

303 self.mark()

304

305 while True:

306 c = self._current

307

308 if c == "\n":

309 break

310 elif c == "#":

311 comment_ws = self.extract()

312

313 self.mark()

314 self.inc() # Skip #

315

316 # The comment itself

317 while not self.end() and self._current not in _NL:

318 code = ord(self._current)

319 if code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I):

320 raise self.parse_error(InvalidControlChar, code, "comments")

321

322 if not self.inc():

323 break

324

325 comment = self.extract()

326 self.mark()

327

328 break

329 elif c in " \t\r":

330 if c == "\r":

331 with self._state(restore=True):

332 if not self.inc() or self._current != "\n":

333 raise self.parse_error(

334 InvalidControlChar, CTRL_M, "comments"

335 )

336 self.inc()

337 else:

338 raise self.parse_error(UnexpectedCharError, c)

339

340 if self.end():

341 break

342

343 trail = ""

344 if parse_trail:

345 self._src.advance_while(_SPACES)

346

347 if self._current == "\r":

348 with self._state(restore=True):

349 if not self.inc() or self._current != "\n":

350 raise self.parse_error(InvalidControlChar, CTRL_M, "documents")

351 self.inc()

352

353 if self._current == "\n":

354 self.inc()

355

356 if self._idx != self._marker or self._current in _WS:

357 trail = self.extract()

358

359 return comment_ws, comment, trail

360

361 def _parse_key_value(self, parse_comment: bool = False) -> tuple[Key, Item]:

362 # Leading indent

363 self.mark()

364

365 self._src.advance_while(_SPACES)

366

367 indent = self.extract()

368

369 # Key

370 key = self._parse_key()

371

372 self.mark()

373

374 found_equals = self._current == "="

375 while self._current in _KV and self.inc():

376 if self._current == "=":

377 if found_equals:

378 raise self.parse_error(UnexpectedCharError, "=")

379 else:

380 found_equals = True

381 if not found_equals:

382 raise self.parse_error(UnexpectedCharError, self._current)

383

384 if not key.sep:

385 key.sep = self.extract()

386 else:

387 key.sep += self.extract()

388

389 # Value

390 val = self._parse_value()

391 # Comment

392 if parse_comment:

393 cws, comment, trail = self._parse_comment_trail()

394 meta = val.trivia

395 if not meta.comment_ws:

396 meta.comment_ws = cws

397

398 meta.comment = comment

399 meta.trail = trail

400 else:

401 val.trivia.trail = ""

402

403 val.trivia.indent = indent

404

405 return key, val

406

407 def _parse_key(self) -> Key:

408 """

409 Parses a Key at the current position;

410 WS before the key must be exhausted first at the callsite.

411 """

412 key = self._parse_simple_key()

413 fragments = 1

414 while self._current == ".":

415 fragments += 1

416 if fragments > self.MAX_NESTING_DEPTH:

417 raise self.parse_error(

418 ParseError,

419 f"TOML key nested more than {self.MAX_NESTING_DEPTH} levels deep",

420 )

421 self.inc()

422 key = key.concat(self._parse_simple_key())

423

424 return key

425

426 def _parse_simple_key(self) -> Key:

427 """

428 Parses a single (non-dotted) key fragment.

429 """

430 self.mark()

431 # Skip any leading whitespace (bulk scan)

432 self._src.advance_while(_SPACES)

433 if self._current in "\"'":

434 return self._parse_quoted_key()

435 else:

436 return self._parse_bare_key()

437

438 def _parse_quoted_key(self) -> Key:

439 """

440 Parses a key enclosed in either single or double quotes.

441 """

442 # Extract the leading whitespace

443 original = self.extract()

444 quote_style = self._current

445 key_type = next((t for t in KeyType if t.value == quote_style), None)

446

447 if key_type is None:

448 raise RuntimeError("Should not have entered _parse_quoted_key()")

449

450 key_str = self._parse_string(

451 StringType.SLB if key_type == KeyType.Basic else StringType.SLL

452 )

453 if key_str._t.is_multiline():

454 raise self.parse_error(UnexpectedCharError, key_str._t.value)

455 original += key_str.as_string()

456 self.mark()

457 self._src.advance_while(_SPACES)

458 original += self.extract()

459

460 return SingleKey(str(key_str), t=key_type, sep="", original=original)

461

462 def _parse_bare_key(self) -> Key:

463 """

464 Parses a bare key.

465 """

466 self._src.advance_while(_BARE_KEY_OR_SPACE)

467

468 original = self.extract()

469 key_s = original.strip()

470 if not key_s:

471 # Empty key

472 raise self.parse_error(EmptyKeyError)

473

474 if " " in key_s or "\t" in key_s:

475 # Bare key with whitespace in it

476 raise self.parse_error(ParseError, f'Invalid key "{key_s}"')

477

478 return SingleKey(key_s, KeyType.Bare, "", original)

479

480 def _parse_value(self) -> Item:

481 """

482 Attempts to parse a value at the current position.

483 """

484 self.mark()

485 c = self._current

486 trivia = Trivia()

487

488 if c == StringType.SLB.value:

489 return self._parse_basic_string()

490 elif c == StringType.SLL.value:

491 return self._parse_literal_string()

492 elif c == BoolType.TRUE.value[0]:

493 return self._parse_true()

494 elif c == BoolType.FALSE.value[0]:

495 return self._parse_false()

496 elif c == "[":

497 return self._parse_nested(self._parse_array)

498 elif c == "{":

499 return self._parse_nested(self._parse_inline_table)

500 elif c in "+-" or self._peek(4) in {

501 "+inf",

502 "-inf",

503 "inf",

504 "+nan",

505 "-nan",

506 "nan",

507 }:

508 # Number

509 self._src.advance_until(_NUM_STOP)

510

511 raw = self.extract()

512

513 item = self._parse_number(raw, trivia)

514 if item is not None:

515 return item

516

517 raise self.parse_error(InvalidNumberError)

518 elif c in string.digits:

519 # Integer, Float, Date, Time or DateTime

520 self._src.advance_until(_NUM_STOP)

521

522 raw = self.extract()

523

524 m = RFC_3339_LOOSE.match(raw)

525 if m:

526 if m.group("date") and m.group("time"):

527 # datetime

528 try:

529 dt = parse_rfc3339(raw)

530 assert isinstance(dt, datetime.datetime)

531 return DateTime(

532 dt.year,

533 dt.month,

534 dt.day,

535 dt.hour,

536 dt.minute,

537 dt.second,

538 dt.microsecond,

539 dt.tzinfo,

540 trivia,

541 raw,

542 )

543 except ValueError:

544 raise self.parse_error(InvalidDateTimeError) from None

545

546 if m.group("date"):

547 try:

548 dt = parse_rfc3339(raw)

549 assert isinstance(dt, datetime.date)

550 date = Date(dt.year, dt.month, dt.day, trivia, raw)

551 self.mark()

552 self._src.advance_until(_DATE_TAIL_STOP)

553

554 time_raw = self.extract()

555 time_part = time_raw.rstrip()

556 trivia.comment_ws = time_raw[len(time_part) :]

557 if not time_part:

558 return date

559

560 dt = parse_rfc3339(raw + time_part)

561 assert isinstance(dt, datetime.datetime)

562 return DateTime(

563 dt.year,

564 dt.month,

565 dt.day,

566 dt.hour,

567 dt.minute,

568 dt.second,

569 dt.microsecond,

570 dt.tzinfo,

571 trivia,

572 raw + time_part,

573 )

574 except ValueError:

575 raise self.parse_error(InvalidDateError) from None

576

577 if m.group("time"):

578 try:

579 t = parse_rfc3339(raw)

580 assert isinstance(t, datetime.time)

581 return Time(

582 t.hour,

583 t.minute,

584 t.second,

585 t.microsecond,

586 t.tzinfo,

587 trivia,

588 raw,

589 )

590 except ValueError:

591 raise self.parse_error(InvalidTimeError) from None

592

593 item = self._parse_number(raw, trivia)

594 if item is not None:

595 return item

596

597 raise self.parse_error(InvalidNumberError)

598 else:

599 raise self.parse_error(UnexpectedCharError, c)

600

601 def _parse_true(self) -> Bool:

602 return self._parse_bool(BoolType.TRUE)

603

604 def _parse_false(self) -> Bool:

605 return self._parse_bool(BoolType.FALSE)

606

607 def _parse_bool(self, style: BoolType) -> Bool:

608 with self._state:

609 style = BoolType(style)

610

611 # only keep parsing for bool if the characters match the style

612 # try consuming rest of chars in style

613 for c in style:

614 self.consume(c, min=1, max=1)

615

616 return Bool(style, Trivia())

617

618 def _parse_nested(self, parse: Callable[[], Item]) -> Item:

619 """

620 Parses an array or inline table, enforcing the nesting depth limit.

621 """

622 self._nesting_depth += 1

623 if self._nesting_depth > self.MAX_NESTING_DEPTH:

624 raise self.parse_error(

625 ParseError,

626 f"TOML value nested more than {self.MAX_NESTING_DEPTH} levels deep",

627 )

628 try:

629 return parse()

630 finally:

631 self._nesting_depth -= 1

632

633 def _parse_array(self) -> Array:

634 # Consume opening bracket, EOF here is an issue (middle of array)

635 self.inc(exception=UnexpectedEofError)

636

637 elems: list[Item] = []

638 prev_value = None

639 while True:

640 # consume whitespace

641 mark = self._idx

642 self.consume(" \t\n\r")

643 indent = self._src[mark : self._idx]

644 newline = _NL & set(indent)

645 if newline:

646 elems.append(Whitespace(indent))

647 continue

648

649 # consume comment

650 if self._current == "#":

651 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

652 elems.append(Comment(Trivia(indent, cws, comment, trail)))

653 continue

654

655 # consume indent

656 if indent:

657 elems.append(Whitespace(indent))

658 continue

659

660 # consume value

661 # Skip the value attempt when sitting on the closing bracket: a

662 # value-less position followed by "]" (an empty or trailing-comma

663 # array) would otherwise call _parse_value() only for it to raise

664 # UnexpectedCharError immediately -- and building that discarded

665 # exception eagerly computes a line/column, which scans the whole

666 # source. On a large file with many arrays this is a big, pure waste.

667 if not prev_value and self._current != "]":

668 elems.append(self._parse_value())

669 prev_value = True

670 continue

671

672 # consume comma

673 if prev_value and self._current == ",":

674 self.inc(exception=UnexpectedEofError)

675 # If the previous item is Whitespace, add to it

676 if isinstance(elems[-1], Whitespace):

677 elems[-1]._s = elems[-1].s + ","

678 else:

679 elems.append(Whitespace(","))

680 prev_value = False

681 continue

682

683 # consume closing bracket

684 if self._current == "]":

685 # consume closing bracket, EOF here doesn't matter

686 self.inc()

687 break

688

689 raise self.parse_error(UnexpectedCharError, self._current)

690

691 try:

692 res = Array(elems, Trivia())

693 except ValueError:

694 pass

695 else:

696 return res

697

698 raise self.parse_error(ParseError, "Failed to parse array")

699

700 def _parse_inline_table(self) -> InlineTable:

701 # consume opening bracket, EOF here is an issue (middle of array)

702 self.inc(exception=UnexpectedEofError)

703

704 elems = Container(True)

705 expect_key = True

706 while True:

707 while True:

708 # consume whitespace and newlines

709 mark = self._idx

710 self.consume(" \t\n\r")

711 raw = self._src[mark : self._idx]

712 if raw:

713 elems.add(Whitespace(raw))

714

715 if self._current != "#":

716 break

717

718 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

719 elems.add(Comment(Trivia("", cws, comment, trail)))

720

721 if self._current == "}":

722 # consume closing bracket, EOF here doesn't matter

723 self.inc()

724 break

725

726 if expect_key:

727 if self._current == ",":

728 raise self.parse_error(UnexpectedCharError, self._current)

729 key, val = self._parse_key_value(False)

730 elems.add(key, val)

731 expect_key = False

732 continue

733

734 if self._current != ",":

735 raise self.parse_error(UnexpectedCharError, self._current)

736

737 elems.add(Whitespace(","))

738 # consume comma, EOF here is an issue (middle of inline table)

739 self.inc(exception=UnexpectedEofError)

740 expect_key = True

741

742 return InlineTable(elems, Trivia())

743

744 def _parse_number(self, raw: str, trivia: Trivia) -> Item | None:

745 # Leading zeros are not allowed

746 sign = ""

747 if raw.startswith(("+", "-")):

748 sign = raw[0]

749 raw = raw[1:]

750

751 if len(raw) > 1 and (

752 (raw.startswith("0") and not raw.startswith(("0.", "0o", "0x", "0b", "0e")))

753 or (sign and raw.startswith("."))

754 ):

755 return None

756

757 if raw.startswith(("0o", "0x", "0b")) and sign:

758 return None

759

760 digits = "[0-9]"

761 base = 10

762 if raw.startswith("0b"):

763 digits = "[01]"

764 base = 2

765 elif raw.startswith("0o"):

766 digits = "[0-7]"

767 base = 8

768 elif raw.startswith("0x"):

769 digits = "[0-9a-f]"

770 base = 16

771

772 # Underscores should be surrounded by digits

773 clean = re.sub(f"(?i)(?<={digits})_(?={digits})", "", raw).lower()

774

775 if "_" in clean:

776 return None

777

778 if clean.endswith(".") or (

779 not clean.startswith("0x") and clean.split("e", 1)[0].endswith(".")

780 ):

781 return None

782

783 try:

784 return Integer(int(sign + clean, base), trivia, sign + raw)

785 except ValueError:

786 pass

787

788 # Only fall back to float for an actual float literal (a fractional

789 # dot, a base-10 exponent, or inf/nan). A decimal integer whose digit

790 # count exceeds Python's int-from-string conversion limit also raises

791 # ValueError above; it must be rejected, not silently coerced to inf.

792 if base == 10 and ("." in clean or "e" in clean or clean in ("inf", "nan")):

793 try:

794 return Float(float(sign + clean), trivia, sign + raw)

795 except ValueError:

796 return None

797

798 return None

799

800 def _parse_literal_string(self) -> String:

801 with self._state:

802 return self._parse_string(StringType.SLL)

803

804 def _parse_basic_string(self) -> String:

805 with self._state:

806 return self._parse_string(StringType.SLB)

807

808 def _parse_escaped_char(self, multiline: bool) -> str:

809 if multiline and self._current in _WS:

810 # When the last non-whitespace character on a line is

811 # a \, it will be trimmed along with all whitespace

812 # (including newlines) up to the next non-whitespace

813 # character or closing delimiter.

814 # """\

815 # hello \

816 # world"""

817 tmp = ""

818 while self._current in _WS:

819 tmp += self._current

820 # consume the whitespace, EOF here is an issue

821 # (middle of string)

822 self.inc(exception=UnexpectedEofError)

823 continue

824

825 # the escape followed by whitespace must have a newline

826 # before any other chars

827 if "\n" not in tmp:

828 raise self.parse_error(InvalidCharInStringError, self._current)

829

830 return ""

831

832 if self._current in _escaped:

833 c = _escaped[self._current]

834

835 # consume this char, EOF here is an issue (middle of string)

836 self.inc(exception=UnexpectedEofError)

837

838 return c

839

840 if self._current in {"u", "U"}:

841 # this needs to be a unicode

842 u, ue = self._peek_unicode(self._current == "U")

843 if u is not None:

844 assert ue is not None

845 # consume the U char and the unicode value

846 self.inc_n(len(ue) + 1)

847

848 return u

849

850 raise self.parse_error(InvalidUnicodeValueError)

851

852 if self._current == "x":

853 h, he = self._peek_hex()

854 if h is not None:

855 assert he is not None

856 # consume the x char and the hex value

857 self.inc_n(len(he) + 1)

858 return h

859

860 raise self.parse_error(InvalidUnicodeValueError)

861

862 raise self.parse_error(InvalidCharInStringError, self._current)

863

864 def _parse_string(self, delim: StringType) -> String:

865 # only keep parsing for string if the current character matches the delim

866 if self._current != delim.unit:

867 raise self.parse_error(

868 InternalParserError,

869 f"Invalid character for string type {delim}",

870 )

871

872 # consume the opening/first delim, EOF here is an issue

873 # (middle of string or middle of delim)

874 self.inc(exception=UnexpectedEofError)

875

876 if self._current == delim.unit:

877 # consume the closing/second delim, we do not care if EOF occurs as

878 # that would simply imply an empty single line string

879 if not self.inc() or self._current != delim.unit:

880 # Empty string

881 return String(delim, "", "", Trivia())

882

883 # consume the third delim, EOF here is an issue (middle of string)

884 self.inc(exception=UnexpectedEofError)

885

886 delim = delim.toggle() # convert delim to multi delim

887

888 self.mark() # to extract the original string with whitespace and all

889 value = ""

890

891 # A newline immediately following the opening delimiter will be trimmed.

892 if delim.is_multiline():

893 if self._current == "\n":

894 # consume the newline, EOF here is an issue (middle of string)

895 self.inc(exception=UnexpectedEofError)

896 else:

897 cur: str = self._current

898 with self._state(restore=True):

899 if self.inc():

900 cur += self._current

901 if cur == "\r\n":

902 self.inc_n(2, exception=UnexpectedEofError)

903

904 # PERF: stop-set for the string-body bulk fast-path. The body run is

905 # appended in a single slice up to the next delimiter / escape / control

906 # char (and, for multiline, CR); that stop char is then handled by the

907 # branches above on the next iteration.

908 src = self._src

909 if delim.is_singleline():

910 body_stop = _SINGLE_BASIC_STOP if delim.is_basic() else _SINGLE_LITERAL_STOP

911 else:

912 body_stop = _MULTI_BASIC_STOP if delim.is_basic() else _MULTI_LITERAL_STOP

913

914 escaped = False # whether the previous key was ESCAPE

915 while True:

916 code = ord(self._current)

917 if (

918 delim.is_singleline()

919 and not escaped

920 and (code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I))

921 ) or (

922 delim.is_multiline()

923 and not escaped

924 and (

925 code == CHR_DEL

926 or (

927 code <= CTRL_CHAR_LIMIT and code not in [CTRL_I, CTRL_J, CTRL_M]

928 )

929 )

930 ):

931 raise self.parse_error(InvalidControlChar, code, "strings")

932 elif delim.is_multiline() and not escaped and self._current == "\r":

933 with self._state(restore=True):

934 if not self.inc() or self._current != "\n":

935 raise self.parse_error(InvalidControlChar, CTRL_M, "strings")

936 value += self._current

937 self.inc(exception=UnexpectedEofError)

938 elif not escaped and self._current == delim.unit:

939 # try to process current as a closing delim

940 original = self.extract()

941

942 close = ""

943 if delim.is_multiline():

944 # Consume the delimiters to see if we are at the end of the string

945 close = ""

946 while self._current == delim.unit:

947 close += self._current

948 self.inc()

949

950 if len(close) < 3:

951 # Not a triple quote, leave in result as-is.

952 # Adding back the characters we already consumed

953 value += close

954 continue

955

956 if len(close) == 3:

957 # We are at the end of the string

958 return String(delim, value, original, Trivia())

959

960 if len(close) >= 6:

961 raise self.parse_error(InvalidCharInStringError, self._current)

962

963 value += close[:-3]

964 original += close[:-3]

965

966 return String(delim, value, original, Trivia())

967 else:

968 # consume the closing delim, we do not care if EOF occurs as

969 # that would simply imply the end of self._src

970 self.inc()

971

972 return String(delim, value, original, Trivia())

973 elif delim.is_basic() and escaped:

974 # attempt to parse the current char as an escaped value, an exception

975 # is raised if this fails

976 value += self._parse_escaped_char(delim.is_multiline())

977

978 # no longer escaped

979 escaped = False

980 elif delim.is_basic() and self._current == "\\":

981 # the next char is being escaped

982 escaped = True

983

984 # consume this char, EOF here is an issue (middle of string)

985 self.inc(exception=UnexpectedEofError)

986 else:

987 # this is either a literal string where we keep everything as is,

988 # or this is not a special escaped char in a basic string.

989 # PERF fast-path: bulk-append the run of ordinary characters up to

990 # the next delimiter / backslash / control char (and CR for

991 # multiline) in one slice, instead of one `value += cur; inc()`

992 # iteration per character. The stop char is then handled by the

993 # branches above on the next iteration. For multiline, raw LF and

994 # tab are not stop chars, so a whole multi-line body is consumed

995 # in a single pass.

996 run_start = src._idx

997 src.advance_until(body_stop)

998 if src.end():

999 # mid-string EOF — same error as the per-char inc()

1000 raise self.parse_error(UnexpectedEofError)

1001 value += src[run_start : src._idx]

1002

1003 def _parse_table(

1004 self, parent_name: Key | None = None, parent: Table | None = None

1005 ) -> tuple[Key, Table | AoT]:

1006 """

1007 Parses a table element.

1008 """

1009 if self._current != "[":

1010 raise self.parse_error(

1011 InternalParserError, "_parse_table() called on non-bracket character."

1012 )

1013

1014 indent = self.extract()

1015 self.inc() # Skip opening bracket

1016

1017 if self.end():

1018 raise self.parse_error(UnexpectedEofError)

1019

1020 is_aot = False

1021 if self._current == "[":

1022 if not self.inc():

1023 raise self.parse_error(UnexpectedEofError)

1024

1025 is_aot = True

1026 try:

1027 key = self._parse_key()

1028 except EmptyKeyError:

1029 raise self.parse_error(EmptyTableNameError) from None

1030 if self.end():

1031 raise self.parse_error(UnexpectedEofError)

1032 elif self._current != "]":

1033 raise self.parse_error(UnexpectedCharError, self._current)

1034

1035 key.sep = ""

1036 full_key = key

1037 name_parts = tuple(key)

1038 if any(" " in part.key.strip() and part.is_bare() for part in name_parts):

1039 raise self.parse_error(

1040 ParseError, f'Invalid table name "{full_key.as_string()}"'

1041 )

1042

1043 missing_table = False

1044 if parent_name:

1045 parent_name_parts = tuple(parent_name)

1046 else:

1047 parent_name_parts = ()

1048

1049 if len(name_parts) > len(parent_name_parts) + 1:

1050 missing_table = True

1051

1052 name_parts = name_parts[len(parent_name_parts) :]

1053

1054 values = Container(True)

1055

1056 self.inc() # Skip closing bracket

1057 if is_aot:

1058 if self.end():

1059 raise self.parse_error(UnexpectedEofError)

1060 elif self._current != "]":

1061 raise self.parse_error(UnexpectedCharError, self._current)

1062

1063 self.inc() # Skip second closing bracket

1064

1065 cws, comment, trail = self._parse_comment_trail()

1066

1067 result: Table | AoT = Null() # type: ignore[assignment]

1068 table = Table(

1069 values,

1070 Trivia(indent, cws, comment, trail),

1071 is_aot,

1072 name=name_parts[0].key if name_parts else key.key,

1073 display_name=full_key.as_string(),

1074 is_super_table=False,

1075 )

1076

1077 if len(name_parts) > 1:

1078 if missing_table:

1079 # Missing super table

1080 # i.e. a table initialized like this: [foo.bar]

1081 # without initializing [foo]

1082 #

1083 # So we have to create the parent tables

1084 table = Table(

1085 Container(True),

1086 Trivia("", cws, comment, trail),

1087 is_aot and name_parts[0] in self._aot_stack,

1088 is_super_table=True,

1089 name=name_parts[0].key,

1090 )

1091

1092 result = table

1093 key = name_parts[0]

1094

1095 for i, _name in enumerate(name_parts[1:]):

1096 child = table.get(

1097 _name,

1098 Table(

1099 Container(True),

1100 Trivia(indent, cws, comment, trail),

1101 is_aot and i == len(name_parts) - 2,

1102 is_super_table=i < len(name_parts) - 2,

1103 name=_name.key,

1104 display_name=(

1105 full_key.as_string() if i == len(name_parts) - 2 else None

1106 ),

1107 ),

1108 )

1109

1110 if is_aot and i == len(name_parts) - 2:

1111 table.raw_append(_name, AoT([child], name=table.name, parsed=True))

1112 else:

1113 table.raw_append(_name, child)

1114

1115 table = child

1116 values = table.value

1117 else:

1118 if name_parts:

1119 key = name_parts[0]

1120

1121 while not self.end():

1122 parsed = self._parse_item()

1123 if parsed:

1124 _key, _val = parsed

1125 if not self._merge_ws(_val, values):

1126 table.raw_append(_key, _val)

1127 else:

1128 if self._current == "[":

1129 _, key_next = self._peek_table()

1130

1131 if self._is_child(full_key, key_next):

1132 key_next, table_next = self._parse_table(full_key, table)

1133

1134 table.raw_append(key_next, table_next)

1135

1136 # Picking up any sibling

1137 while not self.end():

1138 _, key_next = self._peek_table()

1139

1140 if not self._is_child(full_key, key_next):

1141 break

1142

1143 key_next, table_next = self._parse_table(full_key, table)

1144

1145 table.raw_append(key_next, table_next)

1146

1147 break

1148 else:

1149 raise self.parse_error(

1150 InternalParserError,

1151 "_parse_item() returned None on a non-bracket character.",

1152 )

1153 table.value._validate_out_of_order_table()

1154 if isinstance(result, Null):

1155 result = table

1156

1157 if is_aot and (not self._aot_stack or full_key != self._aot_stack[-1]):

1158 result = self._parse_aot(result, full_key)

1159

1160 return key, result

1161

1162 def _peek_table(self) -> tuple[bool, Key]:

1163 """

1164 Peeks ahead non-intrusively by cloning then restoring the

1165 initial state of the parser.

1166

1167 Returns the name of the table about to be parsed,

1168 as well as whether it is part of an AoT.

1169 """

1170 # we always want to restore after exiting this scope

1171 with self._state(save_marker=True, restore=True):

1172 if self._current != "[":

1173 raise self.parse_error(

1174 InternalParserError,

1175 "_peek_table() entered on non-bracket character",

1176 )

1177

1178 # AoT

1179 self.inc()

1180 is_aot = False

1181 if self._current == "[":

1182 self.inc()

1183 is_aot = True

1184 try:

1185 return is_aot, self._parse_key()

1186 except EmptyKeyError:

1187 raise self.parse_error(EmptyTableNameError) from None

1188

1189 def _parse_aot(self, first: Table, name_first: Key) -> AoT:

1190 """

1191 Parses all siblings of the provided table first and bundles them into

1192 an AoT.

1193 """

1194 payload: list[Table] = [first]

1195 self._aot_stack.append(name_first)

1196 while not self.end():

1197 is_aot_next, name_next = self._peek_table()

1198 if is_aot_next and name_next == name_first:

1199 _, table = self._parse_table(name_first)

1200 assert isinstance(table, Table)

1201 payload.append(table)

1202 else:

1203 break

1204

1205 self._aot_stack.pop()

1206

1207 return AoT(payload, parsed=True)

1208

1209 def _peek(self, n: int) -> str:

1210 """

1211 Peeks ahead n characters.

1212

1213 n is the max number of characters that will be peeked.

1214 """

1215 # we always want to restore after exiting this scope

1216 with self._state(restore=True):

1217 buf = ""

1218 for _ in range(n):

1219 if self._current not in " \t\n\r#,]}" + self._src.EOF:

1220 buf += self._current

1221 self.inc()

1222 continue

1223

1224 break

1225 return buf

1226

1227 def _peek_unicode(self, is_long: bool) -> tuple[str | None, str | None]:

1228 """

1229 Peeks ahead non-intrusively by cloning then restoring the

1230 initial state of the parser.

1231

1232 Returns the unicode value is it's a valid one else None.

1233 """

1234 # we always want to restore after exiting this scope

1235 with self._state(save_marker=True, restore=True):

1236 if self._current not in {"u", "U"}:

1237 raise self.parse_error(

1238 InternalParserError, "_peek_unicode() entered on non-unicode value"

1239 )

1240

1241 self.inc() # Dropping prefix

1242 self.mark()

1243

1244 if is_long:

1245 chars = 8

1246 else:

1247 chars = 4

1248

1249 if not self.inc_n(chars):

1250 value, extracted = None, None

1251 else:

1252 extracted = self.extract()

1253

1254 if extracted.strip("0123456789abcdefABCDEF"):

1255 return None, extracted

1256

1257 codepoint = int(extracted, 16)

1258

1259 # Unicode scalar values exclude the surrogate range

1260 # (U+D800 to U+DFFF). The 8-digit \U form reaches this range

1261 # with leading zeros, so it must be checked on the value itself.

1262 if 0xD800 <= codepoint <= 0xDFFF:

1263 return None, extracted

1264

1265 try:

1266 value = chr(codepoint)

1267 except (ValueError, OverflowError):

1268 value = None

1269

1270 return value, extracted

1271

1272 def _peek_hex(self) -> tuple[str | None, str | None]:

1273 with self._state(save_marker=True, restore=True):

1274 if self._current != "x":

1275 raise self.parse_error(

1276 InternalParserError, "_peek_hex() entered on non-hex value"

1277 )

1278

1279 self.inc() # Dropping prefix

1280 self.mark()

1281

1282 if not self.inc_n(2):

1283 return None, None

1284

1285 extracted = self.extract()

1286 if extracted.strip("0123456789abcdefABCDEF"):

1287 return None, None

1288

1289 try:

1290 value = chr(int(extracted, 16))

1291 except (ValueError, OverflowError):

1292 value = None

1293

1294 return value, extracted