Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tomlkit/parser.py: 98%

1from __future__ import annotations

3import datetime

4import re

5import string

7from typing import Any

8from typing import Callable

10from tomlkit._compat import decode

11from tomlkit._utils import RFC_3339_LOOSE

12from tomlkit._utils import _escaped

13from tomlkit._utils import parse_rfc3339

14from tomlkit.container import Container

15from tomlkit.exceptions import EmptyKeyError

16from tomlkit.exceptions import EmptyTableNameError

17from tomlkit.exceptions import InternalParserError

18from tomlkit.exceptions import InvalidCharInStringError

19from tomlkit.exceptions import InvalidControlChar

20from tomlkit.exceptions import InvalidDateError

21from tomlkit.exceptions import InvalidDateTimeError

22from tomlkit.exceptions import InvalidNumberError

23from tomlkit.exceptions import InvalidTimeError

24from tomlkit.exceptions import InvalidUnicodeValueError

25from tomlkit.exceptions import ParseError

26from tomlkit.exceptions import UnexpectedCharError

27from tomlkit.exceptions import UnexpectedEofError

28from tomlkit.items import AoT

29from tomlkit.items import Array

30from tomlkit.items import Bool

31from tomlkit.items import BoolType

32from tomlkit.items import Comment

33from tomlkit.items import Date

34from tomlkit.items import DateTime

35from tomlkit.items import Float

36from tomlkit.items import InlineTable

37from tomlkit.items import Integer

38from tomlkit.items import Item

39from tomlkit.items import Key

40from tomlkit.items import KeyType

41from tomlkit.items import Null

42from tomlkit.items import SingleKey

43from tomlkit.items import String

44from tomlkit.items import StringType

45from tomlkit.items import Table

46from tomlkit.items import Time

47from tomlkit.items import Trivia

48from tomlkit.items import Whitespace

49from tomlkit.source import Source

50from tomlkit.source import _StateHandler

51from tomlkit.toml_document import TOMLDocument

54CTRL_I = 0x09 # Tab

55CTRL_J = 0x0A # Line feed

56CTRL_M = 0x0D # Carriage return

57CTRL_CHAR_LIMIT = 0x1F

58CHR_DEL = 0x7F

60# TOML character classes (formerly the `TOMLChar` constants), as frozensets for

61# O(1) membership tests; also the stop-sets for the Source.advance_while /

62# advance_until bulk run scans that replace per-character

63# `while self._current in <set> and self.inc()` loops with a single scan.

64_SPACES = frozenset(" \t")

65_NL = frozenset("\n\r")

66_WS = _SPACES | _NL

67_KV = frozenset("= \t")

68_BARE_KEY_OR_SPACE = frozenset(string.ascii_letters + string.digits + "-_ \t")

69_NUM_STOP = frozenset(" \t\n\r#,]}")

70_DATE_TAIL_STOP = frozenset("\t\n\r#,]}")

71# Control chars invalid inside a single-line string (DEL + everything <= 0x1F

72# except tab) — exactly the set that raises InvalidControlChar in the per-char

73# string loop. The single-line string-body fast-path stops its bulk scan at the

74# first delimiter / backslash / control char, then the main loop handles that

75# char with its existing branch (raising InvalidControlChar where needed).

76_CTRL_SINGLE = frozenset(chr(c) for c in range(0x20) if c != CTRL_I) | {chr(CHR_DEL)}

77_SINGLE_LITERAL_STOP = _CTRL_SINGLE | {"'"} # literal: only the closing quote

78_SINGLE_BASIC_STOP = _CTRL_SINGLE | {'"', "\\"} # basic: quote or escape

80# Same idea for multiline string bodies. A multiline string may contain raw tab,

81# line feed and carriage return, so those are NOT in the control-reject set; but

82# the bulk scan must still stop at CR (the per-char loop validates the \r\n pair

83# and rejects a lone \r) and at the control chars that DO raise. LF and tab are

84# left out of the stop-set entirely, so a multiline body is scanned in one slice

85# across newlines up to the next delimiter / backslash / CR / invalid control.

86_CTRL_MULTI = frozenset(

87 chr(c) for c in range(0x20) if c not in (CTRL_I, CTRL_J, CTRL_M)

88) | {chr(CHR_DEL)}

89_MULTI_LITERAL_STOP = _CTRL_MULTI | {"'", "\r"} # literal: closing quote or CR

90_MULTI_BASIC_STOP = _CTRL_MULTI | {'"', "\\", "\r"} # basic: quote, escape or CR

93class Parser:

94 """

95 Parser for TOML documents.

96 """

98 # Deeply nested documents would overflow the interpreter stack: arrays and

99 # inline tables are parsed recursively, and every fragment of a dotted key

100 # adds a level of nested containers. Refuse documents beyond this depth.

101 MAX_NESTING_DEPTH = 100

102

103 def __init__(self, string: str | bytes) -> None:

104 # Input to parse

105 self._src = Source(decode(string))

106

107 self._aot_stack: list[Key] = []

108 self._nesting_depth = 0

109

110 @property

111 def _state(self) -> _StateHandler:

112 return self._src.state

113

114 @property

115 def _idx(self) -> int:

116 return self._src.idx

117

118 @property

119 def _current(self) -> str:

120 return self._src.current

121

122 @property

123 def _marker(self) -> int:

124 return self._src.marker

125

126 def extract(self) -> str:

127 """

128 Extracts the value between marker and index

129 """

130 return self._src.extract()

131

132 def inc(self, exception: type[ParseError] | None = None) -> bool:

133 """

134 Increments the parser if the end of the input has not been reached.

135 Returns whether or not it was able to advance.

136 """

137 return self._src.inc(exception=exception)

138

139 def inc_n(self, n: int, exception: type[ParseError] | None = None) -> bool:

140 """

141 Increments the parser by n characters

142 if the end of the input has not been reached.

143 """

144 return self._src.inc_n(n=n, exception=exception)

145

146 def consume(self, chars: str, min: int = 0, max: int = -1) -> None:

147 """

148 Consume chars until min/max is satisfied is valid.

149 """

150 return self._src.consume(chars=chars, min=min, max=max)

151

152 def end(self) -> bool:

153 """

154 Returns True if the parser has reached the end of the input.

155 """

156 return self._src.end()

157

158 def mark(self) -> None:

159 """

160 Sets the marker to the index's current position

161 """

162 self._src.mark()

163

164 def parse_error(

165 self,

166 exception: type[ParseError] = ParseError,

167 *args: Any,

168 **kwargs: Any,

169 ) -> ParseError:

170 """

171 Creates a generic "parse error" at the current position.

172 """

173 return self._src.parse_error(exception, *args, **kwargs)

174

175 def parse(self) -> TOMLDocument:

176 body = TOMLDocument(True)

177

178 # Take all keyvals outside of tables/AoT's.

179 while not self.end():

180 # Break out if a table is found

181 if self._current == "[":

182 break

183

184 # Otherwise, take and append one KV

185 item = self._parse_item()

186 if not item:

187 break

188

189 key, value = item

190 if (key is not None and key.is_multi()) or not self._merge_ws(value, body):

191 # We actually have a table

192 try:

193 body.append(key, value)

194 except Exception as e:

195 raise self.parse_error(ParseError, str(e)) from e

196

197 self.mark()

198

199 while not self.end():

200 key, value = self._parse_table()

201 if isinstance(value, Table) and value.is_aot_element():

202 # This is just the first table in an AoT. Parse the rest of the array

203 # along with it.

204 value = self._parse_aot(value, key)

205

206 try:

207 body.append(key, value)

208 except Exception as e:

209 raise self.parse_error(ParseError, str(e)) from e

210

211 body.parsing(False)

212

213 return body

214

215 def _merge_ws(self, item: Item, container: Container) -> bool:

216 """

217 Merges the given Item with the last one currently in the given Container if

218 both are whitespace items.

219

220 Returns True if the items were merged.

221 """

222 last = container.last_item()

223 if not last:

224 return False

225

226 if not isinstance(item, Whitespace) or not isinstance(last, Whitespace):

227 return False

228

229 start = self._idx - (len(last.s) + len(item.s))

230 container.body[-1] = (

231 container.body[-1][0],

232 Whitespace(self._src[start : self._idx]),

233 )

234

235 return True

236

237 def _is_child(self, parent: Key, child: Key) -> bool:

238 """

239 Returns whether a key is strictly a child of another key.

240 AoT siblings are not considered children of one another.

241 """

242 parent_parts = tuple(parent)

243 child_parts = tuple(child)

244

245 if parent_parts == child_parts:

246 return False

247

248 return parent_parts == child_parts[: len(parent_parts)]

249

250 def _parse_item(self) -> tuple[Key | None, Item] | None:

251 """

252 Attempts to parse the next item and returns it, along with its key

253 if the item is value-like.

254 """

255 self.mark()

256 with self._state as state:

257 while True:

258 c = self._current

259 if c == "\n":

260 # Found a newline; Return all whitespace found up to this point.

261 self.inc()

262

263 return None, Whitespace(self.extract())

264 elif c in " \t\r":

265 if c == "\r":

266 with self._state(restore=True):

267 if not self.inc() or self._current != "\n":

268 raise self.parse_error(

269 InvalidControlChar, CTRL_M, "documents"

270 )

271 # Skip whitespace.

272 if not self.inc():

273 return None, Whitespace(self.extract())

274 elif c == "#":

275 # Found a comment, parse it

276 indent = self.extract()

277 cws, comment, trail = self._parse_comment_trail()

278

279 return None, Comment(Trivia(indent, cws, comment, trail))

280 elif c == "[":

281 # Found a table, delegate to the calling function.

282 return None

283 else:

284 # Beginning of a KV pair.

285 # Return to beginning of whitespace so it gets included

286 # as indentation for the KV about to be parsed.

287 state.restore = True

288 break

289

290 return self._parse_key_value(True)

291

292 def _parse_comment_trail(self, parse_trail: bool = True) -> tuple[str, str, str]:

293 """

294 Returns (comment_ws, comment, trail)

295 If there is no comment, comment_ws and comment will

296 simply be empty.

297 """

298 if self.end():

299 return "", "", ""

300

301 comment = ""

302 comment_ws = ""

303 self.mark()

304

305 while True:

306 c = self._current

307

308 if c == "\n":

309 break

310 elif c == "#":

311 comment_ws = self.extract()

312

313 self.mark()

314 self.inc() # Skip #

315

316 # The comment itself

317 while not self.end() and self._current not in _NL:

318 code = ord(self._current)

319 if code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I):

320 raise self.parse_error(InvalidControlChar, code, "comments")

321

322 if not self.inc():

323 break

324

325 comment = self.extract()

326 self.mark()

327

328 break

329 elif c in " \t\r":

330 if c == "\r":

331 with self._state(restore=True):

332 if not self.inc() or self._current != "\n":

333 raise self.parse_error(

334 InvalidControlChar, CTRL_M, "comments"

335 )

336 self.inc()

337 else:

338 raise self.parse_error(UnexpectedCharError, c)

339

340 if self.end():

341 break

342

343 trail = ""

344 if parse_trail:

345 self._src.advance_while(_SPACES)

346

347 if self._current == "\r":

348 with self._state(restore=True):

349 if not self.inc() or self._current != "\n":

350 raise self.parse_error(InvalidControlChar, CTRL_M, "documents")

351 self.inc()

352

353 if self._current == "\n":

354 self.inc()

355

356 if self._idx != self._marker or self._current in _WS:

357 trail = self.extract()

358

359 return comment_ws, comment, trail

360

361 def _parse_key_value(self, parse_comment: bool = False) -> tuple[Key, Item]:

362 # Leading indent

363 self.mark()

364

365 self._src.advance_while(_SPACES)

366

367 indent = self.extract()

368

369 # Key

370 key = self._parse_key()

371

372 self.mark()

373

374 found_equals = self._current == "="

375 while self._current in _KV and self.inc():

376 if self._current == "=":

377 if found_equals:

378 raise self.parse_error(UnexpectedCharError, "=")

379 else:

380 found_equals = True

381 if not found_equals:

382 raise self.parse_error(UnexpectedCharError, self._current)

383

384 if not key.sep:

385 key.sep = self.extract()

386 else:

387 key.sep += self.extract()

388

389 # Value

390 val = self._parse_value()

391 # Comment

392 if parse_comment:

393 cws, comment, trail = self._parse_comment_trail()

394 meta = val.trivia

395 if not meta.comment_ws:

396 meta.comment_ws = cws

397

398 meta.comment = comment

399 meta.trail = trail

400 else:

401 val.trivia.trail = ""

402

403 val.trivia.indent = indent

404

405 return key, val

406

407 def _parse_key(self) -> Key:

408 """

409 Parses a Key at the current position;

410 WS before the key must be exhausted first at the callsite.

411 """

412 key = self._parse_simple_key()

413 fragments = 1

414 while self._current == ".":

415 fragments += 1

416 if fragments > self.MAX_NESTING_DEPTH:

417 raise self.parse_error(

418 ParseError,

419 f"TOML key nested more than {self.MAX_NESTING_DEPTH} levels deep",

420 )

421 self.inc()

422 key = key.concat(self._parse_simple_key())

423

424 return key

425

426 def _parse_simple_key(self) -> Key:

427 """

428 Parses a single (non-dotted) key fragment.

429 """

430 self.mark()

431 # Skip any leading whitespace (bulk scan)

432 self._src.advance_while(_SPACES)

433 if self._current in "\"'":

434 return self._parse_quoted_key()

435 else:

436 return self._parse_bare_key()

437

438 def _parse_quoted_key(self) -> Key:

439 """

440 Parses a key enclosed in either single or double quotes.

441 """

442 # Extract the leading whitespace

443 original = self.extract()

444 quote_style = self._current

445 key_type = next((t for t in KeyType if t.value == quote_style), None)

446

447 if key_type is None:

448 raise RuntimeError("Should not have entered _parse_quoted_key()")

449

450 key_str = self._parse_string(

451 StringType.SLB if key_type == KeyType.Basic else StringType.SLL

452 )

453 if key_str._t.is_multiline():

454 raise self.parse_error(UnexpectedCharError, key_str._t.value)

455 original += key_str.as_string()

456 self.mark()

457 self._src.advance_while(_SPACES)

458 original += self.extract()

459

460 return SingleKey(str(key_str), t=key_type, sep="", original=original)

461

462 def _parse_bare_key(self) -> Key:

463 """

464 Parses a bare key.

465 """

466 self._src.advance_while(_BARE_KEY_OR_SPACE)

467

468 original = self.extract()

469 key_s = original.strip()

470 if not key_s:

471 # Empty key

472 raise self.parse_error(EmptyKeyError)

473

474 if " " in key_s or "\t" in key_s:

475 # Bare key with whitespace in it

476 raise self.parse_error(ParseError, f'Invalid key "{key_s}"')

477

478 return SingleKey(key_s, KeyType.Bare, "", original)

479

480 def _parse_value(self) -> Item:

481 """

482 Attempts to parse a value at the current position.

483 """

484 self.mark()

485 c = self._current

486 trivia = Trivia()

487

488 if c == StringType.SLB.value:

489 return self._parse_basic_string()

490 elif c == StringType.SLL.value:

491 return self._parse_literal_string()

492 elif c == BoolType.TRUE.value[0]:

493 return self._parse_true()

494 elif c == BoolType.FALSE.value[0]:

495 return self._parse_false()

496 elif c == "[":

497 return self._parse_nested(self._parse_array)

498 elif c == "{":

499 return self._parse_nested(self._parse_inline_table)

500 elif c in "+-" or self._peek(4) in {

501 "+inf",

502 "-inf",

503 "inf",

504 "+nan",

505 "-nan",

506 "nan",

507 }:

508 # Number

509 self._src.advance_until(_NUM_STOP)

510

511 raw = self.extract()

512

513 item = self._parse_number(raw, trivia)

514 if item is not None:

515 return item

516

517 raise self.parse_error(InvalidNumberError)

518 elif c in string.digits:

519 # Integer, Float, Date, Time or DateTime

520 self._src.advance_until(_NUM_STOP)

521

522 raw = self.extract()

523

524 m = RFC_3339_LOOSE.match(raw)

525 if m:

526 if m.group("date") and m.group("time"):

527 # datetime

528 try:

529 dt = parse_rfc3339(raw)

530 assert isinstance(dt, datetime.datetime)

531 return DateTime(

532 dt.year,

533 dt.month,

534 dt.day,

535 dt.hour,

536 dt.minute,

537 dt.second,

538 dt.microsecond,

539 dt.tzinfo,

540 trivia,

541 raw,

542 )

543 except ValueError:

544 raise self.parse_error(InvalidDateTimeError) from None

545

546 if m.group("date"):

547 try:

548 dt = parse_rfc3339(raw)

549 assert isinstance(dt, datetime.date)

550 date = Date(dt.year, dt.month, dt.day, trivia, raw)

551 self.mark()

552 self._src.advance_until(_DATE_TAIL_STOP)

553

554 time_raw = self.extract()

555 time_part = time_raw.rstrip()

556 trivia.comment_ws = time_raw[len(time_part) :]

557 if not time_part:

558 return date

559

560 dt = parse_rfc3339(raw + time_part)

561 assert isinstance(dt, datetime.datetime)

562 return DateTime(

563 dt.year,

564 dt.month,

565 dt.day,

566 dt.hour,

567 dt.minute,

568 dt.second,

569 dt.microsecond,

570 dt.tzinfo,

571 trivia,

572 raw + time_part,

573 )

574 except ValueError:

575 raise self.parse_error(InvalidDateError) from None

576

577 if m.group("time"):

578 try:

579 t = parse_rfc3339(raw)

580 assert isinstance(t, datetime.time)

581 return Time(

582 t.hour,

583 t.minute,

584 t.second,

585 t.microsecond,

586 t.tzinfo,

587 trivia,

588 raw,

589 )

590 except ValueError:

591 raise self.parse_error(InvalidTimeError) from None

592

593 item = self._parse_number(raw, trivia)

594 if item is not None:

595 return item

596

597 raise self.parse_error(InvalidNumberError)

598 else:

599 raise self.parse_error(UnexpectedCharError, c)

600

601 def _parse_true(self) -> Bool:

602 return self._parse_bool(BoolType.TRUE)

603

604 def _parse_false(self) -> Bool:

605 return self._parse_bool(BoolType.FALSE)

606

607 def _parse_bool(self, style: BoolType) -> Bool:

608 with self._state:

609 style = BoolType(style)

610

611 # only keep parsing for bool if the characters match the style

612 # try consuming rest of chars in style

613 for c in style:

614 self.consume(c, min=1, max=1)

615

616 return Bool(style, Trivia())

617

618 def _parse_nested(self, parse: Callable[[], Item]) -> Item:

619 """

620 Parses an array or inline table, enforcing the nesting depth limit.

621 """

622 self._nesting_depth += 1

623 if self._nesting_depth > self.MAX_NESTING_DEPTH:

624 raise self.parse_error(

625 ParseError,

626 f"TOML value nested more than {self.MAX_NESTING_DEPTH} levels deep",

627 )

628 try:

629 return parse()

630 finally:

631 self._nesting_depth -= 1

632

633 def _parse_array(self) -> Array:

634 # Consume opening bracket, EOF here is an issue (middle of array)

635 self.inc(exception=UnexpectedEofError)

636

637 elems: list[Item] = []

638 prev_value = None

639 while True:

640 # consume whitespace

641 mark = self._idx

642 self.consume(" \t\n\r")

643 indent = self._src[mark : self._idx]

644 newline = _NL & set(indent)

645 if newline:

646 elems.append(Whitespace(indent))

647 continue

648

649 # consume comment

650 if self._current == "#":

651 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

652 elems.append(Comment(Trivia(indent, cws, comment, trail)))

653 continue

654

655 # consume indent

656 if indent:

657 elems.append(Whitespace(indent))

658 continue

659

660 # consume value

661 # Skip the value attempt when sitting on the closing bracket: a

662 # value-less position followed by "]" (an empty or trailing-comma

663 # array) would otherwise call _parse_value() only for it to raise

664 # UnexpectedCharError immediately -- and building that discarded

665 # exception eagerly computes a line/column, which scans the whole

666 # source. On a large file with many arrays this is a big, pure waste.

667 if not prev_value and self._current != "]":

668 elems.append(self._parse_value())

669 prev_value = True

670 continue

671

672 # consume comma

673 if prev_value and self._current == ",":

674 self.inc(exception=UnexpectedEofError)

675 # If the previous item is Whitespace, add to it

676 if isinstance(elems[-1], Whitespace):

677 elems[-1]._s = elems[-1].s + ","

678 else:

679 elems.append(Whitespace(","))

680 prev_value = False

681 continue

682

683 # consume closing bracket

684 if self._current == "]":

685 # consume closing bracket, EOF here doesn't matter

686 self.inc()

687 break

688

689 raise self.parse_error(UnexpectedCharError, self._current)

690

691 try:

692 res = Array(elems, Trivia())

693 except ValueError:

694 pass

695 else:

696 return res

697

698 raise self.parse_error(ParseError, "Failed to parse array")

699

700 def _parse_inline_table(self) -> InlineTable:

701 # consume opening bracket, EOF here is an issue (middle of array)

702 self.inc(exception=UnexpectedEofError)

703

704 elems = Container(True)

705 expect_key = True

706 while True:

707 while True:

708 # consume whitespace and newlines

709 mark = self._idx

710 self.consume(" \t\n\r")

711 raw = self._src[mark : self._idx]

712 if raw:

713 elems.add(Whitespace(raw))

714

715 if self._current != "#":

716 break

717

718 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

719 elems.add(Comment(Trivia("", cws, comment, trail)))

720

721 if self._current == "}":

722 # consume closing bracket, EOF here doesn't matter

723 self.inc()

724 break

725

726 if expect_key:

727 if self._current == ",":

728 raise self.parse_error(UnexpectedCharError, self._current)

729 key, val = self._parse_key_value(False)

730 elems.add(key, val)

731 expect_key = False

732 continue

733

734 if self._current != ",":

735 raise self.parse_error(UnexpectedCharError, self._current)

736

737 elems.add(Whitespace(","))

738 # consume comma, EOF here is an issue (middle of inline table)

739 self.inc(exception=UnexpectedEofError)

740 expect_key = True

741

742 return InlineTable(elems, Trivia())

743

744 def _parse_number(self, raw: str, trivia: Trivia) -> Item | None:

745 # Leading zeros are not allowed

746 sign = ""

747 if raw.startswith(("+", "-")):

748 sign = raw[0]

749 raw = raw[1:]

750

751 if len(raw) > 1 and (

752 (

753 raw.startswith("0")

754 and not raw.startswith(("0.", "0o", "0x", "0b", "0e", "0E"))

755 )

756 or (sign and raw.startswith("."))

757 ):

758 return None

759

760 if raw.startswith(("0o", "0x", "0b")) and sign:

761 return None

762

763 digits = "[0-9]"

764 base = 10

765 if raw.startswith("0b"):

766 digits = "[01]"

767 base = 2

768 elif raw.startswith("0o"):

769 digits = "[0-7]"

770 base = 8

771 elif raw.startswith("0x"):

772 digits = "[0-9a-f]"

773 base = 16

774

775 # Underscores should be surrounded by digits

776 clean = re.sub(f"(?i)(?<={digits})_(?={digits})", "", raw).lower()

777

778 if "_" in clean:

779 return None

780

781 if clean.endswith(".") or (

782 not clean.startswith("0x") and clean.split("e", 1)[0].endswith(".")

783 ):

784 return None

785

786 try:

787 return Integer(int(sign + clean, base), trivia, sign + raw)

788 except ValueError:

789 pass

790

791 # Only fall back to float for an actual float literal (a fractional

792 # dot, a base-10 exponent, or inf/nan). A decimal integer whose digit

793 # count exceeds Python's int-from-string conversion limit also raises

794 # ValueError above; it must be rejected, not silently coerced to inf.

795 if base == 10 and ("." in clean or "e" in clean or clean in ("inf", "nan")):

796 try:

797 return Float(float(sign + clean), trivia, sign + raw)

798 except ValueError:

799 return None

800

801 return None

802

803 def _parse_literal_string(self) -> String:

804 with self._state:

805 return self._parse_string(StringType.SLL)

806

807 def _parse_basic_string(self) -> String:

808 with self._state:

809 return self._parse_string(StringType.SLB)

810

811 def _parse_escaped_char(self, multiline: bool) -> str:

812 if multiline and self._current in _WS:

813 # When the last non-whitespace character on a line is

814 # a \, it will be trimmed along with all whitespace

815 # (including newlines) up to the next non-whitespace

816 # character or closing delimiter.

817 # """\

818 # hello \

819 # world"""

820 tmp = ""

821 while self._current in _WS:

822 tmp += self._current

823 # consume the whitespace, EOF here is an issue

824 # (middle of string)

825 self.inc(exception=UnexpectedEofError)

826 continue

827

828 # the escape followed by whitespace must have a newline

829 # before any other chars

830 if "\n" not in tmp:

831 raise self.parse_error(InvalidCharInStringError, self._current)

832

833 return ""

834

835 if self._current in _escaped:

836 c = _escaped[self._current]

837

838 # consume this char, EOF here is an issue (middle of string)

839 self.inc(exception=UnexpectedEofError)

840

841 return c

842

843 if self._current in {"u", "U"}:

844 # this needs to be a unicode

845 u, ue = self._peek_unicode(self._current == "U")

846 if u is not None:

847 assert ue is not None

848 # consume the U char and the unicode value

849 self.inc_n(len(ue) + 1)

850

851 return u

852

853 raise self.parse_error(InvalidUnicodeValueError)

854

855 if self._current == "x":

856 h, he = self._peek_hex()

857 if h is not None:

858 assert he is not None

859 # consume the x char and the hex value

860 self.inc_n(len(he) + 1)

861 return h

862

863 raise self.parse_error(InvalidUnicodeValueError)

864

865 raise self.parse_error(InvalidCharInStringError, self._current)

866

867 def _parse_string(self, delim: StringType) -> String:

868 # only keep parsing for string if the current character matches the delim

869 if self._current != delim.unit:

870 raise self.parse_error(

871 InternalParserError,

872 f"Invalid character for string type {delim}",

873 )

874

875 # consume the opening/first delim, EOF here is an issue

876 # (middle of string or middle of delim)

877 self.inc(exception=UnexpectedEofError)

878

879 if self._current == delim.unit:

880 # consume the closing/second delim, we do not care if EOF occurs as

881 # that would simply imply an empty single line string

882 if not self.inc() or self._current != delim.unit:

883 # Empty string

884 return String(delim, "", "", Trivia())

885

886 # consume the third delim, EOF here is an issue (middle of string)

887 self.inc(exception=UnexpectedEofError)

888

889 delim = delim.toggle() # convert delim to multi delim

890

891 self.mark() # to extract the original string with whitespace and all

892 value = ""

893

894 # A newline immediately following the opening delimiter will be trimmed.

895 if delim.is_multiline():

896 if self._current == "\n":

897 # consume the newline, EOF here is an issue (middle of string)

898 self.inc(exception=UnexpectedEofError)

899 else:

900 cur: str = self._current

901 with self._state(restore=True):

902 if self.inc():

903 cur += self._current

904 if cur == "\r\n":

905 self.inc_n(2, exception=UnexpectedEofError)

906

907 # PERF: stop-set for the string-body bulk fast-path. The body run is

908 # appended in a single slice up to the next delimiter / escape / control

909 # char (and, for multiline, CR); that stop char is then handled by the

910 # branches above on the next iteration.

911 src = self._src

912 if delim.is_singleline():

913 body_stop = _SINGLE_BASIC_STOP if delim.is_basic() else _SINGLE_LITERAL_STOP

914 else:

915 body_stop = _MULTI_BASIC_STOP if delim.is_basic() else _MULTI_LITERAL_STOP

916

917 escaped = False # whether the previous key was ESCAPE

918 while True:

919 code = ord(self._current)

920 if (

921 delim.is_singleline()

922 and not escaped

923 and (code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I))

924 ) or (

925 delim.is_multiline()

926 and not escaped

927 and (

928 code == CHR_DEL

929 or (

930 code <= CTRL_CHAR_LIMIT and code not in [CTRL_I, CTRL_J, CTRL_M]

931 )

932 )

933 ):

934 raise self.parse_error(InvalidControlChar, code, "strings")

935 elif delim.is_multiline() and not escaped and self._current == "\r":

936 with self._state(restore=True):

937 if not self.inc() or self._current != "\n":

938 raise self.parse_error(InvalidControlChar, CTRL_M, "strings")

939 value += self._current

940 self.inc(exception=UnexpectedEofError)

941 elif not escaped and self._current == delim.unit:

942 # try to process current as a closing delim

943 original = self.extract()

944

945 close = ""

946 if delim.is_multiline():

947 # Consume the delimiters to see if we are at the end of the string

948 close = ""

949 while self._current == delim.unit:

950 close += self._current

951 self.inc()

952

953 if len(close) < 3:

954 # Not a triple quote, leave in result as-is.

955 # Adding back the characters we already consumed

956 value += close

957 continue

958

959 if len(close) == 3:

960 # We are at the end of the string

961 return String(delim, value, original, Trivia())

962

963 if len(close) >= 6:

964 raise self.parse_error(InvalidCharInStringError, self._current)

965

966 value += close[:-3]

967 original += close[:-3]

968

969 return String(delim, value, original, Trivia())

970 else:

971 # consume the closing delim, we do not care if EOF occurs as

972 # that would simply imply the end of self._src

973 self.inc()

974

975 return String(delim, value, original, Trivia())

976 elif delim.is_basic() and escaped:

977 # attempt to parse the current char as an escaped value, an exception

978 # is raised if this fails

979 value += self._parse_escaped_char(delim.is_multiline())

980

981 # no longer escaped

982 escaped = False

983 elif delim.is_basic() and self._current == "\\":

984 # the next char is being escaped

985 escaped = True

986

987 # consume this char, EOF here is an issue (middle of string)

988 self.inc(exception=UnexpectedEofError)

989 else:

990 # this is either a literal string where we keep everything as is,

991 # or this is not a special escaped char in a basic string.

992 # PERF fast-path: bulk-append the run of ordinary characters up to

993 # the next delimiter / backslash / control char (and CR for

994 # multiline) in one slice, instead of one `value += cur; inc()`

995 # iteration per character. The stop char is then handled by the

996 # branches above on the next iteration. For multiline, raw LF and

997 # tab are not stop chars, so a whole multi-line body is consumed

998 # in a single pass.

999 run_start = src._idx

1000 src.advance_until(body_stop)

1001 if src.end():

1002 # mid-string EOF — same error as the per-char inc()

1003 raise self.parse_error(UnexpectedEofError)

1004 value += src[run_start : src._idx]

1005

1006 def _parse_table(

1007 self, parent_name: Key | None = None, parent: Table | None = None

1008 ) -> tuple[Key, Table | AoT]:

1009 """

1010 Parses a table element.

1011 """

1012 if self._current != "[":

1013 raise self.parse_error(

1014 InternalParserError, "_parse_table() called on non-bracket character."

1015 )

1016

1017 indent = self.extract()

1018 self.inc() # Skip opening bracket

1019

1020 if self.end():

1021 raise self.parse_error(UnexpectedEofError)

1022

1023 is_aot = False

1024 if self._current == "[":

1025 if not self.inc():

1026 raise self.parse_error(UnexpectedEofError)

1027

1028 is_aot = True

1029 try:

1030 key = self._parse_key()

1031 except EmptyKeyError:

1032 raise self.parse_error(EmptyTableNameError) from None

1033 if self.end():

1034 raise self.parse_error(UnexpectedEofError)

1035 elif self._current != "]":

1036 raise self.parse_error(UnexpectedCharError, self._current)

1037

1038 key.sep = ""

1039 full_key = key

1040 name_parts = tuple(key)

1041 if any(" " in part.key.strip() and part.is_bare() for part in name_parts):

1042 raise self.parse_error(

1043 ParseError, f'Invalid table name "{full_key.as_string()}"'

1044 )

1045

1046 missing_table = False

1047 if parent_name:

1048 parent_name_parts = tuple(parent_name)

1049 else:

1050 parent_name_parts = ()

1051

1052 if len(name_parts) > len(parent_name_parts) + 1:

1053 missing_table = True

1054

1055 name_parts = name_parts[len(parent_name_parts) :]

1056

1057 values = Container(True)

1058

1059 self.inc() # Skip closing bracket

1060 if is_aot:

1061 if self.end():

1062 raise self.parse_error(UnexpectedEofError)

1063 elif self._current != "]":

1064 raise self.parse_error(UnexpectedCharError, self._current)

1065

1066 self.inc() # Skip second closing bracket

1067

1068 cws, comment, trail = self._parse_comment_trail()

1069

1070 result: Table | AoT = Null() # type: ignore[assignment]

1071 table = Table(

1072 values,

1073 Trivia(indent, cws, comment, trail),

1074 is_aot,

1075 name=name_parts[0].key if name_parts else key.key,

1076 display_name=full_key.as_string(),

1077 is_super_table=False,

1078 )

1079

1080 if len(name_parts) > 1:

1081 if missing_table:

1082 # Missing super table

1083 # i.e. a table initialized like this: [foo.bar]

1084 # without initializing [foo]

1085 #

1086 # So we have to create the parent tables

1087 table = Table(

1088 Container(True),

1089 Trivia("", cws, comment, trail),

1090 is_aot and name_parts[0] in self._aot_stack,

1091 is_super_table=True,

1092 name=name_parts[0].key,

1093 )

1094

1095 result = table

1096 key = name_parts[0]

1097

1098 for i, _name in enumerate(name_parts[1:]):

1099 child = table.get(

1100 _name,

1101 Table(

1102 Container(True),

1103 Trivia(indent, cws, comment, trail),

1104 is_aot and i == len(name_parts) - 2,

1105 is_super_table=i < len(name_parts) - 2,

1106 name=_name.key,

1107 display_name=(

1108 full_key.as_string() if i == len(name_parts) - 2 else None

1109 ),

1110 ),

1111 )

1112

1113 if is_aot and i == len(name_parts) - 2:

1114 table.raw_append(_name, AoT([child], name=table.name, parsed=True))

1115 else:

1116 table.raw_append(_name, child)

1117

1118 table = child

1119 values = table.value

1120 else:

1121 if name_parts:

1122 key = name_parts[0]

1123

1124 while not self.end():

1125 parsed = self._parse_item()

1126 if parsed:

1127 _key, _val = parsed

1128 if not self._merge_ws(_val, values):

1129 table.raw_append(_key, _val)

1130 else:

1131 if self._current == "[":

1132 _, key_next = self._peek_table()

1133

1134 if self._is_child(full_key, key_next):

1135 key_next, table_next = self._parse_table(full_key, table)

1136

1137 table.raw_append(key_next, table_next)

1138

1139 # Picking up any sibling

1140 while not self.end():

1141 _, key_next = self._peek_table()

1142

1143 if not self._is_child(full_key, key_next):

1144 break

1145

1146 key_next, table_next = self._parse_table(full_key, table)

1147

1148 table.raw_append(key_next, table_next)

1149

1150 break

1151 else:

1152 raise self.parse_error(

1153 InternalParserError,

1154 "_parse_item() returned None on a non-bracket character.",

1155 )

1156 table.value._validate_out_of_order_table()

1157 if isinstance(result, Null):

1158 result = table

1159

1160 if is_aot and (not self._aot_stack or full_key != self._aot_stack[-1]):

1161 result = self._parse_aot(result, full_key)

1162

1163 return key, result

1164

1165 def _peek_table(self) -> tuple[bool, Key]:

1166 """

1167 Peeks ahead non-intrusively by cloning then restoring the

1168 initial state of the parser.

1169

1170 Returns the name of the table about to be parsed,

1171 as well as whether it is part of an AoT.

1172 """

1173 # we always want to restore after exiting this scope

1174 with self._state(save_marker=True, restore=True):

1175 if self._current != "[":

1176 raise self.parse_error(

1177 InternalParserError,

1178 "_peek_table() entered on non-bracket character",

1179 )

1180

1181 # AoT

1182 self.inc()

1183 is_aot = False

1184 if self._current == "[":

1185 self.inc()

1186 is_aot = True

1187 try:

1188 return is_aot, self._parse_key()

1189 except EmptyKeyError:

1190 raise self.parse_error(EmptyTableNameError) from None

1191

1192 def _parse_aot(self, first: Table, name_first: Key) -> AoT:

1193 """

1194 Parses all siblings of the provided table first and bundles them into

1195 an AoT.

1196 """

1197 payload: list[Table] = [first]

1198 self._aot_stack.append(name_first)

1199 while not self.end():

1200 is_aot_next, name_next = self._peek_table()

1201 if is_aot_next and name_next == name_first:

1202 _, table = self._parse_table(name_first)

1203 assert isinstance(table, Table)

1204 payload.append(table)

1205 else:

1206 break

1207

1208 self._aot_stack.pop()

1209

1210 return AoT(payload, parsed=True)

1211

1212 def _peek(self, n: int) -> str:

1213 """

1214 Peeks ahead n characters.

1215

1216 n is the max number of characters that will be peeked.

1217 """

1218 # we always want to restore after exiting this scope

1219 with self._state(restore=True):

1220 buf = ""

1221 for _ in range(n):

1222 if self._current not in " \t\n\r#,]}" + self._src.EOF:

1223 buf += self._current

1224 self.inc()

1225 continue

1226

1227 break

1228 return buf

1229

1230 def _peek_unicode(self, is_long: bool) -> tuple[str | None, str | None]:

1231 """

1232 Peeks ahead non-intrusively by cloning then restoring the

1233 initial state of the parser.

1234

1235 Returns the unicode value is it's a valid one else None.

1236 """

1237 # we always want to restore after exiting this scope

1238 with self._state(save_marker=True, restore=True):

1239 if self._current not in {"u", "U"}:

1240 raise self.parse_error(

1241 InternalParserError, "_peek_unicode() entered on non-unicode value"

1242 )

1243

1244 self.inc() # Dropping prefix

1245 self.mark()

1246

1247 if is_long:

1248 chars = 8

1249 else:

1250 chars = 4

1251

1252 if not self.inc_n(chars):

1253 value, extracted = None, None

1254 else:

1255 extracted = self.extract()

1256

1257 if extracted.strip("0123456789abcdefABCDEF"):

1258 return None, extracted

1259

1260 codepoint = int(extracted, 16)

1261

1262 # Unicode scalar values exclude the surrogate range

1263 # (U+D800 to U+DFFF). The 8-digit \U form reaches this range

1264 # with leading zeros, so it must be checked on the value itself.

1265 if 0xD800 <= codepoint <= 0xDFFF:

1266 return None, extracted

1267

1268 try:

1269 value = chr(codepoint)

1270 except (ValueError, OverflowError):

1271 value = None

1272

1273 return value, extracted

1274

1275 def _peek_hex(self) -> tuple[str | None, str | None]:

1276 with self._state(save_marker=True, restore=True):

1277 if self._current != "x":

1278 raise self.parse_error(

1279 InternalParserError, "_peek_hex() entered on non-hex value"

1280 )

1281

1282 self.inc() # Dropping prefix

1283 self.mark()

1284

1285 if not self.inc_n(2):

1286 return None, None

1287

1288 extracted = self.extract()

1289 if extracted.strip("0123456789abcdefABCDEF"):

1290 return None, None

1291

1292 try:

1293 value = chr(int(extracted, 16))

1294 except (ValueError, OverflowError):

1295 value = None

1296

1297 return value, extracted