Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/tomlkit/parser.py: 98%

1from __future__ import annotations

3import datetime

4import re

5import string

7from typing import Any

8from typing import Callable

10from tomlkit._compat import decode

11from tomlkit._utils import RFC_3339_LOOSE

12from tomlkit._utils import _escaped

13from tomlkit._utils import parse_rfc3339

14from tomlkit.container import Container

15from tomlkit.exceptions import EmptyKeyError

16from tomlkit.exceptions import EmptyTableNameError

17from tomlkit.exceptions import InternalParserError

18from tomlkit.exceptions import InvalidCharInStringError

19from tomlkit.exceptions import InvalidControlChar

20from tomlkit.exceptions import InvalidDateError

21from tomlkit.exceptions import InvalidDateTimeError

22from tomlkit.exceptions import InvalidNumberError

23from tomlkit.exceptions import InvalidTimeError

24from tomlkit.exceptions import InvalidUnicodeValueError

25from tomlkit.exceptions import ParseError

26from tomlkit.exceptions import UnexpectedCharError

27from tomlkit.exceptions import UnexpectedEofError

28from tomlkit.items import AoT

29from tomlkit.items import Array

30from tomlkit.items import Bool

31from tomlkit.items import BoolType

32from tomlkit.items import Comment

33from tomlkit.items import Date

34from tomlkit.items import DateTime

35from tomlkit.items import Float

36from tomlkit.items import InlineTable

37from tomlkit.items import Integer

38from tomlkit.items import Item

39from tomlkit.items import Key

40from tomlkit.items import KeyType

41from tomlkit.items import Null

42from tomlkit.items import SingleKey

43from tomlkit.items import String

44from tomlkit.items import StringType

45from tomlkit.items import Table

46from tomlkit.items import Time

47from tomlkit.items import Trivia

48from tomlkit.items import Whitespace

49from tomlkit.source import Source

50from tomlkit.source import _StateHandler

51from tomlkit.toml_document import TOMLDocument

54CTRL_I = 0x09 # Tab

55CTRL_J = 0x0A # Line feed

56CTRL_M = 0x0D # Carriage return

57CTRL_CHAR_LIMIT = 0x1F

58CHR_DEL = 0x7F

60# TOML character classes (formerly the `TOMLChar` constants), as frozensets for

61# O(1) membership tests; also the stop-sets for the Source.advance_while /

62# advance_until bulk run scans that replace per-character

63# `while self._current in <set> and self.inc()` loops with a single scan.

64_SPACES = frozenset(" \t")

65_NL = frozenset("\n\r")

66_WS = _SPACES | _NL

67_KV = frozenset("= \t")

68_BARE_KEY_OR_SPACE = frozenset(string.ascii_letters + string.digits + "-_ \t")

69_NUM_STOP = frozenset(" \t\n\r#,]}")

70_DATE_TAIL_STOP = frozenset("\t\n\r#,]}")

71# Control chars invalid inside a single-line string (DEL + everything <= 0x1F

72# except tab) — exactly the set that raises InvalidControlChar in the per-char

73# string loop. The single-line string-body fast-path stops its bulk scan at the

74# first delimiter / backslash / control char, then the main loop handles that

75# char with its existing branch (raising InvalidControlChar where needed).

76_CTRL_SINGLE = frozenset(chr(c) for c in range(0x20) if c != CTRL_I) | {chr(CHR_DEL)}

77_SINGLE_LITERAL_STOP = _CTRL_SINGLE | {"'"} # literal: only the closing quote

78_SINGLE_BASIC_STOP = _CTRL_SINGLE | {'"', "\\"} # basic: quote or escape

80# Same idea for multiline string bodies. A multiline string may contain raw tab,

81# line feed and carriage return, so those are NOT in the control-reject set; but

82# the bulk scan must still stop at CR (the per-char loop validates the \r\n pair

83# and rejects a lone \r) and at the control chars that DO raise. LF and tab are

84# left out of the stop-set entirely, so a multiline body is scanned in one slice

85# across newlines up to the next delimiter / backslash / CR / invalid control.

86_CTRL_MULTI = frozenset(

87 chr(c) for c in range(0x20) if c not in (CTRL_I, CTRL_J, CTRL_M)

88) | {chr(CHR_DEL)}

89_MULTI_LITERAL_STOP = _CTRL_MULTI | {"'", "\r"} # literal: closing quote or CR

90_MULTI_BASIC_STOP = _CTRL_MULTI | {'"', "\\", "\r"} # basic: quote, escape or CR

93class Parser:

94 """

95 Parser for TOML documents.

96 """

98 # Deeply nested documents would overflow the interpreter stack: arrays and

99 # inline tables are parsed recursively, and every fragment of a dotted key

100 # adds a level of nested containers. Refuse documents beyond this depth.

101 MAX_NESTING_DEPTH = 100

102

103 def __init__(self, string: str | bytes) -> None:

104 # Input to parse

105 self._src = Source(decode(string))

106

107 self._aot_stack: list[Key] = []

108 self._nesting_depth = 0

109

110 @property

111 def _state(self) -> _StateHandler:

112 return self._src.state

113

114 @property

115 def _idx(self) -> int:

116 return self._src.idx

117

118 @property

119 def _current(self) -> str:

120 return self._src.current

121

122 @property

123 def _marker(self) -> int:

124 return self._src.marker

125

126 def extract(self) -> str:

127 """

128 Extracts the value between marker and index

129 """

130 return self._src.extract()

131

132 def inc(self, exception: type[ParseError] | None = None) -> bool:

133 """

134 Increments the parser if the end of the input has not been reached.

135 Returns whether or not it was able to advance.

136 """

137 return self._src.inc(exception=exception)

138

139 def inc_n(self, n: int, exception: type[ParseError] | None = None) -> bool:

140 """

141 Increments the parser by n characters

142 if the end of the input has not been reached.

143 """

144 return self._src.inc_n(n=n, exception=exception)

145

146 def consume(self, chars: str, min: int = 0, max: int = -1) -> None:

147 """

148 Consume chars until min/max is satisfied is valid.

149 """

150 return self._src.consume(chars=chars, min=min, max=max)

151

152 def end(self) -> bool:

153 """

154 Returns True if the parser has reached the end of the input.

155 """

156 return self._src.end()

157

158 def mark(self) -> None:

159 """

160 Sets the marker to the index's current position

161 """

162 self._src.mark()

163

164 def parse_error(

165 self,

166 exception: type[ParseError] = ParseError,

167 *args: Any,

168 **kwargs: Any,

169 ) -> ParseError:

170 """

171 Creates a generic "parse error" at the current position.

172 """

173 return self._src.parse_error(exception, *args, **kwargs)

174

175 def parse(self) -> TOMLDocument:

176 body = TOMLDocument(True)

177

178 # Take all keyvals outside of tables/AoT's.

179 while not self.end():

180 # Break out if a table is found

181 if self._current == "[":

182 break

183

184 # Otherwise, take and append one KV

185 item = self._parse_item()

186 if not item:

187 break

188

189 key, value = item

190 if (key is not None and key.is_multi()) or not self._merge_ws(value, body):

191 # We actually have a table

192 try:

193 body.append(key, value)

194 except Exception as e:

195 raise self.parse_error(ParseError, str(e)) from e

196

197 self.mark()

198

199 while not self.end():

200 key, value = self._parse_table()

201 if isinstance(value, Table) and value.is_aot_element():

202 # This is just the first table in an AoT. Parse the rest of the array

203 # along with it.

204 value = self._parse_aot(value, key)

205

206 try:

207 body.append(key, value)

208 except Exception as e:

209 raise self.parse_error(ParseError, str(e)) from e

210

211 body.parsing(False)

212

213 return body

214

215 def _merge_ws(self, item: Item, container: Container) -> bool:

216 """

217 Merges the given Item with the last one currently in the given Container if

218 both are whitespace items.

219

220 Returns True if the items were merged.

221 """

222 last = container.last_item()

223 if not last:

224 return False

225

226 if not isinstance(item, Whitespace) or not isinstance(last, Whitespace):

227 return False

228

229 start = self._idx - (len(last.s) + len(item.s))

230 container.body[-1] = (

231 container.body[-1][0],

232 Whitespace(self._src[start : self._idx]),

233 )

234

235 return True

236

237 def _is_child(self, parent: Key, child: Key) -> bool:

238 """

239 Returns whether a key is strictly a child of another key.

240 AoT siblings are not considered children of one another.

241 """

242 parent_parts = tuple(parent)

243 child_parts = tuple(child)

244

245 if parent_parts == child_parts:

246 return False

247

248 return parent_parts == child_parts[: len(parent_parts)]

249

250 def _parse_item(self) -> tuple[Key | None, Item] | None:

251 """

252 Attempts to parse the next item and returns it, along with its key

253 if the item is value-like.

254 """

255 self.mark()

256 with self._state as state:

257 while True:

258 c = self._current

259 if c == "\n":

260 # Found a newline; Return all whitespace found up to this point.

261 self.inc()

262

263 return None, Whitespace(self.extract())

264 elif c in " \t\r":

265 if c == "\r":

266 with self._state(restore=True):

267 if not self.inc() or self._current != "\n":

268 raise self.parse_error(

269 InvalidControlChar, CTRL_M, "documents"

270 )

271 # Skip whitespace.

272 if not self.inc():

273 return None, Whitespace(self.extract())

274 elif c == "#":

275 # Found a comment, parse it

276 indent = self.extract()

277 cws, comment, trail = self._parse_comment_trail()

278

279 return None, Comment(Trivia(indent, cws, comment, trail))

280 elif c == "[":

281 # Found a table, delegate to the calling function.

282 return None

283 else:

284 # Beginning of a KV pair.

285 # Return to beginning of whitespace so it gets included

286 # as indentation for the KV about to be parsed.

287 state.restore = True

288 break

289

290 return self._parse_key_value(True)

291

292 def _parse_comment_trail(self, parse_trail: bool = True) -> tuple[str, str, str]:

293 """

294 Returns (comment_ws, comment, trail)

295 If there is no comment, comment_ws and comment will

296 simply be empty.

297 """

298 if self.end():

299 return "", "", ""

300

301 comment = ""

302 comment_ws = ""

303 self.mark()

304

305 while True:

306 c = self._current

307

308 if c == "\n":

309 break

310 elif c == "#":

311 comment_ws = self.extract()

312

313 self.mark()

314 self.inc() # Skip #

315

316 # The comment itself

317 while not self.end() and self._current not in _NL:

318 code = ord(self._current)

319 if code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I):

320 raise self.parse_error(InvalidControlChar, code, "comments")

321

322 if not self.inc():

323 break

324

325 comment = self.extract()

326 self.mark()

327

328 break

329 elif c in " \t\r":

330 if c == "\r":

331 with self._state(restore=True):

332 if not self.inc() or self._current != "\n":

333 raise self.parse_error(

334 InvalidControlChar, CTRL_M, "comments"

335 )

336 self.inc()

337 else:

338 raise self.parse_error(UnexpectedCharError, c)

339

340 if self.end():

341 break

342

343 trail = ""

344 if parse_trail:

345 self._src.advance_while(_SPACES)

346

347 if self._current == "\r":

348 with self._state(restore=True):

349 if not self.inc() or self._current != "\n":

350 raise self.parse_error(InvalidControlChar, CTRL_M, "documents")

351 self.inc()

352

353 if self._current == "\n":

354 self.inc()

355

356 if self._idx != self._marker or self._current in _WS:

357 trail = self.extract()

358

359 return comment_ws, comment, trail

360

361 def _parse_key_value(self, parse_comment: bool = False) -> tuple[Key, Item]:

362 # Leading indent

363 self.mark()

364

365 self._src.advance_while(_SPACES)

366

367 indent = self.extract()

368

369 # Key

370 key = self._parse_key()

371

372 self.mark()

373

374 found_equals = self._current == "="

375 while self._current in _KV and self.inc():

376 if self._current == "=":

377 if found_equals:

378 raise self.parse_error(UnexpectedCharError, "=")

379 else:

380 found_equals = True

381 if not found_equals:

382 raise self.parse_error(UnexpectedCharError, self._current)

383

384 if not key.sep:

385 key.sep = self.extract()

386 else:

387 key.sep += self.extract()

388

389 # Value

390 val = self._parse_value()

391 # Comment

392 if parse_comment:

393 cws, comment, trail = self._parse_comment_trail()

394 meta = val.trivia

395 if not meta.comment_ws:

396 meta.comment_ws = cws

397

398 meta.comment = comment

399 meta.trail = trail

400 else:

401 val.trivia.trail = ""

402

403 val.trivia.indent = indent

404

405 return key, val

406

407 def _parse_key(self) -> Key:

408 """

409 Parses a Key at the current position;

410 WS before the key must be exhausted first at the callsite.

411 """

412 key = self._parse_simple_key()

413 fragments = 1

414 while self._current == ".":

415 fragments += 1

416 if fragments > self.MAX_NESTING_DEPTH:

417 raise self.parse_error(

418 ParseError,

419 f"TOML key nested more than {self.MAX_NESTING_DEPTH} levels deep",

420 )

421 self.inc()

422 key = key.concat(self._parse_simple_key())

423

424 return key

425

426 def _parse_simple_key(self) -> Key:

427 """

428 Parses a single (non-dotted) key fragment.

429 """

430 self.mark()

431 # Skip any leading whitespace (bulk scan)

432 self._src.advance_while(_SPACES)

433 if self._current in "\"'":

434 return self._parse_quoted_key()

435 else:

436 return self._parse_bare_key()

437

438 def _parse_quoted_key(self) -> Key:

439 """

440 Parses a key enclosed in either single or double quotes.

441 """

442 # Extract the leading whitespace

443 original = self.extract()

444 quote_style = self._current

445 key_type = next((t for t in KeyType if t.value == quote_style), None)

446

447 if key_type is None:

448 raise RuntimeError("Should not have entered _parse_quoted_key()")

449

450 key_str = self._parse_string(

451 StringType.SLB if key_type == KeyType.Basic else StringType.SLL

452 )

453 if key_str._t.is_multiline():

454 raise self.parse_error(UnexpectedCharError, key_str._t.value)

455 original += key_str.as_string()

456 self.mark()

457 self._src.advance_while(_SPACES)

458 original += self.extract()

459

460 return SingleKey(str(key_str), t=key_type, sep="", original=original)

461

462 def _parse_bare_key(self) -> Key:

463 """

464 Parses a bare key.

465 """

466 self._src.advance_while(_BARE_KEY_OR_SPACE)

467

468 original = self.extract()

469 key_s = original.strip()

470 if not key_s:

471 # Empty key

472 raise self.parse_error(EmptyKeyError)

473

474 if " " in key_s or "\t" in key_s:

475 # Bare key with whitespace in it

476 raise self.parse_error(ParseError, f'Invalid key "{key_s}"')

477

478 return SingleKey(key_s, KeyType.Bare, "", original)

479

480 def _parse_value(self) -> Item:

481 """

482 Attempts to parse a value at the current position.

483 """

484 self.mark()

485 c = self._current

486 trivia = Trivia()

487

488 if c == StringType.SLB.value:

489 return self._parse_basic_string()

490 elif c == StringType.SLL.value:

491 return self._parse_literal_string()

492 elif c == BoolType.TRUE.value[0]:

493 return self._parse_true()

494 elif c == BoolType.FALSE.value[0]:

495 return self._parse_false()

496 elif c == "[":

497 return self._parse_nested(self._parse_array)

498 elif c == "{":

499 return self._parse_nested(self._parse_inline_table)

500 elif c in "+-" or self._peek(4) in {

501 "+inf",

502 "-inf",

503 "inf",

504 "+nan",

505 "-nan",

506 "nan",

507 }:

508 # Number

509 self._src.advance_until(_NUM_STOP)

510

511 raw = self.extract()

512

513 item = self._parse_number(raw, trivia)

514 if item is not None:

515 return item

516

517 raise self.parse_error(InvalidNumberError)

518 elif c in string.digits:

519 # Integer, Float, Date, Time or DateTime

520 self._src.advance_until(_NUM_STOP)

521

522 raw = self.extract()

523

524 m = RFC_3339_LOOSE.match(raw)

525 if m:

526 if m.group("date") and m.group("time"):

527 # datetime

528 try:

529 dt = parse_rfc3339(raw)

530 assert isinstance(dt, datetime.datetime)

531 return DateTime(

532 dt.year,

533 dt.month,

534 dt.day,

535 dt.hour,

536 dt.minute,

537 dt.second,

538 dt.microsecond,

539 dt.tzinfo,

540 trivia,

541 raw,

542 )

543 except ValueError:

544 raise self.parse_error(InvalidDateTimeError) from None

545

546 if m.group("date"):

547 try:

548 dt = parse_rfc3339(raw)

549 assert isinstance(dt, datetime.date)

550 date = Date(dt.year, dt.month, dt.day, trivia, raw)

551 self.mark()

552 self._src.advance_until(_DATE_TAIL_STOP)

553

554 time_raw = self.extract()

555 time_part = time_raw.rstrip()

556 trivia.comment_ws = time_raw[len(time_part) :]

557 if not time_part:

558 return date

559

560 dt = parse_rfc3339(raw + time_part)

561 assert isinstance(dt, datetime.datetime)

562 return DateTime(

563 dt.year,

564 dt.month,

565 dt.day,

566 dt.hour,

567 dt.minute,

568 dt.second,

569 dt.microsecond,

570 dt.tzinfo,

571 trivia,

572 raw + time_part,

573 )

574 except ValueError:

575 raise self.parse_error(InvalidDateError) from None

576

577 if m.group("time"):

578 try:

579 t = parse_rfc3339(raw)

580 assert isinstance(t, datetime.time)

581 return Time(

582 t.hour,

583 t.minute,

584 t.second,

585 t.microsecond,

586 t.tzinfo,

587 trivia,

588 raw,

589 )

590 except ValueError:

591 raise self.parse_error(InvalidTimeError) from None

592

593 item = self._parse_number(raw, trivia)

594 if item is not None:

595 return item

596

597 raise self.parse_error(InvalidNumberError)

598 else:

599 raise self.parse_error(UnexpectedCharError, c)

600

601 def _parse_true(self) -> Bool:

602 return self._parse_bool(BoolType.TRUE)

603

604 def _parse_false(self) -> Bool:

605 return self._parse_bool(BoolType.FALSE)

606

607 def _parse_bool(self, style: BoolType) -> Bool:

608 with self._state:

609 style = BoolType(style)

610

611 # only keep parsing for bool if the characters match the style

612 # try consuming rest of chars in style

613 for c in style:

614 self.consume(c, min=1, max=1)

615

616 return Bool(style, Trivia())

617

618 def _parse_nested(self, parse: Callable[[], Item]) -> Item:

619 """

620 Parses an array or inline table, enforcing the nesting depth limit.

621 """

622 self._nesting_depth += 1

623 if self._nesting_depth > self.MAX_NESTING_DEPTH:

624 raise self.parse_error(

625 ParseError,

626 f"TOML value nested more than {self.MAX_NESTING_DEPTH} levels deep",

627 )

628 try:

629 return parse()

630 finally:

631 self._nesting_depth -= 1

632

633 def _parse_array(self) -> Array:

634 # Consume opening bracket, EOF here is an issue (middle of array)

635 self.inc(exception=UnexpectedEofError)

636

637 elems: list[Item] = []

638 prev_value = None

639 while True:

640 # consume whitespace

641 mark = self._idx

642 self.consume(" \t\n\r")

643 indent = self._src[mark : self._idx]

644 newline = _NL & set(indent)

645 if newline:

646 elems.append(Whitespace(indent))

647 continue

648

649 # consume comment

650 if self._current == "#":

651 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

652 elems.append(Comment(Trivia(indent, cws, comment, trail)))

653 continue

654

655 # consume indent

656 if indent:

657 elems.append(Whitespace(indent))

658 continue

659

660 # consume value

661 # Skip the value attempt when sitting on the closing bracket: a

662 # value-less position followed by "]" (an empty or trailing-comma

663 # array) would otherwise call _parse_value() only for it to raise

664 # UnexpectedCharError immediately -- and building that discarded

665 # exception eagerly computes a line/column, which scans the whole

666 # source. On a large file with many arrays this is a big, pure waste.

667 if not prev_value and self._current != "]":

668 elems.append(self._parse_value())

669 prev_value = True

670 continue

671

672 # consume comma

673 if prev_value and self._current == ",":

674 self.inc(exception=UnexpectedEofError)

675 # If the previous item is Whitespace, add to it

676 if isinstance(elems[-1], Whitespace):

677 elems[-1]._s = elems[-1].s + ","

678 else:

679 elems.append(Whitespace(","))

680 prev_value = False

681 continue

682

683 # consume closing bracket

684 if self._current == "]":

685 # consume closing bracket, EOF here doesn't matter

686 self.inc()

687 break

688

689 raise self.parse_error(UnexpectedCharError, self._current)

690

691 try:

692 res = Array(elems, Trivia())

693 except ValueError:

694 pass

695 else:

696 return res

697

698 raise self.parse_error(ParseError, "Failed to parse array")

699

700 def _parse_inline_table(self) -> InlineTable:

701 # consume opening bracket, EOF here is an issue (middle of array)

702 self.inc(exception=UnexpectedEofError)

703

704 elems = Container(True)

705 expect_key = True

706 while True:

707 while True:

708 # consume whitespace and newlines

709 mark = self._idx

710 self.consume(" \t\n\r")

711 raw = self._src[mark : self._idx]

712 if raw:

713 elems.add(Whitespace(raw))

714

715 if self._current != "#":

716 break

717

718 cws, comment, trail = self._parse_comment_trail(parse_trail=False)

719 elems.add(Comment(Trivia("", cws, comment, trail)))

720

721 if self._current == "}":

722 # consume closing bracket, EOF here doesn't matter

723 self.inc()

724 break

725

726 if expect_key:

727 if self._current == ",":

728 raise self.parse_error(UnexpectedCharError, self._current)

729 key, val = self._parse_key_value(False)

730 elems.add(key, val)

731 expect_key = False

732 continue

733

734 if self._current != ",":

735 raise self.parse_error(UnexpectedCharError, self._current)

736

737 elems.add(Whitespace(","))

738 # consume comma, EOF here is an issue (middle of inline table)

739 self.inc(exception=UnexpectedEofError)

740 expect_key = True

741

742 return InlineTable(elems, Trivia())

743

744 def _parse_number(self, raw: str, trivia: Trivia) -> Item | None:

745 # Leading zeros are not allowed

746 sign = ""

747 if raw.startswith(("+", "-")):

748 sign = raw[0]

749 raw = raw[1:]

750

751 if len(raw) > 1 and (

752 (

753 raw.startswith("0")

754 and not raw.startswith(("0.", "0o", "0x", "0b", "0e", "0E"))

755 )

756 or (sign and raw.startswith("."))

757 ):

758 return None

759

760 if raw.startswith(("0o", "0x", "0b")) and sign:

761 return None

762

763 digits = "[0-9]"

764 base = 10

765 if raw.startswith("0b"):

766 digits = "[01]"

767 base = 2

768 elif raw.startswith("0o"):

769 digits = "[0-7]"

770 base = 8

771 elif raw.startswith("0x"):

772 digits = "[0-9a-f]"

773 base = 16

774

775 # Underscores should be surrounded by digits

776 clean = re.sub(f"(?i)(?<={digits})_(?={digits})", "", raw).lower()

777

778 if "_" in clean:

779 return None

780

781 # Special float values must be lowercase in the original token.

782 if clean in ("inf", "nan") and raw != clean:

783 return None

784

785 if clean.endswith(".") or (

786 not clean.startswith("0x") and clean.split("e", 1)[0].endswith(".")

787 ):

788 return None

789

790 try:

791 return Integer(int(sign + clean, base), trivia, sign + raw)

792 except ValueError:

793 pass

794

795 # Only fall back to float for an actual float literal (a fractional

796 # dot, a base-10 exponent, or inf/nan). A decimal integer whose digit

797 # count exceeds Python's int-from-string conversion limit also raises

798 # ValueError above; it must be rejected, not silently coerced to inf.

799 if base == 10 and ("." in clean or "e" in clean or clean in ("inf", "nan")):

800 try:

801 return Float(float(sign + clean), trivia, sign + raw)

802 except ValueError:

803 return None

804

805 return None

806

807 def _parse_literal_string(self) -> String:

808 with self._state:

809 return self._parse_string(StringType.SLL)

810

811 def _parse_basic_string(self) -> String:

812 with self._state:

813 return self._parse_string(StringType.SLB)

814

815 def _parse_escaped_char(self, multiline: bool) -> str:

816 if multiline and self._current in _WS:

817 # When the last non-whitespace character on a line is

818 # a \, it will be trimmed along with all whitespace

819 # (including newlines) up to the next non-whitespace

820 # character or closing delimiter.

821 # """\

822 # hello \

823 # world"""

824 tmp = ""

825 while self._current in _WS:

826 tmp += self._current

827 # consume the whitespace, EOF here is an issue

828 # (middle of string)

829 self.inc(exception=UnexpectedEofError)

830 continue

831

832 # the escape followed by whitespace must have a newline

833 # before any other chars

834 if "\n" not in tmp:

835 raise self.parse_error(InvalidCharInStringError, self._current)

836

837 return ""

838

839 if self._current in _escaped:

840 c = _escaped[self._current]

841

842 # consume this char, EOF here is an issue (middle of string)

843 self.inc(exception=UnexpectedEofError)

844

845 return c

846

847 if self._current in {"u", "U"}:

848 # this needs to be a unicode

849 u, ue = self._peek_unicode(self._current == "U")

850 if u is not None:

851 assert ue is not None

852 # consume the U char and the unicode value

853 self.inc_n(len(ue) + 1)

854

855 return u

856

857 raise self.parse_error(InvalidUnicodeValueError)

858

859 if self._current == "x":

860 h, he = self._peek_hex()

861 if h is not None:

862 assert he is not None

863 # consume the x char and the hex value

864 self.inc_n(len(he) + 1)

865 return h

866

867 raise self.parse_error(InvalidUnicodeValueError)

868

869 raise self.parse_error(InvalidCharInStringError, self._current)

870

871 def _parse_string(self, delim: StringType) -> String:

872 # only keep parsing for string if the current character matches the delim

873 if self._current != delim.unit:

874 raise self.parse_error(

875 InternalParserError,

876 f"Invalid character for string type {delim}",

877 )

878

879 # consume the opening/first delim, EOF here is an issue

880 # (middle of string or middle of delim)

881 self.inc(exception=UnexpectedEofError)

882

883 if self._current == delim.unit:

884 # consume the closing/second delim, we do not care if EOF occurs as

885 # that would simply imply an empty single line string

886 if not self.inc() or self._current != delim.unit:

887 # Empty string

888 return String(delim, "", "", Trivia())

889

890 # consume the third delim, EOF here is an issue (middle of string)

891 self.inc(exception=UnexpectedEofError)

892

893 delim = delim.toggle() # convert delim to multi delim

894

895 self.mark() # to extract the original string with whitespace and all

896 value = ""

897

898 # A newline immediately following the opening delimiter will be trimmed.

899 if delim.is_multiline():

900 if self._current == "\n":

901 # consume the newline, EOF here is an issue (middle of string)

902 self.inc(exception=UnexpectedEofError)

903 else:

904 cur: str = self._current

905 with self._state(restore=True):

906 if self.inc():

907 cur += self._current

908 if cur == "\r\n":

909 self.inc_n(2, exception=UnexpectedEofError)

910

911 # PERF: stop-set for the string-body bulk fast-path. The body run is

912 # appended in a single slice up to the next delimiter / escape / control

913 # char (and, for multiline, CR); that stop char is then handled by the

914 # branches above on the next iteration.

915 src = self._src

916 if delim.is_singleline():

917 body_stop = _SINGLE_BASIC_STOP if delim.is_basic() else _SINGLE_LITERAL_STOP

918 else:

919 body_stop = _MULTI_BASIC_STOP if delim.is_basic() else _MULTI_LITERAL_STOP

920

921 escaped = False # whether the previous key was ESCAPE

922 while True:

923 code = ord(self._current)

924 if (

925 delim.is_singleline()

926 and not escaped

927 and (code == CHR_DEL or (code <= CTRL_CHAR_LIMIT and code != CTRL_I))

928 ) or (

929 delim.is_multiline()

930 and not escaped

931 and (

932 code == CHR_DEL

933 or (

934 code <= CTRL_CHAR_LIMIT and code not in [CTRL_I, CTRL_J, CTRL_M]

935 )

936 )

937 ):

938 raise self.parse_error(InvalidControlChar, code, "strings")

939 elif delim.is_multiline() and not escaped and self._current == "\r":

940 with self._state(restore=True):

941 if not self.inc() or self._current != "\n":

942 raise self.parse_error(InvalidControlChar, CTRL_M, "strings")

943 value += self._current

944 self.inc(exception=UnexpectedEofError)

945 elif not escaped and self._current == delim.unit:

946 # try to process current as a closing delim

947 original = self.extract()

948

949 close = ""

950 if delim.is_multiline():

951 # Consume the delimiters to see if we are at the end of the string

952 close = ""

953 while self._current == delim.unit:

954 close += self._current

955 self.inc()

956

957 if len(close) < 3:

958 # Not a triple quote, leave in result as-is.

959 # Adding back the characters we already consumed

960 value += close

961 continue

962

963 if len(close) == 3:

964 # We are at the end of the string

965 return String(delim, value, original, Trivia())

966

967 if len(close) >= 6:

968 raise self.parse_error(InvalidCharInStringError, self._current)

969

970 value += close[:-3]

971 original += close[:-3]

972

973 return String(delim, value, original, Trivia())

974 else:

975 # consume the closing delim, we do not care if EOF occurs as

976 # that would simply imply the end of self._src

977 self.inc()

978

979 return String(delim, value, original, Trivia())

980 elif delim.is_basic() and escaped:

981 # attempt to parse the current char as an escaped value, an exception

982 # is raised if this fails

983 value += self._parse_escaped_char(delim.is_multiline())

984

985 # no longer escaped

986 escaped = False

987 elif delim.is_basic() and self._current == "\\":

988 # the next char is being escaped

989 escaped = True

990

991 # consume this char, EOF here is an issue (middle of string)

992 self.inc(exception=UnexpectedEofError)

993 else:

994 # this is either a literal string where we keep everything as is,

995 # or this is not a special escaped char in a basic string.

996 # PERF fast-path: bulk-append the run of ordinary characters up to

997 # the next delimiter / backslash / control char (and CR for

998 # multiline) in one slice, instead of one `value += cur; inc()`

999 # iteration per character. The stop char is then handled by the

1000 # branches above on the next iteration. For multiline, raw LF and

1001 # tab are not stop chars, so a whole multi-line body is consumed

1002 # in a single pass.

1003 run_start = src._idx

1004 src.advance_until(body_stop)

1005 if src.end():

1006 # mid-string EOF — same error as the per-char inc()

1007 raise self.parse_error(UnexpectedEofError)

1008 value += src[run_start : src._idx]

1009

1010 def _parse_table(

1011 self, parent_name: Key | None = None, parent: Table | None = None

1012 ) -> tuple[Key, Table | AoT]:

1013 """

1014 Parses a table element.

1015 """

1016 if self._current != "[":

1017 raise self.parse_error(

1018 InternalParserError, "_parse_table() called on non-bracket character."

1019 )

1020

1021 indent = self.extract()

1022 self.inc() # Skip opening bracket

1023

1024 if self.end():

1025 raise self.parse_error(UnexpectedEofError)

1026

1027 is_aot = False

1028 if self._current == "[":

1029 if not self.inc():

1030 raise self.parse_error(UnexpectedEofError)

1031

1032 is_aot = True

1033 try:

1034 key = self._parse_key()

1035 except EmptyKeyError:

1036 raise self.parse_error(EmptyTableNameError) from None

1037 if self.end():

1038 raise self.parse_error(UnexpectedEofError)

1039 elif self._current != "]":

1040 raise self.parse_error(UnexpectedCharError, self._current)

1041

1042 key.sep = ""

1043 full_key = key

1044 name_parts = tuple(key)

1045 if any(" " in part.key.strip() and part.is_bare() for part in name_parts):

1046 raise self.parse_error(

1047 ParseError, f'Invalid table name "{full_key.as_string()}"'

1048 )

1049

1050 missing_table = False

1051 if parent_name:

1052 parent_name_parts = tuple(parent_name)

1053 else:

1054 parent_name_parts = ()

1055

1056 if len(name_parts) > len(parent_name_parts) + 1:

1057 missing_table = True

1058

1059 name_parts = name_parts[len(parent_name_parts) :]

1060

1061 values = Container(True)

1062

1063 self.inc() # Skip closing bracket

1064 if is_aot:

1065 if self.end():

1066 raise self.parse_error(UnexpectedEofError)

1067 elif self._current != "]":

1068 raise self.parse_error(UnexpectedCharError, self._current)

1069

1070 self.inc() # Skip second closing bracket

1071

1072 cws, comment, trail = self._parse_comment_trail()

1073

1074 result: Table | AoT = Null() # type: ignore[assignment]

1075 table = Table(

1076 values,

1077 Trivia(indent, cws, comment, trail),

1078 is_aot,

1079 name=name_parts[0].key if name_parts else key.key,

1080 display_name=full_key.as_string(),

1081 is_super_table=False,

1082 )

1083

1084 if len(name_parts) > 1:

1085 if missing_table:

1086 # Missing super table

1087 # i.e. a table initialized like this: [foo.bar]

1088 # without initializing [foo]

1089 #

1090 # So we have to create the parent tables

1091 table = Table(

1092 Container(True),

1093 Trivia("", cws, comment, trail),

1094 is_aot and name_parts[0] in self._aot_stack,

1095 is_super_table=True,

1096 name=name_parts[0].key,

1097 )

1098

1099 result = table

1100 key = name_parts[0]

1101

1102 for i, _name in enumerate(name_parts[1:]):

1103 child = table.get(

1104 _name,

1105 Table(

1106 Container(True),

1107 Trivia(indent, cws, comment, trail),

1108 is_aot and i == len(name_parts) - 2,

1109 is_super_table=i < len(name_parts) - 2,

1110 name=_name.key,

1111 display_name=(

1112 full_key.as_string() if i == len(name_parts) - 2 else None

1113 ),

1114 ),

1115 )

1116

1117 if is_aot and i == len(name_parts) - 2:

1118 table.raw_append(_name, AoT([child], name=table.name, parsed=True))

1119 else:

1120 table.raw_append(_name, child)

1121

1122 table = child

1123 values = table.value

1124 else:

1125 if name_parts:

1126 key = name_parts[0]

1127

1128 while not self.end():

1129 parsed = self._parse_item()

1130 if parsed:

1131 _key, _val = parsed

1132 if not self._merge_ws(_val, values):

1133 table.raw_append(_key, _val)

1134 else:

1135 if self._current == "[":

1136 _, key_next = self._peek_table()

1137

1138 if self._is_child(full_key, key_next):

1139 key_next, table_next = self._parse_table(full_key, table)

1140

1141 table.raw_append(key_next, table_next)

1142

1143 # Picking up any sibling

1144 while not self.end():

1145 _, key_next = self._peek_table()

1146

1147 if not self._is_child(full_key, key_next):

1148 break

1149

1150 key_next, table_next = self._parse_table(full_key, table)

1151

1152 table.raw_append(key_next, table_next)

1153

1154 break

1155 else:

1156 raise self.parse_error(

1157 InternalParserError,

1158 "_parse_item() returned None on a non-bracket character.",

1159 )

1160 table.value._validate_out_of_order_table()

1161 if isinstance(result, Null):

1162 result = table

1163

1164 if is_aot and (not self._aot_stack or full_key != self._aot_stack[-1]):

1165 result = self._parse_aot(result, full_key)

1166

1167 return key, result

1168

1169 def _peek_table(self) -> tuple[bool, Key]:

1170 """

1171 Peeks ahead non-intrusively by cloning then restoring the

1172 initial state of the parser.

1173

1174 Returns the name of the table about to be parsed,

1175 as well as whether it is part of an AoT.

1176 """

1177 # we always want to restore after exiting this scope

1178 with self._state(save_marker=True, restore=True):

1179 if self._current != "[":

1180 raise self.parse_error(

1181 InternalParserError,

1182 "_peek_table() entered on non-bracket character",

1183 )

1184

1185 # AoT

1186 self.inc()

1187 is_aot = False

1188 if self._current == "[":

1189 self.inc()

1190 is_aot = True

1191 try:

1192 return is_aot, self._parse_key()

1193 except EmptyKeyError:

1194 raise self.parse_error(EmptyTableNameError) from None

1195

1196 def _parse_aot(self, first: Table, name_first: Key) -> AoT:

1197 """

1198 Parses all siblings of the provided table first and bundles them into

1199 an AoT.

1200 """

1201 payload: list[Table] = [first]

1202 self._aot_stack.append(name_first)

1203 while not self.end():

1204 is_aot_next, name_next = self._peek_table()

1205 if is_aot_next and name_next == name_first:

1206 _, table = self._parse_table(name_first)

1207 assert isinstance(table, Table)

1208 payload.append(table)

1209 else:

1210 break

1211

1212 self._aot_stack.pop()

1213

1214 return AoT(payload, parsed=True)

1215

1216 def _peek(self, n: int) -> str:

1217 """

1218 Peeks ahead n characters.

1219

1220 n is the max number of characters that will be peeked.

1221 """

1222 # we always want to restore after exiting this scope

1223 with self._state(restore=True):

1224 buf = ""

1225 for _ in range(n):

1226 if self._current not in " \t\n\r#,]}" + self._src.EOF:

1227 buf += self._current

1228 self.inc()

1229 continue

1230

1231 break

1232 return buf

1233

1234 def _peek_unicode(self, is_long: bool) -> tuple[str | None, str | None]:

1235 """

1236 Peeks ahead non-intrusively by cloning then restoring the

1237 initial state of the parser.

1238

1239 Returns the unicode value is it's a valid one else None.

1240 """

1241 # we always want to restore after exiting this scope

1242 with self._state(save_marker=True, restore=True):

1243 if self._current not in {"u", "U"}:

1244 raise self.parse_error(

1245 InternalParserError, "_peek_unicode() entered on non-unicode value"

1246 )

1247

1248 self.inc() # Dropping prefix

1249 self.mark()

1250

1251 if is_long:

1252 chars = 8

1253 else:

1254 chars = 4

1255

1256 if not self.inc_n(chars):

1257 value, extracted = None, None

1258 else:

1259 extracted = self.extract()

1260

1261 if extracted.strip("0123456789abcdefABCDEF"):

1262 return None, extracted

1263

1264 codepoint = int(extracted, 16)

1265

1266 # Unicode scalar values exclude the surrogate range

1267 # (U+D800 to U+DFFF). The 8-digit \U form reaches this range

1268 # with leading zeros, so it must be checked on the value itself.

1269 if 0xD800 <= codepoint <= 0xDFFF:

1270 return None, extracted

1271

1272 try:

1273 value = chr(codepoint)

1274 except (ValueError, OverflowError):

1275 value = None

1276

1277 return value, extracted

1278

1279 def _peek_hex(self) -> tuple[str | None, str | None]:

1280 with self._state(save_marker=True, restore=True):

1281 if self._current != "x":

1282 raise self.parse_error(

1283 InternalParserError, "_peek_hex() entered on non-hex value"

1284 )

1285

1286 self.inc() # Dropping prefix

1287 self.mark()

1288

1289 if not self.inc_n(2):

1290 return None, None

1291

1292 extracted = self.extract()

1293 if extracted.strip("0123456789abcdefABCDEF"):

1294 return None, None

1295

1296 try:

1297 value = chr(int(extracted, 16))

1298 except (ValueError, OverflowError):

1299 value = None

1300

1301 return value, extracted