package toml
import (
"bytes"
"fmt"
"math"
"strconv"
"time"
"github.com/pelletier/go-toml/v2/unstable"
)
func parseInteger(b []byte) (int64, error) {
if len(b) > 2 && b[0] == '0' {
switch b[1] {
case 'x':
return parseIntHex(b)
case 'b':
return parseIntBin(b)
case 'o':
return parseIntOct(b)
default:
panic(fmt.Errorf("invalid base '%c', should have been checked by scanIntOrFloat", b[1]))
}
}
return parseIntDec(b)
}
func parseIntHex(b []byte) (int64, error) {
var v uint64
for _, c := range b[2:] {
if c == '_' {
continue
}
var d byte
switch {
case c >= '0' && c <= '9':
d = c - '0'
case c >= 'a' && c <= 'f':
d = c - 'a' + 10
case c >= 'A' && c <= 'F':
d = c - 'A' + 10
}
if v > math.MaxInt64>>4 {
return 0, unstable.NewParserError(b, "hexadecimal number is too large to fit in a 64-bit signed integer")
}
v = v<<4 | uint64(d)
}
return int64(v), nil
}
func parseIntOct(b []byte) (int64, error) {
var v uint64
for _, c := range b[2:] {
if c == '_' {
continue
}
if v > math.MaxInt64>>3 {
return 0, unstable.NewParserError(b, "octal number is too large to fit in a 64-bit signed integer")
}
v = v<<3 | uint64(c-'0')
}
return int64(v), nil
}
func parseIntBin(b []byte) (int64, error) {
var v uint64
for _, c := range b[2:] {
if c == '_' {
continue
}
if v > math.MaxInt64>>1 {
return 0, unstable.NewParserError(b, "binary number is too large to fit in a 64-bit signed integer")
}
v = v<<1 | uint64(c-'0')
}
return int64(v), nil
}
func parseIntDec(b []byte) (int64, error) {
i := 0
neg := false
switch b[0] {
case '-':
neg = true
i++
case '+':
i++
}
var limit uint64 = math.MaxInt64
if neg {
limit = math.MaxInt64 + 1
}
var v uint64
for ; i < len(b); i++ {
c := b[i]
if c == '_' {
continue
}
if v > limit/10 {
return 0, unstable.NewParserError(b, "decimal number is too large to fit in a 64-bit signed integer")
}
v = v*10 + uint64(c-'0')
if v > limit {
return 0, unstable.NewParserError(b, "decimal number is too large to fit in a 64-bit signed integer")
}
}
if neg {
return -int64(v), nil //nolint:gosec // v <= MaxInt64+1, the conversion wraps to the intended negative value
}
return int64(v), nil //nolint:gosec // v <= MaxInt64
}
func parseFloat(b []byte) (float64, error) {
i := 0
if len(b) > 0 && (b[0] == '+' || b[0] == '-') {
i = 1
}
if len(b) == i+3 {
switch b[i] {
case 'i':
// inf
if b[0] == '-' {
return math.Inf(-1), nil
}
return math.Inf(1), nil
case 'n':
// nan
return math.NaN(), nil
}
}
// Fast path: a plain decimal whose significand fits in 53 bits and whose
// base-10 exponent is within [-22, 22] is parsed exactly with a single
// rounding (Clinger's method) straight from the bytes, with no string
// allocation and no full strconv parse. This is the common shape for
// numeric data (e.g. coordinate lists). Anything outside those bounds, or
// with underscores, falls through to strconv, which is the reference.
if f, ok := fastParseFloat(b); ok {
return f, nil
}
// strconv.ParseFloat is the reference implementation for parsing
// floating point numbers. The position of underscores has already been
// validated by the parser; strip them so that they do not interfere with
// Go's own underscore rules.
cleaned := b
if bytes.IndexByte(b, '_') >= 0 {
cleaned = make([]byte, 0, len(b))
for _, c := range b {
if c != '_' {
cleaned = append(cleaned, c)
}
}
}
f, err := strconv.ParseFloat(string(cleaned), 64)
if err != nil {
return 0, unstable.NewParserError(b, "unable to parse float: %s", err)
}
return f, nil
}
// float64pow10 holds the powers of ten that are exactly representable as a
// float64 (10^0 .. 10^22).
var float64pow10 = [...]float64{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22,
}
// fastParseFloat parses b as a float64 using Clinger's exact method and reports
// whether it applied. It accepts only plain decimal numbers (optional sign,
// digits, one optional '.', optional 'e'/'E' exponent) whose significand fits
// in 53 bits and whose effective base-10 exponent is within [-22, 22]; under
// those conditions float64(significand) * 10^exp (or / 10^-exp) is the exact,
// correctly-rounded result, identical to strconv.ParseFloat. It returns
// ok=false (deferring to strconv) for underscores, hexadecimal floats, large
// significands or exponents, and any other shape.
func fastParseFloat(b []byte) (float64, bool) {
i := 0
neg := false
if i < len(b) && (b[i] == '+' || b[i] == '-') {
neg = b[i] == '-'
i++
}
var mantissa uint64
digits := 0
fracDigits := 0
sawDot := false
sawDigit := false
for ; i < len(b); i++ {
c := b[i]
switch {
case c >= '0' && c <= '9':
if digits >= 19 {
// Too many significant digits to accumulate without risking a
// uint64 overflow (and well past the 53-bit exact range).
return 0, false
}
mantissa = mantissa*10 + uint64(c-'0')
digits++
if sawDot {
fracDigits++
}
sawDigit = true
case c == '.':
if sawDot {
return 0, false
}
sawDot = true
default:
goto exponent
}
}
exponent:
if !sawDigit {
return 0, false
}
exp := -fracDigits
if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
i++
esign := 1
if i < len(b) && (b[i] == '+' || b[i] == '-') {
if b[i] == '-' {
esign = -1
}
i++
}
if i >= len(b) {
return 0, false
}
eval := 0
for ; i < len(b); i++ {
c := b[i]
if c < '0' || c > '9' {
return 0, false
}
eval = eval*10 + int(c-'0')
if eval > 1000 {
return 0, false
}
}
exp += esign * eval
}
if i != len(b) {
// Trailing bytes (an underscore, a hexadecimal marker, ...).
return 0, false
}
if mantissa > 1<<53 {
return 0, false
}
f := float64(mantissa)
switch {
case exp == 0:
case exp > 0 && exp <= 22:
f *= float64pow10[exp]
case exp < 0 && exp >= -22:
f /= float64pow10[-exp]
default:
return 0, false
}
if neg {
f = -f
}
return f, true
}
func isDecimalDigit(c byte) bool {
return c >= '0' && c <= '9'
}
// parseLocalDate parses a date of the exact form YYYY-MM-DD and validates
// its components.
func parseLocalDate(b []byte) (LocalDate, error) {
var date LocalDate
if len(b) != 10 || b[4] != '-' || b[7] != '-' {
return date, unstable.NewParserError(b, "dates are expected to have the format YYYY-MM-DD")
}
var err error
date.Year, err = parseDecimalDigits(b[0:4])
if err != nil {
return date, err
}
date.Month, err = parseDecimalDigits(b[5:7])
if err != nil {
return date, err
}
date.Day, err = parseDecimalDigits(b[8:10])
if err != nil {
return date, err
}
if date.Month < 1 || date.Month > 12 {
return date, unstable.NewParserError(b[5:7], "impossible date")
}
maxDay := daysIn(date.Month, date.Year)
if date.Day < 1 || date.Day > maxDay {
return date, unstable.NewParserError(b[8:10], "impossible date")
}
return date, nil
}
func daysIn(month int, year int) int {
switch month {
case 2:
if isLeapYear(year) {
return 29
}
return 28
case 4, 6, 9, 11:
return 30
default:
return 31
}
}
func isLeapYear(year int) bool {
return year%4 == 0 && (year%100 != 0 || year%400 == 0)
}
// parseDecimalDigits parses a sequence of digits as a decimal number.
func parseDecimalDigits(b []byte) (int, error) {
v := 0
for i, c := range b {
if !isDecimalDigit(c) {
return 0, unstable.NewParserError(b[i:i+1], "expected digit (0-9)")
}
v = v*10 + int(c-'0')
}
return v, nil
}
// parseLocalTime parses a time of the form HH:MM with optional seconds and an
// optional fractional part (TOML v1.1.0). It returns the remaining bytes after
// the time.
func parseLocalTime(b []byte) (LocalTime, []byte, error) {
var (
nspow = [10]int{0, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 1e2, 1e1, 1e0}
t LocalTime
)
// check if b matches to have expected format HH:MM[:SS[.NNNNNN]]
const localTimeByteMinLen = 5
if len(b) < localTimeByteMinLen {
return t, nil, unstable.NewParserError(b, "times are expected to have the format HH:MM[:SS[.NNNNNN]]")
}
var err error
t.Hour, err = parseDecimalDigits(b[0:2])
if err != nil {
return t, nil, err
}
if t.Hour > 23 {
return t, nil, unstable.NewParserError(b[0:2], "hour cannot be greater 23")
}
if b[2] != ':' {
return t, nil, unstable.NewParserError(b[2:3], "expecting colon between hours and minutes")
}
t.Minute, err = parseDecimalDigits(b[3:5])
if err != nil {
return t, nil, err
}
if t.Minute > 59 {
return t, nil, unstable.NewParserError(b[3:5], "minutes cannot be greater 59")
}
b = b[5:]
// Seconds are optional (TOML v1.1.0). Fractional seconds may only appear
// when seconds are present:
// partial-time = time-hour ":" time-minute [ ":" time-second [ time-secfrac ] ]
secondsPresent := false
if len(b) >= 1 && b[0] == ':' {
if len(b) < 3 {
return t, nil, unstable.NewParserError(b, "incomplete seconds")
}
t.Second, err = parseDecimalDigits(b[1:3])
if err != nil {
return t, nil, err
}
if t.Second > 59 {
return t, nil, unstable.NewParserError(b[1:3], "seconds cannot be greater than 59")
}
b = b[3:]
secondsPresent = true
}
if secondsPresent && len(b) >= 1 && b[0] == '.' {
frac := 0
precision := 0
digits := 0
for i, c := range b[1:] {
if !isDecimalDigit(c) {
if i == 0 {
return t, nil, unstable.NewParserError(b[0:1], "need at least one digit after fraction point")
}
break
}
digits++
if i < 9 {
frac = frac*10 + int(c-'0')
precision++
}
}
if digits == 0 {
return t, nil, unstable.NewParserError(b[0:1], "need at least one digit after fraction point")
}
t.Nanosecond = frac * nspow[precision]
t.Precision = precision
return t, b[1+digits:], nil
}
return t, b, nil
}
// parseLocalDateTime parses a local date time of the form
// YYYY-MM-DD(T| )HH:MM:SS[.NNNNNN]. It returns the remaining bytes after the
// date-time.
func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) {
var dt LocalDateTime
const localDateTimeByteMinLen = 11
if len(b) < localDateTimeByteMinLen {
return dt, nil, unstable.NewParserError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM[:SS[.NNNNNNNNN]]")
}
date, err := parseLocalDate(b[:10])
if err != nil {
return dt, nil, err
}
dt.LocalDate = date
sep := b[10]
if sep != 'T' && sep != ' ' && sep != 't' {
return dt, nil, unstable.NewParserError(b[10:11], "datetime separator is expected to be T or a space")
}
t, rest, err := parseLocalTime(b[11:])
if err != nil {
return dt, nil, err
}
dt.LocalTime = t
return dt, rest, nil
}
// parseDateTime parses a date-time with a timezone offset (Z or +/-HH:MM).
func parseDateTime(b []byte) (time.Time, error) {
dt, b, err := parseLocalDateTime(b)
if err != nil {
return time.Time{}, err
}
var zone *time.Location
if len(b) == 0 {
// parser should have checked that there is a timezone
return time.Time{}, unstable.NewParserError(b, "date-time is missing timezone")
}
if b[0] == 'Z' || b[0] == 'z' {
b = b[1:]
zone = time.UTC
} else {
const dateTimeByteLen = 6
if len(b) != dateTimeByteLen {
return time.Time{}, unstable.NewParserError(b, "invalid date-time timezone")
}
var direction int
switch b[0] {
case '-':
direction = -1
case '+':
direction = +1
default:
return time.Time{}, unstable.NewParserError(b[:1], "invalid timezone offset character")
}
if b[3] != ':' {
return time.Time{}, unstable.NewParserError(b[3:4], "expected a : separator")
}
hours, err := parseDecimalDigits(b[1:3])
if err != nil {
return time.Time{}, err
}
if hours > 23 {
return time.Time{}, unstable.NewParserError(b[1:3], "invalid timezone offset hours")
}
minutes, err := parseDecimalDigits(b[4:6])
if err != nil {
return time.Time{}, err
}
if minutes > 59 {
return time.Time{}, unstable.NewParserError(b[4:6], "invalid timezone offset minutes")
}
seconds := direction * (hours*3600 + minutes*60)
if seconds == 0 {
zone = time.UTC
} else {
zone = time.FixedZone("", seconds)
}
b = b[dateTimeByteLen:]
}
if len(b) > 0 {
return time.Time{}, unstable.NewParserError(b, "extra bytes at the end of the timezone")
}
t := time.Date(
dt.Year,
time.Month(dt.Month),
dt.Day,
dt.Hour,
dt.Minute,
dt.Second,
dt.Nanosecond,
zone)
return t, nil
}
package toml
import (
"errors"
"reflect"
"strings"
"github.com/pelletier/go-toml/v2/internal/parserbridge"
"github.com/pelletier/go-toml/v2/unstable"
)
// unmarshalFused decodes a whole document into a native map[string]interface{}
// tree with no reflection on the document structure, and without building an
// AST for table headers and scalar key-values. Only container values (arrays
// and inline tables) are parsed into the parser arena, so that the seen-tracker
// can validate them and decodeAny can presize the resulting slices and maps —
// the AST is what makes that cheap O(1) presizing possible.
//
// It is used when the target is a fully generic value (interface{} or
// map[string]interface{}) and the unmarshaler interface is disabled. The
// seen-tracker validates the document (duplicate keys, type consistency), so
// the builder creates and merges containers without revalidating. Strict mode
// never applies to a generic target (a map has no "unknown fields"), and
// captures never apply (a generic value implements no Unmarshaler).
func (d *decoder) unmarshalFused(root reflect.Value, data []byte) error {
var m map[string]interface{}
if !root.IsNil() {
// Decode into (merge with) an existing generic map when present.
if em, ok := root.Interface().(map[string]interface{}); ok {
m = em
}
}
if m == nil {
m = map[string]interface{}{}
}
if err := d.fusedDocument(m, data); err != nil {
return d.wrapFusedError(data, err)
}
if root.CanSet() {
root.Set(reflect.ValueOf(m))
}
return nil
}
// fusedDocument runs the top-level expression loop, mirroring
// Parser.NextExpression but storing values directly into native maps.
func (d *decoder) fusedDocument(m map[string]interface{}, b []byte) error {
cur := m
for {
b = fusedSkipWS(b)
if len(b) == 0 {
return nil
}
switch b[0] {
case '\n':
b = b[1:]
case '\r':
if len(b) > 1 && b[1] == '\n' {
b = b[2:]
continue
}
return unstable.NewParserError(b[:1], "expected newline but got %#U", b[0])
case '#':
_, rest, err := parserbridge.ScanComment(b)
if err != nil {
return err
}
rest, err = fusedConsumeEOL(rest)
if err != nil {
return err
}
b = rest
case '[':
rest, err := d.fusedTable(b, m, &cur)
if err != nil {
return err
}
b = rest
default:
rest, err := d.fusedKeyVal(b, cur)
if err != nil {
return err
}
b = rest
}
}
}
// fusedTable handles a [table] or [[array table]] header. b starts at '['. It
// updates *cur to the table the following key-values belong to.
func (d *decoder) fusedTable(b []byte, root map[string]interface{}, cur *map[string]interface{}) ([]byte, error) {
arrayTable := len(b) > 1 && b[1] == '['
var start []byte
if arrayTable {
start = fusedSkipWS(b[2:])
} else {
start = fusedSkipWS(b[1:])
}
var err error
var rawKey []byte
d.keyParts, rawKey, b, err = parserbridge.ScanKey(&d.p, start, d.keyParts[:0])
if err != nil {
return nil, err
}
if arrayTable {
if len(b) < 2 || b[0] != ']' || b[1] != ']' {
return nil, unstable.NewParserError(fusedHL1(b), "expected ']]' to close array table name")
}
b = b[2:]
} else {
if len(b) == 0 || b[0] != ']' {
return nil, unstable.NewParserError(fusedHL1(b), "expected ']' to close table name")
}
b = b[1:]
}
// The whole expression (including its line termination) is parsed before
// it is validated, to keep error precedence identical to the AST path.
b, err = d.fusedFinishLine(b)
if err != nil {
return nil, err
}
if arrayTable {
first, err := d.seen.CheckArrayTable(d.keyParts)
if err != nil {
return nil, d.fusedSeenError(rawKey, d.keyParts, err)
}
*cur = d.anyArrayTableParts(root, d.keyParts, first)
} else {
if _, err := d.seen.CheckTable(d.keyParts); err != nil {
return nil, d.fusedSeenError(rawKey, d.keyParts, err)
}
*cur = d.anyTableParts(root, d.keyParts)
}
return b, nil
}
// fusedKeyVal handles a `key = value` expression relative to the current table
// cur. b starts at the first character of the key.
func (d *decoder) fusedKeyVal(b []byte, cur map[string]interface{}) ([]byte, error) {
var err error
var rawKey []byte
d.keyParts, rawKey, b, err = parserbridge.ScanKey(&d.p, b, d.keyParts[:0])
if err != nil {
return nil, err
}
if len(b) == 0 || b[0] != '=' {
return nil, unstable.NewParserError(fusedHL1(b), "expected '=' after key")
}
b = fusedSkipWS(b[1:])
if len(b) == 0 {
return nil, unstable.NewParserError(b, "expected value, not end of input")
}
if c := b[0]; c == '[' || c == '{' {
// Container value: build its AST so the seen-tracker can validate it
// and decodeAny can presize the resulting slices and maps.
nodeAny, rest, err := parserbridge.ParseValue(&d.p, b)
if err != nil {
return nil, err
}
node := nodeAny.(*unstable.Node)
rest, err = d.fusedFinishLine(rest)
if err != nil {
return nil, err
}
leafID, err := d.seen.CheckKeyValue(d.keyParts)
if err != nil {
return nil, d.fusedSeenError(rawKey, d.keyParts, err)
}
if err := d.seen.CheckValueUnder(leafID, node); err != nil {
return nil, d.fusedSeenError(rawKey, d.keyParts, err)
}
av, err := d.decodeAny(node)
if err != nil {
return nil, err
}
d.setFusedLeaf(cur, d.keyParts, av)
return rest, nil
}
// Scalar value: scan it without building a node, then validate and convert
// it natively.
k, _, value, rest, err := parserbridge.ScanScalar(&d.p, b)
if err != nil {
return nil, err
}
kind := unstable.Kind(k)
rest, err = d.fusedFinishLine(rest)
if err != nil {
return nil, err
}
if _, err := d.seen.CheckKeyValue(d.keyParts); err != nil {
return nil, d.fusedSeenError(rawKey, d.keyParts, err)
}
av, err := d.fusedScalar(kind, value)
if err != nil {
return nil, err
}
d.setFusedLeaf(cur, d.keyParts, av)
return rest, nil
}
// fusedSeenError turns a bare error returned by a SeenTracker parts-method
// into a ParserError carrying the position (the raw key span) and key path of
// the offending expression, so that it is reported as a DecodeError with
// context. It mirrors decoder.wrapSeenError for the fused (AST-less) path.
func (d *decoder) fusedSeenError(rawKey []byte, parts [][]byte, err error) error {
key := make(Key, len(parts))
for i, p := range parts {
key[i] = string(p)
}
return &unstable.ParserError{
Highlight: rawKey,
Message: strings.TrimPrefix(err.Error(), "toml: "),
Key: key,
}
}
// fusedScalar converts a scanned scalar value into the native Go value used
// for generic targets. It mirrors the scalar cases of decodeAny.
func (d *decoder) fusedScalar(kind unstable.Kind, value []byte) (interface{}, error) {
switch kind {
case unstable.String:
return string(value), nil
case unstable.Integer:
i, err := parseInteger(value)
return i, err
case unstable.Float:
f, err := parseFloat(value)
return f, err
case unstable.Bool:
return value[0] == 't', nil
case unstable.DateTime:
t, err := parseDateTime(value)
return t, err
case unstable.LocalDateTime:
dt, rest, err := parseLocalDateTime(value)
if err != nil {
return nil, err
}
if len(rest) > 0 {
return nil, unstable.NewParserError(rest, "extra characters at the end of a local date time")
}
return dt, nil
case unstable.LocalDate:
date, err := parseLocalDate(value)
return date, err
case unstable.LocalTime:
t, rest, err := parseLocalTime(value)
if err != nil {
return nil, err
}
if len(rest) > 0 {
return nil, unstable.NewParserError(rest, "extra characters at the end of a local time")
}
return t, nil
default:
return nil, unstable.NewParserError(value, "unsupported value kind %s", kind)
}
}
// anyTableParts navigates a [table] header (given its key parts) to the map it
// designates, creating intermediate tables as needed.
func (d *decoder) anyTableParts(m map[string]interface{}, parts [][]byte) map[string]interface{} {
cur := m
for _, p := range parts {
cur = d.anyChildTable(cur, d.intern(p))
}
return cur
}
// anyArrayTableParts navigates a [[array table]] header (given its key parts),
// appends a fresh element to the designated array, and returns it. first is
// true the first time this header is seen, in which case any pre-existing array
// (from a reused target) is reset.
func (d *decoder) anyArrayTableParts(m map[string]interface{}, parts [][]byte, first bool) map[string]interface{} {
cur := m
name := d.intern(parts[0])
for i := 1; i < len(parts); i++ {
cur = d.anyChildTable(cur, name)
name = d.intern(parts[i])
}
s, _ := cur[name].([]interface{})
if first {
s = s[:0]
}
elem := map[string]interface{}{}
cur[name] = append(s, elem)
return elem
}
// setFusedLeaf assigns av at the (possibly dotted) key parts within cur,
// creating intermediate maps as needed.
func (d *decoder) setFusedLeaf(cur map[string]interface{}, parts [][]byte, av interface{}) {
for i := 0; i < len(parts)-1; i++ {
cur = d.anyChildTable(cur, d.intern(parts[i]))
}
cur[d.intern(parts[len(parts)-1])] = av
}
// wrapFusedError gives document context to errors produced by the fused
// decoder.
func (d *decoder) wrapFusedError(data []byte, err error) error {
var perr *unstable.ParserError
if errors.As(err, &perr) && len(perr.Highlight) == 0 {
// Mirror NextExpression: give end-of-input errors a usable position by
// extending the empty highlight to the last byte of the document.
if offset := cap(data) - cap(perr.Highlight); offset > 0 && offset == len(data) {
perr.Highlight = data[offset-1 : offset]
}
}
return d.wrapError(data, err)
}
func fusedSkipWS(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}
func fusedConsumeEOL(b []byte) ([]byte, error) {
if len(b) == 0 {
return b, nil
}
switch b[0] {
case '\n':
return b[1:], nil
case '\r':
if len(b) > 1 && b[1] == '\n' {
return b[2:], nil
}
}
return nil, unstable.NewParserError(b[:1], "expected newline but got %#U", b[0])
}
// fusedFinishLine consumes `ws [comment] (newline|eof)` after an expression.
func (d *decoder) fusedFinishLine(b []byte) ([]byte, error) {
b = fusedSkipWS(b)
if len(b) > 0 && b[0] == '#' {
_, rest, err := parserbridge.ScanComment(b)
if err != nil {
return nil, err
}
b = rest
}
return fusedConsumeEOL(b)
}
func fusedHL1(b []byte) []byte {
if len(b) > 0 {
return b[:1]
}
return b
}
package toml
import (
"errors"
"strconv"
"strings"
"github.com/pelletier/go-toml/v2/unstable"
)
// DecodeError represents an error encountered during the parsing or decoding
// of a TOML document.
//
// In addition to the error message, it contains the position in the document
// where it happened, as well as a human-readable representation that shows
// where the error occurred in the document.
type DecodeError struct {
message string
line int
column int
key Key
human string
}
// StrictMissingError occurs in a TOML document that does not have a
// corresponding field in the target value. It contains all the missing fields
// in Errors.
//
// Emitted by Decoder when DisallowUnknownFields() was called.
type StrictMissingError struct {
// One error per field that could not be found.
Errors []DecodeError
}
// Error returns the canonical string for this error.
func (s *StrictMissingError) Error() string {
return "strict mode: fields in the document are missing in the target struct"
}
// String returns a human readable description of all errors.
func (s *StrictMissingError) String() string {
var buf strings.Builder
for i, e := range s.Errors {
if i > 0 {
buf.WriteString("\n---\n")
}
buf.WriteString(e.String())
}
return buf.String()
}
// Unwrap returns wrapped decode errors
//
// Implements errors.Join() interface.
func (s *StrictMissingError) Unwrap() []error {
errs := make([]error, len(s.Errors))
for i := range s.Errors {
errs[i] = &s.Errors[i]
}
return errs
}
// Key represents a TOML key as a sequence of key parts.
type Key []string
// Error returns the error message contained in the DecodeError.
func (e *DecodeError) Error() string {
return "toml: " + e.message
}
// String returns the human-readable contextualized error. This string is
// multi-line.
func (e *DecodeError) String() string {
return e.human
}
// Position returns the (line, column) pair indicating where the error
// occurred in the document. Positions are 1-indexed.
func (e *DecodeError) Position() (row int, column int) {
return e.line, e.column
}
// Key that was being processed when the error occurred.
func (e *DecodeError) Key() Key {
return e.key
}
// wrapDecodeError creates a DecodeError from a ParserError. The highlight of
// the ParserError needs to be a subslice of the document.
func wrapDecodeError(document []byte, de *unstable.ParserError) *DecodeError {
if de == nil {
return nil
}
return newDecodeError(document, de.Highlight, de.Key, de.Message)
}
// newDecodeError creates a DecodeError pointing at the given highlight, which
// needs to be a subslice of the document.
func newDecodeError(document []byte, highlight []byte, key Key, message string) *DecodeError {
offset := subsliceOffset(document, highlight)
errLineIdx, errColumn := positionAt(document, offset)
human := buildHumanContext(document, errLineIdx, errColumn, len(highlight), message)
return &DecodeError{
message: message,
line: errLineIdx + 1,
column: errColumn,
key: key,
human: human,
}
}
// subsliceOffset returns the offset of the subslice b within the document.
func subsliceOffset(document, b []byte) int {
// Highlights are subslices of the document, which means they share the
// same backing array, and their capacity counts the bytes between their
// start and the end of the backing array.
offset := cap(document) - cap(b)
if offset < 0 || offset+len(b) > len(document) {
panic(errors.New("highlight is not a subslice of the document"))
}
return offset
}
// positionAt returns the 0-indexed line and the 1-indexed column of the given
// offset in the document.
func positionAt(document []byte, offset int) (lineIdx int, column int) {
lineStart := 0
for i := 0; i < offset; i++ {
if document[i] == '\n' {
lineIdx++
lineStart = i + 1
}
}
return lineIdx, offset - lineStart + 1
}
// docLines splits the document into lines, removing the trailing newline
// characters.
func docLines(document []byte) []string {
s := string(document)
lines := strings.Split(s, "\n")
for i, l := range lines {
lines[i] = strings.TrimSuffix(l, "\r")
}
return lines
}
// buildHumanContext renders the human-readable multi-line context of an
// error: a window of up to 3 lines before and after the error line, with
// the error position underlined.
func buildHumanContext(document []byte, errLineIdx, errColumn, highlightLen int, message string) string {
lines := docLines(document)
const window = 3
firstIdx := errLineIdx - window
if firstIdx < 0 {
firstIdx = 0
}
lastIdx := errLineIdx + window
if lastIdx > len(lines)-1 {
lastIdx = len(lines) - 1
}
// Empty lines at the edges of the window are dropped, unless the error
// is about that very position.
for firstIdx < errLineIdx && lines[firstIdx] == "" {
firstIdx++
}
for lastIdx > errLineIdx && lines[lastIdx] == "" {
lastIdx--
}
// Width of the column of line numbers.
width := len(strconv.Itoa(lastIdx + 1))
var buf strings.Builder
writeLine := func(idx int) {
number := strconv.Itoa(idx + 1)
for i := len(number); i < width; i++ {
buf.WriteByte(' ')
}
buf.WriteString(number)
buf.WriteByte('|')
if len(lines[idx]) > 0 {
buf.WriteByte(' ')
buf.WriteString(lines[idx])
}
buf.WriteByte('\n')
}
for idx := firstIdx; idx <= errLineIdx; idx++ {
writeLine(idx)
}
// Underline the error.
for i := 0; i < width; i++ {
buf.WriteByte(' ')
}
buf.WriteString("| ")
for i := 1; i < errColumn; i++ {
buf.WriteByte(' ')
}
// The highlight cannot extend past the end of its line.
tildes := highlightLen
if errLineIdx < len(lines) {
if avail := len(lines[errLineIdx]) - errColumn + 1; tildes > avail {
tildes = avail
}
}
if tildes < 1 {
tildes = 1
}
for i := 0; i < tildes; i++ {
buf.WriteByte('~')
}
if message != "" {
buf.WriteByte(' ')
buf.WriteString(message)
}
buf.WriteByte('\n')
for idx := errLineIdx + 1; idx <= lastIdx; idx++ {
writeLine(idx)
}
return strings.TrimSuffix(buf.String(), "\n")
}
package tracker
import "github.com/pelletier/go-toml/v2/unstable"
// KeyTracker is a tracker that keeps track of the current Key as the AST is
// walked.
type KeyTracker struct {
k []string
}
// UpdateTable sets the state of the tracker with the AST table node.
func (t *KeyTracker) UpdateTable(node *unstable.Node) {
t.reset()
t.Push(node)
}
// UpdateArrayTable sets the state of the tracker with the AST array table
// node.
func (t *KeyTracker) UpdateArrayTable(node *unstable.Node) {
t.reset()
t.Push(node)
}
// Push the given key on the stack.
func (t *KeyTracker) Push(node *unstable.Node) {
it := node.Key()
for it.Next() {
t.k = append(t.k, string(it.Node().Data))
}
}
// Pop key from stack.
func (t *KeyTracker) Pop(node *unstable.Node) {
it := node.Key()
for it.Next() {
t.k = t.k[:len(t.k)-1]
}
}
// Key returns the current key.
func (t *KeyTracker) Key() []string {
k := make([]string, len(t.k))
copy(k, t.k)
return k
}
func (t *KeyTracker) reset() {
t.k = t.k[:0]
}
package tracker
import (
"bytes"
"fmt"
"github.com/pelletier/go-toml/v2/unstable"
)
type keyKind uint8
const (
invalidKind keyKind = iota
// valueKind is a regular value (scalar, array, or inline table). It
// cannot be extended.
valueKind
// kvTableKind is a table created implicitly by a dotted key. It can only
// be extended by other dotted keys.
kvTableKind
// tableKind is a table created by a [header]. The explicit flag tells
// whether the table was created by its own header (true) or as an
// intermediate step of a longer key (false).
tableKind
// arrayTableKind is an array of tables created by [[header]].
arrayTableKind
// anonymousKind is an entry that cannot be looked up by name. It serves
// as the parent of the content of inline tables stored inside arrays.
anonymousKind
)
func (k keyKind) String() string {
switch k {
case invalidKind:
return "invalid"
case valueKind:
return "value"
case kvTableKind:
return "kv-table"
case tableKind:
return "table"
case arrayTableKind:
return "array-table"
case anonymousKind:
return "anonymous"
}
panic("missing keyKind string mapping")
}
// entry represents a node that has been seen in the document. Its size has a
// direct impact on the performance of unmarshaling documents: keep it as
// small as possible.
type entry struct {
parent int32
kind keyKind
explicit bool
name []byte
}
// SeenTracker tracks which keys have been seen with which TOML type to flag
// duplicates and mismatches according to the spec.
//
// Each node in the visited tree is represented by an entry. Each entry has
// an identifier, which is provided by a counter. Entries are stored in the
// array entries. As new nodes are discovered (referenced for the first time
// in the TOML document), entries are created and appended to the array. An
// entry points to its parent using its id.
//
// To find whether a given key (sequence of []byte) has already been visited,
// the entries are linearly searched, looking for one with the right name and
// parent id.
//
// Given that all keys appear in the document after their parent, it is
// guaranteed that all descendants of a node are stored after the node, this
// speeds up the search process.
//
// When encountering [[array tables]], the descendants of that node are removed
// to allow that branch of the tree to be "rediscovered". To maintain the
// invariant above, the deletion process needs to keep the order of entries.
// This results in more copies in that case.
type SeenTracker struct {
entries []entry
currentTable int32
// scratch buffers for clear()
removedBuf []bool
remapBuf []int32
}
// Reset brings the tracker to its initial state, with just a root table, so
// that it can be reused across documents.
func (s *SeenTracker) Reset() {
s.reset()
}
// reset brings the tracker to its initial state, with just a root table.
func (s *SeenTracker) reset() {
s.entries = append(s.entries[:0], entry{
parent: -1,
kind: tableKind,
})
s.currentTable = 0
}
// find returns the id of the entry with the given parent and name, or -1.
// Anonymous entries are never returned.
func (s *SeenTracker) find(parent int32, name []byte) int32 {
// Children always appear after their parent.
for i := int(parent) + 1; i < len(s.entries); i++ {
e := &s.entries[i]
if e.parent == parent && e.kind != anonymousKind && bytes.Equal(e.name, name) {
return int32(i) //nolint:gosec // entry counts are bounded by document size
}
}
return -1
}
// create appends a new entry and returns its id.
func (s *SeenTracker) create(parent int32, name []byte, kind keyKind, explicit bool) int32 {
id := int32(len(s.entries)) //nolint:gosec // entry counts are bounded by document size
s.entries = append(s.entries, entry{
parent: parent,
kind: kind,
explicit: explicit,
name: name,
})
return id
}
// clear removes all the descendants of the entry with the given id, keeping
// the order of the remaining entries.
func (s *SeenTracker) clear(id int32) {
// Compute which entries are removed. Given that children always appear
// after their parent, a single forward pass is enough.
if cap(s.removedBuf) < len(s.entries) {
s.removedBuf = make([]bool, len(s.entries))
s.remapBuf = make([]int32, len(s.entries))
}
removed := s.removedBuf[:len(s.entries)]
remap := s.remapBuf[:len(s.entries)]
for i := range removed {
removed[i] = false
}
n := int32(0)
for i := 0; i < len(s.entries); i++ {
parent := s.entries[i].parent
if parent >= 0 && (parent == id && s.entries[i].kind != invalidKind || removed[parent]) {
removed[i] = true
continue
}
remap[i] = n
if int32(i) != n { //nolint:gosec // entry counts are bounded by document size
e := s.entries[i]
e.parent = remap[e.parent]
s.entries[n] = e
}
n++
}
s.entries = s.entries[:n]
}
// CheckExpression takes a top-level node and checks that it does not contain
// keys that have been seen in previous calls, and validates that types are
// consistent. It returns true if it is the first time this node's key is
// seen. Useful to clear array tables on first use.
func (s *SeenTracker) CheckExpression(node *unstable.Node) (bool, error) {
if len(s.entries) == 0 {
s.reset()
}
switch node.Kind {
case unstable.KeyValue:
return false, s.checkKeyValue(s.currentTable, node)
case unstable.Table:
return s.checkTable(node)
case unstable.ArrayTable:
return s.checkArrayTable(node)
default:
return false, fmt.Errorf("toml: unexpected expression kind %s", node.Kind)
}
}
// CheckTable validates a [table] header given the decoded parts of its key.
// It mirrors checkTable but is driven directly from the key parts instead of
// an AST, for callers that decode without building one. It returns whether the
// table is seen for the first time.
func (s *SeenTracker) CheckTable(parts [][]byte) (bool, error) {
parent := int32(0)
for k := 0; k < len(parts); k++ {
name := parts[k]
if k == len(parts)-1 {
// Final part of the key.
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, true)
s.currentTable = i
return true, nil
}
e := &s.entries[i]
switch e.kind {
case tableKind:
if e.explicit {
return false, fmt.Errorf("toml: table %s already exists", name)
}
e.explicit = true
s.currentTable = i
return false, nil
case kvTableKind:
return false, fmt.Errorf("toml: table %s already exists as defined by a dotted key", name)
case arrayTableKind:
return false, fmt.Errorf("toml: table %s already exists as an array of tables", name)
default:
return false, fmt.Errorf("toml: key %s should be a table, not a %s", name, e.kind)
}
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, false)
} else {
switch s.entries[i].kind {
case tableKind, arrayTableKind, kvTableKind:
// Tables created by dotted keys can receive new sub-tables,
// but cannot be redefined (handled by the last-part case).
default:
return false, fmt.Errorf("toml: key %s already exists as a value", name)
}
}
parent = i
}
panic("unreachable: table expression without key")
}
// CheckArrayTable validates a [[array table]] header given the decoded parts
// of its key. It mirrors checkArrayTable but is driven directly from the key
// parts. It returns whether the array table is seen for the first time.
func (s *SeenTracker) CheckArrayTable(parts [][]byte) (bool, error) {
parent := int32(0)
for k := 0; k < len(parts); k++ {
name := parts[k]
if k == len(parts)-1 {
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, arrayTableKind, true)
s.currentTable = i
return true, nil
}
if s.entries[i].kind != arrayTableKind {
return false, fmt.Errorf("toml: key %s already exists as a %s, but should be an array table", name, s.entries[i].kind)
}
// Make the descendants of this array table re-discoverable for
// the new element.
s.clear(i)
s.currentTable = i
return false, nil
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, false)
} else {
switch s.entries[i].kind {
case tableKind, arrayTableKind, kvTableKind:
// Tables created by dotted keys can receive new sub-tables,
// but cannot be redefined (handled by the last-part case).
default:
return false, fmt.Errorf("toml: key %s already exists as a value", name)
}
}
parent = i
}
panic("unreachable: array table expression without key")
}
// CheckKeyValue validates the (possibly dotted) key of a key-value under the
// current table, WITHOUT validating its value. It returns the id of the leaf
// entry, so the caller can validate a container value with CheckValueUnder.
func (s *SeenTracker) CheckKeyValue(parts [][]byte) (int32, error) {
parent := s.currentTable
for k := 0; k < len(parts); k++ {
name := parts[k]
if k == len(parts)-1 {
if i := s.find(parent, name); i >= 0 {
return -1, fmt.Errorf("toml: key %s is already defined", name)
}
return s.create(parent, name, valueKind, false), nil
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, kvTableKind, false)
} else if s.entries[i].kind != kvTableKind {
return -1, fmt.Errorf("toml: key %s is already defined", name)
}
parent = i
}
panic("unreachable: key-value expression without key")
}
// CheckValueUnder validates the content of a value stored under the given
// entry (typically the leaf returned by CheckKeyValue): inline tables cannot
// contain duplicate keys, including in the inline tables and arrays they
// contain.
func (s *SeenTracker) CheckValueUnder(parent int32, value *unstable.Node) error {
return s.checkValue(parent, value)
}
func (s *SeenTracker) checkTable(node *unstable.Node) (bool, error) {
parent := int32(0)
it := node.Key()
// Handle the intermediate parts of the key.
for it.Next() {
part := it.Node()
name := part.Data
if it.IsLast() {
// Final part of the key.
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, true)
s.currentTable = i
return true, nil
}
e := &s.entries[i]
switch e.kind {
case tableKind:
if e.explicit {
return false, fmt.Errorf("toml: table %s already exists", name)
}
e.explicit = true
s.currentTable = i
return false, nil
case kvTableKind:
return false, fmt.Errorf("toml: table %s already exists as defined by a dotted key", name)
case arrayTableKind:
return false, fmt.Errorf("toml: table %s already exists as an array of tables", name)
default:
return false, fmt.Errorf("toml: key %s should be a table, not a %s", name, e.kind)
}
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, false)
} else {
switch s.entries[i].kind {
case tableKind, arrayTableKind, kvTableKind:
// Tables created by dotted keys can receive new sub-tables,
// but cannot be redefined (handled by the last-part case).
default:
return false, fmt.Errorf("toml: key %s already exists as a value", name)
}
}
parent = i
}
panic("unreachable: table expression without key")
}
func (s *SeenTracker) checkArrayTable(node *unstable.Node) (bool, error) {
parent := int32(0)
it := node.Key()
for it.Next() {
part := it.Node()
name := part.Data
if it.IsLast() {
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, arrayTableKind, true)
s.currentTable = i
return true, nil
}
if s.entries[i].kind != arrayTableKind {
return false, fmt.Errorf("toml: key %s already exists as a %s, but should be an array table", name, s.entries[i].kind)
}
// Make the descendants of this array table re-discoverable for
// the new element.
s.clear(i)
// Note: clear cannot move i because i comes before all its
// descendants.
s.currentTable = i
return false, nil
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, tableKind, false)
} else {
switch s.entries[i].kind {
case tableKind, arrayTableKind, kvTableKind:
// Tables created by dotted keys can receive new sub-tables,
// but cannot be redefined (handled by the last-part case).
default:
return false, fmt.Errorf("toml: key %s already exists as a value", name)
}
}
parent = i
}
panic("unreachable: array table expression without key")
}
func (s *SeenTracker) checkKeyValue(parent int32, node *unstable.Node) error {
it := node.Key()
for it.Next() {
part := it.Node()
name := part.Data
if it.IsLast() {
if i := s.find(parent, name); i >= 0 {
return fmt.Errorf("toml: key %s is already defined", name)
}
id := s.create(parent, name, valueKind, false)
return s.checkValue(id, node.Value())
}
i := s.find(parent, name)
if i < 0 {
i = s.create(parent, name, kvTableKind, false)
} else if s.entries[i].kind != kvTableKind {
return fmt.Errorf("toml: key %s is already defined", name)
}
parent = i
}
panic("unreachable: key-value expression without key")
}
// checkValue verifies the content of a value: inline tables cannot contain
// duplicate keys, including in the inline tables and arrays they contain.
func (s *SeenTracker) checkValue(id int32, value *unstable.Node) error {
switch value.Kind {
case unstable.InlineTable:
it := value.Children()
for it.Next() {
if err := s.checkKeyValue(id, it.Node()); err != nil {
return err
}
}
case unstable.Array:
it := value.Children()
for it.Next() {
elem := it.Node()
if elem.Kind == unstable.InlineTable || elem.Kind == unstable.Array {
elemID := s.create(id, nil, anonymousKind, false)
if err := s.checkValue(elemID, elem); err != nil {
return err
}
}
}
default:
}
return nil
}
package toml
import (
"fmt"
"strings"
"time"
"github.com/pelletier/go-toml/v2/unstable"
)
// LocalDate represents a calendar day in no specific timezone.
type LocalDate struct {
Year int
Month int
Day int
}
// AsTime converts d into a specific time instance at midnight in zone.
func (d LocalDate) AsTime(zone *time.Location) time.Time {
return time.Date(d.Year, time.Month(d.Month), d.Day, 0, 0, 0, 0, zone)
}
// String returns RFC 3339 representation of d.
func (d LocalDate) String() string {
return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}
// MarshalText returns RFC 3339 representation of d.
func (d LocalDate) MarshalText() ([]byte, error) {
return []byte(d.String()), nil
}
// UnmarshalText parses b using RFC 3339 to fill d.
func (d *LocalDate) UnmarshalText(b []byte) error {
res, err := parseLocalDate(b)
if err != nil {
return err
}
*d = res
return nil
}
// LocalTime represents a time of day of no specific day in no specific
// timezone.
type LocalTime struct {
Hour int // Hour of the day: [0; 24[
Minute int // Minute of the hour: [0; 60[
Second int // Second of the minute: [0; 59]
Nanosecond int // Nanoseconds within the second: [0, 1000000000[
Precision int // Number of digits to display for Nanosecond.
}
// String returns RFC 3339 representation of d.
// If d.Nanosecond and d.Precision are zero, the time won't have a nanosecond
// component. If d.Nanosecond > 0 but d.Precision = 0, then the minimum number
// of digits for nanoseconds is provided.
func (d LocalTime) String() string {
s := fmt.Sprintf("%02d:%02d:%02d", d.Hour, d.Minute, d.Second)
if d.Precision > 0 {
s += fmt.Sprintf(".%09d", d.Nanosecond)[:d.Precision+1]
} else if d.Nanosecond > 0 {
// Nanoseconds are specified, but precision is not provided. Use the
// minimum.
s += strings.TrimRight(fmt.Sprintf(".%09d", d.Nanosecond), "0")
}
return s
}
// MarshalText returns RFC 3339 representation of d.
func (d LocalTime) MarshalText() ([]byte, error) {
return []byte(d.String()), nil
}
// UnmarshalText parses b using RFC 3339 to fill d.
func (d *LocalTime) UnmarshalText(b []byte) error {
res, left, err := parseLocalTime(b)
if err == nil && len(left) != 0 {
err = unstable.NewParserError(left, "extra characters at the end of a local time")
}
if err != nil {
return err
}
*d = res
return nil
}
// LocalDateTime represents a time of a specific day in no specific timezone.
type LocalDateTime struct {
LocalDate
LocalTime
}
// AsTime converts d into a specific time instance in zone.
func (d LocalDateTime) AsTime(zone *time.Location) time.Time {
return time.Date(d.Year, time.Month(d.Month), d.Day, d.Hour, d.Minute, d.Second, d.Nanosecond, zone)
}
// String returns RFC 3339 representation of d.
func (d LocalDateTime) String() string {
return d.LocalDate.String() + "T" + d.LocalTime.String()
}
// MarshalText returns RFC 3339 representation of d.
func (d LocalDateTime) MarshalText() ([]byte, error) {
return []byte(d.String()), nil
}
// UnmarshalText parses b using RFC 3339 to fill d.
func (d *LocalDateTime) UnmarshalText(data []byte) error {
res, left, err := parseLocalDateTime(data)
if err == nil && len(left) != 0 {
err = unstable.NewParserError(left, "extra characters at the end of a local date time")
}
if err != nil {
return err
}
*d = res
return nil
}
package toml
import (
"bytes"
"encoding"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
)
// Marshal serializes a Go value as a TOML document.
//
// It is a shortcut for Encoder.Encode() with the default options.
func Marshal(v interface{}) ([]byte, error) {
enc := Encoder{indentSymbol: " "}
e := encoderStatePool.Get().(*encoderState)
e.Encoder = &enc
e.buf = e.buf[:0]
e.keyStack = e.keyStack[:0]
e.lastWasHeader = false
err := e.encodeRoot(v)
if err != nil {
encoderStatePool.Put(e)
return nil, err
}
out := make([]byte, len(e.buf))
copy(out, e.buf)
encoderStatePool.Put(e)
return out, nil
}
// Encoder writes a TOML document to an output stream.
type Encoder struct {
// output
w io.Writer
// global settings
tablesInline bool
arraysMultiline bool
indentSymbol string
indentTables bool
marshalJSONNumbers bool
}
// NewEncoder returns a new Encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: w,
indentSymbol: " ",
}
}
// SetTablesInline forces the encoder to emit all tables inline.
//
// This behavior can be controlled on an individual struct field basis with
// the inline tag:
//
// MyField `toml:",inline"`
func (enc *Encoder) SetTablesInline(inline bool) *Encoder {
enc.tablesInline = inline
return enc
}
// SetArraysMultiline forces the encoder to emit all arrays with one element
// per line.
//
// This behavior can be controlled on an individual struct field basis with
// the multiline tag:
//
// MyField `multiline:"true"`
func (enc *Encoder) SetArraysMultiline(multiline bool) *Encoder {
enc.arraysMultiline = multiline
return enc
}
// SetIndentSymbol defines the string that should be used for indentation. The
// provided string is repeated for each indentation level. Defaults to two
// spaces.
func (enc *Encoder) SetIndentSymbol(s string) *Encoder {
enc.indentSymbol = s
return enc
}
// SetIndentTables forces the encoder to intent tables and array tables.
func (enc *Encoder) SetIndentTables(indent bool) *Encoder {
enc.indentTables = indent
return enc
}
// SetMarshalJSONNumbers forces the encoder to serialize `json.Number` as a
// float or integer instead of relying on TextMarshaler to emit a string.
//
// *Unstable:* This method does not follow the compatibility guarantees of
// semver. It can be changed or removed without a new major version being
// issued.
func (enc *Encoder) SetMarshalJSONNumbers(indent bool) *Encoder {
enc.marshalJSONNumbers = indent
return enc
}
// Encode writes a TOML representation of v to the stream.
//
// If v cannot be represented to TOML it returns an error.
//
// # Encoding rules
//
// A top level slice containing only maps or structs is encoded as [[table
// array]].
//
// All slices not matching rule 1 are encoded as [array]. As a result, any map
// or struct they contain is encoded as an {inline table}.
//
// Nil interfaces and nil pointers are not supported.
//
// Keys in key-values always have one part.
//
// Intermediate tables are always printed.
//
// By default, strings are encoded as literal string, unless they contain
// either a newline character or a single quote. In that case they are emitted
// as quoted strings.
//
// Unsigned integers larger than math.MaxInt64 cannot be encoded. Doing so
// results in an error. This rule exists because the TOML specification only
// requires parsers to support at least the 64 bits integer range. Allowing
// larger numbers would create non-standard TOML documents, which may not be
// readable (at best) by other implementations. To encode such numbers, a
// solution is a custom type that implements encoding.TextMarshaler.
//
// When encoding structs, fields are encoded in order of definition, with
// their exact name.
//
// Tables and array tables are separated by empty lines. However, consecutive
// subtables definitions are not. For example:
//
// [top1]
//
// [top2]
// [top2.child1]
//
// [[array]]
//
// [[array]]
// [array.child2]
//
// # Struct tags
//
// The encoding of each public struct field can be customized by the format
// string in the "toml" key of the struct field's tag. This follows
// encoding/json's convention. The format string starts with the name of the
// field, optionally followed by a comma-separated list of options. The name
// may be empty in order to provide options without overriding the default
// name.
//
// The "multiline" option emits strings as quoted multi-line TOML strings, and
// arrays with one element per line. For strings, it only takes effect when the
// value contains a newline; single-line values are emitted as regular strings.
// It has no effect on fields that would not be encoded as strings or arrays.
//
// The "inline" option turns fields that would be emitted as tables into
// inline tables instead. It has no effect on other fields.
//
// The "omitempty" option prevents empty values or groups from being emitted.
//
// The "omitzero" option prevents zero values or groups from being emitted.
//
// The "commented" option prefixes the value and all its children with a
// comment symbol.
//
// In addition to the "toml" tag struct tag, a "comment" tag can be used to
// emit a TOML comment before the value being annotated. Comments are ignored
// inside inline tables. For array tables, the comment is only present before
// the first element of the array.
func (enc *Encoder) Encode(v interface{}) error {
e := encoderStatePool.Get().(*encoderState)
e.Encoder = enc
e.buf = e.buf[:0]
e.keyStack = e.keyStack[:0]
e.lastWasHeader = false
err := e.encodeRoot(v)
if err != nil {
encoderStatePool.Put(e)
return err
}
_, err = enc.w.Write(e.buf)
encoderStatePool.Put(e)
if err != nil {
return fmt.Errorf("toml: cannot write: %w", err)
}
return nil
}
var encoderStatePool = sync.Pool{
New: func() interface{} { return &encoderState{} },
}
type encoderState struct {
*Encoder
buf []byte
// keyStack is the dotted key of the table being encoded, shared by the
// whole encode as a stack.
keyStack []string
// entriesPool recycles entry slices across tables of the same encode.
entriesPool [][]entry
// lastWasHeader is true when the last line written was a table header,
// used to avoid empty lines between consecutive table definitions.
lastWasHeader bool
// stringKeyBuf is a reusable buffer to read string map keys without
// allocating one per map.
stringKeyBuf reflect.Value
}
// valueOptions are the encoding options attached to one entry of a table.
type valueOptions struct {
multiline bool
inline bool
omitempty bool
omitzero bool
commented bool
comment string
}
// entry is a deferred key-value of a table being encoded.
type entry struct {
key string
value reflect.Value
options valueOptions
}
func (e *encoderState) encodeRoot(v interface{}) error {
if v == nil {
return errors.New("toml: cannot encode a nil interface")
}
rv := reflect.ValueOf(v)
rv, ok := resolve(rv)
if !ok {
return errors.New("toml: cannot encode a nil pointer")
}
switch rv.Kind() {
case reflect.Map, reflect.Struct:
if isValueKind(rv) {
return fmt.Errorf("toml: cannot encode a %s as a document root", rv.Type())
}
return e.encodeTable(rv, false, 0)
default:
return fmt.Errorf("toml: cannot encode a %s as a document root", rv.Type())
}
}
// resolve unwraps pointers and interfaces until a concrete value is found.
// Returns false if it resolves to nil.
func resolve(v reflect.Value) (reflect.Value, bool) {
for {
switch v.Kind() {
case reflect.Ptr:
if v.IsNil() {
return v, false
}
v = v.Elem()
case reflect.Interface:
if v.IsNil() {
return v, false
}
v = v.Elem()
default:
return v, true
}
}
}
// typeEncProps caches the per-type facts used on every value encode.
type typeEncProps struct {
// 0: not a TextMarshaler, 1: the type implements it, 2: its pointer does
text uint8
// encoded as a TOML value (as opposed to a table)
isValue bool
}
var typeEncPropsCache sync.Map // reflect.Type -> typeEncProps
func encPropsForType(t reflect.Type) typeEncProps {
if p, ok := typeEncPropsCache.Load(t); ok {
return p.(typeEncProps)
}
var p typeEncProps
switch {
case t.Implements(textMarshalerType):
p.text = 1
case reflect.PtrTo(t).Implements(textMarshalerType):
p.text = 2
}
switch t {
case timeType, localDateType, localTimeType, localDateTimeType:
p.isValue = true
default:
if p.text != 0 {
p.isValue = true
} else {
switch t.Kind() {
case reflect.Map, reflect.Struct:
p.isValue = false
default:
p.isValue = true
}
}
}
typeEncPropsCache.Store(t, p)
return p
}
// isValueKind returns true when the resolved value is encoded as a TOML
// value (as opposed to a table).
func isValueKind(v reflect.Value) bool {
return encPropsForType(v.Type()).isValue
}
// isTableLike returns true when the value should be encoded as a table (or
// an array of tables for slices).
func (e *encoderState) isTableLike(v reflect.Value) bool {
v, ok := resolve(v)
if !ok {
// Unresolvable values (interface-held nil pointers) are encoded as
// the zero value of their element type by the value path.
return false
}
return !isValueKind(v)
}
// isArrayOfTables returns true when the value is a non-empty slice or array
// containing only table-like values.
func (e *encoderState) isArrayOfTables(v reflect.Value) bool {
v, ok := resolve(v)
if !ok {
return false
}
if v.Kind() != reflect.Slice && v.Kind() != reflect.Array {
return false
}
if v.Len() == 0 {
return false
}
for i := 0; i < v.Len(); i++ {
elem, ok := resolve(v.Index(i))
if !ok || isValueKind(elem) {
return false
}
}
return true
}
// encodeTable writes the content of a table at the given key path.
func (e *encoderState) encodeTable(v reflect.Value, commented bool, indent int) error {
entries, err := e.collectEntries(v)
if err != nil {
return err
}
// First pass: emit all key-values; tables are handled by the second
// pass.
for i := range entries {
ent := &entries[i]
if e.entryIsTable(ent) {
continue
}
err := e.encodeKeyValue(*ent, commented, indent)
if err != nil {
return err
}
}
// Second pass: emit the sub-tables, extending the shared key stack.
for i := range entries {
ent := entries[i]
if !e.entryIsTable(&ent) {
continue
}
entCommented := commented || ent.options.commented
e.keyStack = append(e.keyStack, ent.key)
if e.isArrayOfTables(ent.value) {
err := e.encodeArrayTable(ent, entCommented, indent)
if err != nil {
return err
}
e.keyStack = e.keyStack[:len(e.keyStack)-1]
continue
}
// The value is resolvable: entryIsTable already resolved it.
tv, _ := resolve(ent.value)
e.writeTableHeader(ent.options.comment, entCommented, false, indent)
err := e.encodeTable(tv, entCommented, indent+1)
if err != nil {
return err
}
e.keyStack = e.keyStack[:len(e.keyStack)-1]
}
e.putEntries(entries)
return nil
}
// entryIsTable reports whether the entry is emitted as a (sub-)table rather
// than a key-value.
func (e *encoderState) entryIsTable(ent *entry) bool {
return !e.tablesInline && !ent.options.inline && (e.isTableLike(ent.value) || e.isArrayOfTables(ent.value))
}
// getEntries returns a reusable entry slice.
func (e *encoderState) getEntries() []entry {
if n := len(e.entriesPool); n > 0 {
s := e.entriesPool[n-1]
e.entriesPool = e.entriesPool[:n-1]
return s[:0]
}
return nil
}
// putEntries returns an entry slice to the pool.
func (e *encoderState) putEntries(s []entry) {
if cap(s) > 0 {
e.entriesPool = append(e.entriesPool, s)
}
}
// encodeArrayTable writes all the elements of an array of tables.
func (e *encoderState) encodeArrayTable(ent entry, commented bool, indent int) error {
v, _ := resolve(ent.value)
comment := ent.options.comment
for i := 0; i < v.Len(); i++ {
// Elements are resolvable: isArrayOfTables already resolved them.
elem, _ := resolve(v.Index(i))
e.writeTableHeader(comment, commented, true, indent)
// The comment is only present before the first element.
comment = ""
err := e.encodeTable(elem, commented, indent+1)
if err != nil {
return err
}
}
return nil
}
// writeTableHeader emits a [table] or [[array table]] header line, preceded
// by an empty line and comments as needed.
func (e *encoderState) writeTableHeader(comment string, commented bool, array bool, indent int) {
key := e.keyStack
if len(e.buf) > 0 && !e.lastWasHeader {
e.buf = append(e.buf, '\n')
}
headerIndent := indent
e.writeComment(comment, headerIndent)
// The "commented" marker is emitted at column zero, ahead of any table
// indentation, so that the indentation appears inside the comment
// (e.g. `# [a.b]`). This matches the historical v2.3 layout.
if commented {
e.buf = append(e.buf, "# "...)
}
e.writeIndent(headerIndent)
e.buf = append(e.buf, '[')
if array {
e.buf = append(e.buf, '[')
}
for i, part := range key {
if i > 0 {
e.buf = append(e.buf, '.')
}
e.buf = e.appendKey(e.buf, part)
}
e.buf = append(e.buf, ']')
if array {
e.buf = append(e.buf, ']')
}
e.buf = append(e.buf, '\n')
e.lastWasHeader = true
}
func (e *encoderState) writeIndent(indent int) {
if !e.indentTables {
return
}
for i := 0; i < indent; i++ {
e.buf = append(e.buf, e.indentSymbol...)
}
}
// writeComment emits the comment lines attached to an entry.
func (e *encoderState) writeComment(comment string, indent int) {
if comment == "" {
return
}
for _, line := range strings.Split(comment, "\n") {
e.writeIndent(indent)
e.buf = append(e.buf, "# "...)
e.buf = append(e.buf, line...)
e.buf = append(e.buf, '\n')
}
}
// encodeKeyValue writes one `key = value` line of a table.
func (e *encoderState) encodeKeyValue(ent entry, commented bool, indent int) error {
commented = commented || ent.options.commented
e.writeComment(ent.options.comment, indent)
// The "commented" marker is emitted at column zero, ahead of any table
// indentation, so that the indentation appears inside the comment
// (e.g. `# key = value`). This matches the historical v2.3 layout.
lineStart := len(e.buf)
if commented {
e.buf = append(e.buf, "# "...)
}
e.writeIndent(indent)
e.buf = e.appendKey(e.buf, ent.key)
e.buf = append(e.buf, " = "...)
// When tables are not indented, the key is emitted at column zero
// regardless of its nesting depth. Value continuation lines (most
// notably the elements of a multiline array) must line up with that key,
// so the value indentation starts from zero as well rather than from the
// table nesting depth.
valueIndent := indent
if !e.indentTables {
valueIndent = 0
}
var err error
e.buf, err = e.appendValue(e.buf, ent.value, ent.options, valueIndent)
if err != nil {
return err
}
// A commented value that renders across multiple lines (a multiline string
// or a multiline array) must have every physical line prefixed with the
// comment marker, not just the first; otherwise the continuation lines are
// emitted as live, syntactically invalid TOML.
if commented {
if bytes.IndexByte(e.buf[lineStart:], '\n') >= 0 {
region := bytes.ReplaceAll(
append([]byte(nil), e.buf[lineStart:]...),
[]byte("\n"), []byte("\n# "),
)
e.buf = append(e.buf[:lineStart], region...)
}
}
e.buf = append(e.buf, '\n')
e.lastWasHeader = false
return nil
}
// collectEntries builds the ordered list of the entries of a table,
// applying tags and omission rules.
func (e *encoderState) collectEntries(v reflect.Value) ([]entry, error) {
switch v.Kind() {
case reflect.Map:
return e.collectMapEntries(v)
case reflect.Struct:
entries := e.getEntries()
e.collectStructEntries(&entries, v)
return entries, nil
default:
return nil, fmt.Errorf("toml: cannot encode a %s as a table", v.Type())
}
}
func (e *encoderState) collectMapEntries(v reflect.Value) ([]entry, error) {
entries := e.getEntries()
// Keys are converted to strings right away: read them into a reusable
// buffer to avoid one allocation per key.
var kbuf reflect.Value
if v.Type().Key() == stringType {
if !e.stringKeyBuf.IsValid() {
e.stringKeyBuf = reflect.New(stringType).Elem()
}
kbuf = e.stringKeyBuf
} else {
kbuf = reflect.New(v.Type().Key()).Elem()
}
iter := v.MapRange()
for iter.Next() {
kbuf.SetIterKey(iter)
key, err := mapKeyString(kbuf)
if err != nil {
return nil, err
}
value := iter.Value()
if value.Kind() == reflect.Interface && value.IsNil() {
// nil interface values are skipped
continue
}
if value.Kind() == reflect.Ptr && value.IsNil() {
// nil pointers in maps are encoded as their zero value
value = reflect.New(value.Type().Elem()).Elem()
}
entries = append(entries, entry{key: key, value: value})
}
if len(entries) > 1 {
// slices.SortFunc avoids boxing the slice into a sort.Interface (an
// allocation that sort.Sort incurs for every table).
slices.SortFunc(entries, func(a, b entry) int {
return strings.Compare(a.key, b.key)
})
}
return entries, nil
}
// mapKeyString converts a map key to its string representation.
func mapKeyString(k reflect.Value) (string, error) {
kr, ok := resolve(k)
if !ok {
return "", errors.New("toml: cannot encode a nil map key")
}
if kr.Type().Implements(textMarshalerType) {
b, err := kr.Interface().(encoding.TextMarshaler).MarshalText()
if err != nil {
return "", fmt.Errorf("toml: cannot marshal map key: %w", err)
}
return string(b), nil
}
if kr.CanAddr() && reflect.PtrTo(kr.Type()).Implements(textMarshalerType) {
b, err := kr.Addr().Interface().(encoding.TextMarshaler).MarshalText()
if err != nil {
return "", fmt.Errorf("toml: cannot marshal map key: %w", err)
}
return string(b), nil
}
switch kr.Kind() {
case reflect.String:
return kr.String(), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return strconv.FormatInt(kr.Int(), 10), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return strconv.FormatUint(kr.Uint(), 10), nil
case reflect.Float32:
return strconv.FormatFloat(kr.Float(), 'f', -1, 32), nil
case reflect.Float64:
return strconv.FormatFloat(kr.Float(), 'f', -1, 64), nil
default:
return "", fmt.Errorf("toml: cannot encode a map with key type %s", k.Type())
}
}
// encPlanField is the static encoding information of one field of a struct.
type encPlanField struct {
name string
index []int
depth int
options valueOptions
}
// encPlan caches the per-type information needed to encode a struct:
// flattened fields with parsed tags, in order of definition, with shadowed
// duplicates already removed.
type encPlan struct {
fields []encPlanField
}
var encPlans sync.Map // reflect.Type -> *encPlan
func encPlanForType(t reflect.Type) *encPlan {
if plan, ok := encPlans.Load(t); ok {
return plan.(*encPlan)
}
plan := &encPlan{}
visited := map[reflect.Type]bool{}
buildEncPlan(plan, t, nil, 0, visited)
dedupEncPlan(plan)
encPlans.Store(t, plan)
return plan
}
func buildEncPlan(plan *encPlan, t reflect.Type, prefix []int, depth int, visited map[reflect.Type]bool) {
if visited[t] {
return
}
visited[t] = true
defer delete(visited, t)
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
tag, tagged := f.Tag.Lookup("toml")
if tag == "-" {
continue
}
name := f.Name
var opts valueOptions
if tagged {
parts := strings.Split(tag, ",")
if parts[0] != "" {
name = parts[0]
}
for _, opt := range parts[1:] {
switch opt {
case "multiline":
opts.multiline = true
case "inline":
opts.inline = true
case "omitempty":
opts.omitempty = true
case "omitzero":
opts.omitzero = true
case "commented":
opts.commented = true
}
}
}
// Standalone boolean tags, e.g. multiline:"true".
const tagTrue = "true"
if f.Tag.Get("multiline") == tagTrue {
opts.multiline = true
}
if f.Tag.Get("inline") == tagTrue {
opts.inline = true
}
if f.Tag.Get("commented") == tagTrue {
opts.commented = true
}
opts.comment = f.Tag.Get("comment")
index := make([]int, 0, len(prefix)+1)
index = append(index, prefix...)
index = append(index, i)
if f.Anonymous {
ft := f.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
if ft.Kind() == reflect.Struct && (!tagged || tagName(tag) == "") {
buildEncPlan(plan, ft, index, depth+1, visited)
continue
}
if f.PkgPath != "" && ft.Kind() != reflect.Interface {
continue
}
} else if f.PkgPath != "" {
// unexported
continue
}
plan.fields = append(plan.fields, encPlanField{
name: name,
index: index,
depth: depth,
options: opts,
})
}
}
// dedupEncPlan removes the fields shadowed by another one with the same
// name (the shallowest wins), keeping the order of first appearance.
func dedupEncPlan(plan *encPlan) {
byName := make(map[string]int, len(plan.fields))
drop := false
for i := range plan.fields {
f := &plan.fields[i]
j, seen := byName[f.name]
if !seen {
byName[f.name] = i
continue
}
drop = true
// Shallowest wins; on equal depth, the first in order wins.
if f.depth < plan.fields[j].depth {
plan.fields[j].name = ""
byName[f.name] = i
} else {
f.name = ""
}
}
if !drop {
return
}
out := plan.fields[:0]
for _, f := range plan.fields {
if f.name != "" {
out = append(out, f)
}
}
plan.fields = out
}
// collectStructEntries appends the entries of a struct, flattening embedded
// structs in place.
func (e *encoderState) collectStructEntries(entries *[]entry, v reflect.Value) {
plan := encPlanForType(v.Type())
for i := range plan.fields {
f := &plan.fields[i]
fv, ok := fieldByIndexSkipNil(v, f.index)
if !ok {
// nil embedded pointer on the way: skipped
continue
}
// Anonymous interface fields that are nil are skipped.
if fv.Kind() == reflect.Interface && fv.IsNil() {
continue
}
// nil values in struct fields are skipped
if (fv.Kind() == reflect.Ptr || fv.Kind() == reflect.Map) && fv.IsNil() {
continue
}
if f.options.omitempty && isEmptyValue(fv) {
continue
}
if f.options.omitzero && isZeroValue(fv) {
continue
}
*entries = append(*entries, entry{key: f.name, value: fv, options: f.options})
}
}
// fieldByIndexSkipNil returns the field at the given index path, reporting
// false if a nil embedded pointer is found on the way.
func fieldByIndexSkipNil(v reflect.Value, index []int) (reflect.Value, bool) {
for i, x := range index {
if i > 0 {
for v.Kind() == reflect.Ptr {
if v.IsNil() {
return v, false
}
v = v.Elem()
}
}
v = v.Field(x)
}
return v, true
}
func tagName(tag string) string {
if idx := strings.IndexByte(tag, ','); idx >= 0 {
return tag[:idx]
}
return tag
}
// isEmptyValue implements the omitempty rules.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Map, reflect.Slice, reflect.Array:
return v.Len() == 0
case reflect.Ptr, reflect.Interface:
return v.IsNil()
case reflect.Struct:
// Structs that encode as a scalar value (time.Time, the local
// date/time types, or any TextMarshaler) are empty only when they
// equal their zero value; their fields are typically unexported, so
// recursing into them would be meaningless.
if encPropsForType(v.Type()).isValue {
return v.IsZero()
}
// Plain structs encode as tables and are empty when every field that
// would be encoded is itself empty. This matches the recursive rule
// used before the encoder rewrite and, in particular, treats a
// non-nil but empty map or slice as empty (reflect.Value.IsZero does
// not, which would otherwise emit an empty table header).
return isEmptyStruct(v)
default:
return false
}
}
// isEmptyStruct reports whether all of a table-valued struct's encodable
// fields are empty per isEmptyValue. It mirrors the field selection done by
// collectStructEntries (embedded flattening, shadowing, and "-" skips) so the
// emptiness decision matches what would actually be encoded.
func isEmptyStruct(v reflect.Value) bool {
plan := encPlanForType(v.Type())
for i := range plan.fields {
fv, ok := fieldByIndexSkipNil(v, plan.fields[i].index)
if !ok {
// A nil embedded pointer along the path contributes nothing.
continue
}
if !isEmptyValue(fv) {
return false
}
}
return true
}
// isZeroValue implements the omitzero rules: the type's own IsZero() when
// implemented, the reflect zero value otherwise.
func isZeroValue(v reflect.Value) bool {
if v.Type().Implements(isZeroerType) {
return v.Interface().(isZeroer).IsZero()
}
if v.CanAddr() && reflect.PtrTo(v.Type()).Implements(isZeroerType) {
return v.Addr().Interface().(isZeroer).IsZero()
}
if !v.CanAddr() && reflect.PtrTo(v.Type()).Implements(isZeroerType) {
tmp := reflect.New(v.Type())
tmp.Elem().Set(v)
return tmp.Interface().(isZeroer).IsZero()
}
return v.IsZero()
}
// appendKey emits a key, quoted only if necessary.
func (e *encoderState) appendKey(b []byte, key string) []byte {
if isBareKey(key) {
return append(b, key...)
}
return e.appendString(b, key)
}
func isBareKey(key string) bool {
if len(key) == 0 {
return false
}
for _, c := range []byte(key) {
if !isUnquotedKeyByte(c) {
return false
}
}
return true
}
func isUnquotedKeyByte(c byte) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_'
}
// appendValue emits a TOML value.
func (e *encoderState) appendValue(b []byte, v reflect.Value, opts valueOptions, indent int) ([]byte, error) {
t := v.Type()
// Special types take precedence over their kind.
switch t {
case timeType:
return v.Interface().(time.Time).AppendFormat(b, "2006-01-02T15:04:05.999999999Z07:00"), nil
case localDateType:
return append(b, v.Interface().(LocalDate).String()...), nil
case localTimeType:
return append(b, v.Interface().(LocalTime).String()...), nil
case localDateTimeType:
return append(b, v.Interface().(LocalDateTime).String()...), nil
case jsonNumberType:
if e.marshalJSONNumbers {
return appendJSONNumber(b, v.Interface().(json.Number))
}
}
switch encPropsForType(t).text {
case 1:
if t.Kind() != reflect.String {
return e.appendTextMarshaler(b, v.Interface().(encoding.TextMarshaler))
}
case 2:
if v.CanAddr() {
return e.appendTextMarshaler(b, v.Addr().Interface().(encoding.TextMarshaler))
}
tmp := reflect.New(t)
tmp.Elem().Set(v)
return e.appendTextMarshaler(b, tmp.Interface().(encoding.TextMarshaler))
}
switch v.Kind() {
case reflect.Ptr:
if v.IsNil() {
// nil pointers are encoded as the zero value of their element
// type.
return e.appendValue(b, reflect.Zero(t.Elem()), opts, indent)
}
return e.appendValue(b, v.Elem(), opts, indent)
case reflect.Interface:
if v.IsNil() {
return nil, errors.New("toml: cannot encode a nil interface")
}
return e.appendValue(b, v.Elem(), opts, indent)
case reflect.String:
s := v.String()
// The "multiline" option only takes effect when the string actually
// contains a newline. Wrapping a single-line value such as "2" in a
// """...""" block adds noise without improving readability, so it is
// emitted as a regular single-line string instead.
if opts.multiline && strings.IndexByte(s, '\n') >= 0 {
return e.appendMultilineString(b, s), nil
}
return e.appendString(b, s), nil
case reflect.Bool:
if v.Bool() {
return append(b, "true"...), nil
}
return append(b, "false"...), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return strconv.AppendInt(b, v.Int(), 10), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
u := v.Uint()
if u > math.MaxInt64 {
return nil, fmt.Errorf("toml: cannot encode an unsigned integer above math.MaxInt64: %d", u)
}
return strconv.AppendUint(b, u, 10), nil
case reflect.Float32:
return appendFloat(b, v.Float(), 32), nil
case reflect.Float64:
return appendFloat(b, v.Float(), 64), nil
case reflect.Slice, reflect.Array:
return e.appendArray(b, v, opts, indent)
case reflect.Map:
return e.appendInlineTable(b, v, indent)
case reflect.Struct:
return e.appendInlineTable(b, v, indent)
default:
return nil, fmt.Errorf("toml: cannot encode value of type %s", v.Type())
}
}
var jsonNumberType = reflect.TypeOf(json.Number(""))
func appendJSONNumber(b []byte, n json.Number) ([]byte, error) {
if n == "" {
return append(b, '0'), nil
}
if i, err := n.Int64(); err == nil {
return strconv.AppendInt(b, i, 10), nil
}
f, err := n.Float64()
if err != nil {
return nil, fmt.Errorf("toml: cannot encode json.Number %q: %w", string(n), err)
}
return appendFloat(b, f, 64), nil
}
func appendFloat(b []byte, f float64, bitSize int) []byte {
switch {
case math.IsNaN(f):
return append(b, "nan"...)
case math.IsInf(f, 1):
return append(b, "inf"...)
case math.IsInf(f, -1):
return append(b, "-inf"...)
}
start := len(b)
b = strconv.AppendFloat(b, f, 'f', -1, bitSize)
// TOML floats must have a fractional part or an exponent.
if !bytes.ContainsAny(b[start:], ".eE") {
b = append(b, ".0"...)
}
return b
}
func (e *encoderState) appendTextMarshaler(b []byte, m encoding.TextMarshaler) ([]byte, error) {
text, err := m.MarshalText()
if err != nil {
return nil, fmt.Errorf("toml: error calling MarshalText: %w", err)
}
return e.appendString(b, string(text)), nil
}
// appendArray encodes a slice or array value.
func (e *encoderState) appendArray(b []byte, v reflect.Value, opts valueOptions, indent int) ([]byte, error) {
multiline := opts.multiline || e.arraysMultiline
b = append(b, '[')
if multiline && v.Len() > 0 {
for i := 0; i < v.Len(); i++ {
if i > 0 {
b = append(b, ',')
}
b = append(b, '\n')
for j := 0; j <= indent; j++ {
b = append(b, e.indentSymbol...)
}
var err error
b, err = e.appendValue(b, v.Index(i), valueOptions{}, indent+1)
if err != nil {
return nil, err
}
}
b = append(b, '\n')
for j := 0; j < indent; j++ {
b = append(b, e.indentSymbol...)
}
} else {
for i := 0; i < v.Len(); i++ {
if i > 0 {
b = append(b, ", "...)
}
var err error
b, err = e.appendValue(b, v.Index(i), valueOptions{}, indent)
if err != nil {
return nil, err
}
}
}
return append(b, ']'), nil
}
// appendInlineTable encodes a map or a struct as an inline table.
func (e *encoderState) appendInlineTable(b []byte, v reflect.Value, indent int) ([]byte, error) {
entries, err := e.collectEntries(v)
if err != nil {
return nil, err
}
b = append(b, '{')
for i, ent := range entries {
if i > 0 {
b = append(b, ", "...)
}
b = e.appendKey(b, ent.key)
b = append(b, " = "...)
// multiline strings are not allowed inside inline tables: they
// would break the single-line requirement.
opts := ent.options
opts.multiline = false
b, err = e.appendValue(b, ent.value, opts, indent)
if err != nil {
return nil, err
}
}
e.putEntries(entries)
return append(b, '}'), nil
}
// appendString encodes a string, using a literal string when possible and a
// basic string otherwise.
func (e *encoderState) appendString(b []byte, s string) []byte {
if canBeLiteral(s) {
b = append(b, '\'')
b = append(b, s...)
return append(b, '\'')
}
return appendBasicString(b, s)
}
// canBeLiteral returns true when the string can be represented as a TOML
// literal string: no control characters, no single quote, no newline.
func canBeLiteral(s string) bool {
for i := 0; i < len(s); i++ {
c := s[i]
if c == '\'' || c == 0x7f || c < 0x20 {
return false
}
}
return utf8.ValidString(s)
}
// appendBasicString encodes a string as a TOML basic (double-quoted) string.
func appendBasicString(b []byte, s string) []byte {
b = append(b, '"')
for i := 0; i < len(s); {
c := s[i]
switch {
case c == '"':
b = append(b, '\\', '"')
i++
case c == '\\':
b = append(b, '\\', '\\')
i++
case c == '\b':
b = append(b, '\\', 'b')
i++
case c == '\f':
b = append(b, '\\', 'f')
i++
case c == '\n':
b = append(b, '\\', 'n')
i++
case c == '\r':
b = append(b, '\\', 'r')
i++
case c == '\t':
b = append(b, '\\', 't')
i++
case c < 0x20 || c == 0x7f:
b = append(b, fmt.Sprintf("\\u%04X", c)...)
i++
default:
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
// Replace invalid bytes by the replacement character.
b = append(b, fmt.Sprintf("\\u%04X", c)...)
i++
continue
}
b = append(b, s[i:i+size]...)
i += size
}
}
return append(b, '"')
}
// appendMultilineString encodes a string as a TOML multi-line basic string.
func appendMultilineString(b []byte, s string) []byte {
b = append(b, `"""`...)
b = append(b, '\n')
for i := 0; i < len(s); {
c := s[i]
switch {
case c == '"':
// Runs of three or more quotes must be escaped.
j := i
for j < len(s) && s[j] == '"' {
j++
}
if j-i >= 3 {
for ; i < j; i++ {
b = append(b, '\\', '"')
}
} else {
b = append(b, s[i:j]...)
i = j
}
case c == '\\':
b = append(b, '\\', '\\')
i++
case c == '\n':
b = append(b, '\n')
i++
case c == '\b':
b = append(b, '\\', 'b')
i++
case c == '\f':
b = append(b, '\\', 'f')
i++
case c == '\r':
b = append(b, '\\', 'r')
i++
case c == '\t':
b = append(b, '\t')
i++
case c < 0x20 || c == 0x7f:
b = append(b, fmt.Sprintf("\\u%04X", c)...)
i++
default:
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
b = append(b, fmt.Sprintf("\\u%04X", c)...)
i++
continue
}
b = append(b, s[i:i+size]...)
i += size
}
}
return append(b, `"""`...)
}
func (e *encoderState) appendMultilineString(b []byte, s string) []byte {
return appendMultilineString(b, s)
}
// Package ossfuzz provides a fuzzing target for OSS-Fuzz.
package ossfuzz
import (
"fmt"
"reflect"
"strings"
"github.com/pelletier/go-toml/v2"
)
// FuzzToml is the fuzzing target.
func FuzzToml(data []byte) int {
if len(data) >= 2048 {
return 0
}
if strings.Contains(string(data), "nan") {
return 0
}
var v interface{}
err := toml.Unmarshal(data, &v)
if err != nil {
return 0
}
encoded, err := toml.Marshal(v)
if err != nil {
panic(fmt.Sprintf("failed to marshal unmarshaled document: %s", err))
}
var v2 interface{}
err = toml.Unmarshal(encoded, &v2)
if err != nil {
panic(fmt.Sprintf("failed round trip: %s", err))
}
if !reflect.DeepEqual(v, v2) {
panic(fmt.Sprintf("not equal: %#+v %#+v", v, v2))
}
return 1
}
package toml
import (
"github.com/pelletier/go-toml/v2/internal/tracker"
"github.com/pelletier/go-toml/v2/unstable"
)
type strict struct {
Enabled bool
// Tracks the current key being processed.
key tracker.KeyTracker
missing []decodeError
}
// decodeError is the information needed to materialize a DecodeError once the
// whole document is available.
type decodeError struct {
highlight unstable.Range
key Key
message string
}
// Reset clears the state of the tracker so it can be reused for another
// document.
func (s *strict) Reset() {
s.key = tracker.KeyTracker{}
s.missing = s.missing[:0]
}
// EnterTable is called when a new table or array table expression starts
// being processed.
func (s *strict) EnterTable(node *unstable.Node) {
if !s.Enabled {
return
}
s.key.UpdateTable(node)
}
// MissingTable is called when a table is present in the document but has no
// corresponding field in the target.
func (s *strict) MissingTable(node *unstable.Node) {
if !s.Enabled {
return
}
s.missing = append(s.missing, decodeError{
highlight: keyLocation(node),
key: s.key.Key(),
message: "missing table",
})
}
// MissingField is called when a key-value is present in the document but has
// no corresponding field in the target.
func (s *strict) MissingField(node *unstable.Node) {
if !s.Enabled {
return
}
s.key.Push(node)
s.missing = append(s.missing, decodeError{
highlight: keyLocation(node),
key: s.key.Key(),
message: "unknown field",
})
s.key.Pop(node)
}
// Error returns the cumulated StrictMissingError for the document, or nil.
func (s *strict) Error(document []byte) error {
if !s.Enabled || len(s.missing) == 0 {
return nil
}
err := &StrictMissingError{
Errors: make([]DecodeError, 0, len(s.missing)),
}
for _, derr := range s.missing {
highlight := document[derr.highlight.Offset : derr.highlight.Offset+derr.highlight.Length]
err.Errors = append(err.Errors, *newDecodeError(document, highlight, derr.key, derr.message))
}
return err
}
// keyLocation returns the range of the document covering all the parts of
// the key of the given node.
func keyLocation(node *unstable.Node) unstable.Range {
k := node.Key()
hasOne := k.Next()
if !hasOne {
panic("should not be called with empty key")
}
start := k.Node().Raw
end := start
for k.Next() {
end = k.Node().Raw
}
return unstable.Range{
Offset: start.Offset,
Length: end.Offset + end.Length - start.Offset,
}
}
package toml
import (
"encoding"
"errors"
"fmt"
"io"
"math"
"reflect"
"strconv"
"strings"
"sync"
"time"
"github.com/pelletier/go-toml/v2/internal/tracker"
"github.com/pelletier/go-toml/v2/unstable"
)
// decoderPool recycles decoders (and their internal buffers: parser arena,
// seen-tracker entries, scratch buffers) across calls to Unmarshal and
// Decode.
var decoderPool = sync.Pool{
New: func() interface{} { return &decoder{} },
}
func getDecoder(strictMode, unmarshalerInterface bool) *decoder {
d := decoderPool.Get().(*decoder)
d.reset()
d.strict.Enabled = strictMode
d.unmarshalerInterface = unmarshalerInterface
return d
}
func putDecoder(d *decoder) {
decoderPool.Put(d)
}
// reset clears the per-document state of the decoder, keeping the allocated
// buffers for reuse.
func (d *decoder) reset() {
d.seen.Reset()
d.tableKey = d.tableKey[:0]
d.skipUntilTable = false
d.path = d.path[:0]
d.captures = d.captures[:0]
d.captureIdx = -1
d.segIdx = d.segIdx[:0]
// Reuse the array-table counter slots across documents instead of
// deleting them: a zeroed slot is indistinguishable from an absent one,
// and keeping it alive means setArrayCount does not have to allocate a new
// *int every time the same path reappears. A safety valve bounds the table
// for adversarial inputs that introduce unboundedly many distinct paths.
if len(d.arrayCounts) > 1<<14 {
d.arrayCounts = nil
} else {
for _, p := range d.arrayCounts {
*p = 0
}
}
d.tableTarget = reflect.Value{}
d.tableTargetValid = false
d.tableFlush = d.tableFlush[:0]
d.tableParentSlot = slotWriter{}
d.keyParts = d.keyParts[:0]
d.strict.Reset()
}
// Unmarshal deserializes a TOML document into a Go value.
//
// It is a shortcut for Decoder.Decode() with the default options.
func Unmarshal(data []byte, v interface{}) error {
d := getDecoder(false, false)
err := d.unmarshal(data, v)
putDecoder(d)
return err
}
// Decoder reads and decode a TOML document from an input stream.
type Decoder struct {
// input
r io.Reader
// global settings
strict bool
// toggles unmarshaler interface
unmarshalerInterface bool
}
// NewDecoder creates a new Decoder that will read from r.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}
// DisallowUnknownFields causes the Decoder to return an error when the
// destination is a struct and the input contains a key that does not match a
// non-ignored field.
//
// In that case, the Decoder returns a StrictMissingError that can be used to
// retrieve the individual errors as well as generate a human readable
// description of the missing fields.
func (d *Decoder) DisallowUnknownFields() *Decoder {
d.strict = true
return d
}
// EnableUnmarshalerInterface allows to enable unmarshaler interface.
//
// With this feature enabled, types implementing the unstable.Unmarshaler
// interface can be decoded from any structure of the document. It allows types
// that don't have a straightforward TOML representation to provide their own
// decoding logic.
//
// The UnmarshalTOML method receives raw TOML bytes:
// - For single values: the raw value bytes (e.g., `"hello"` for a string)
// - For tables: all key-value lines belonging to that table
// - For inline tables/arrays: the raw bytes of the inline structure
//
// The unstable.RawMessage type can be used to capture raw TOML bytes for
// later processing, similar to json.RawMessage.
//
// *Unstable:* This method does not follow the compatibility guarantees of
// semver. It can be changed or removed without a new major version being
// issued.
func (d *Decoder) EnableUnmarshalerInterface() *Decoder {
d.unmarshalerInterface = true
return d
}
// Decode the whole content of r into v.
//
// By default, values in the document that don't exist in the target Go value
// are ignored. See Decoder.DisallowUnknownFields() to change this behavior.
//
// When a TOML local date, time, or date-time is decoded into a time.Time, its
// value is represented in time.Local timezone. Otherwise the appropriate Local*
// structure is used. For time values, precision up to the nanosecond is
// supported by truncating extra digits.
//
// Empty tables decoded in an interface{} create an empty initialized
// map[string]interface{}.
//
// Types implementing the encoding.TextUnmarshaler interface are decoded from a
// TOML string.
//
// When decoding a number, go-toml will return an error if the number is out of
// bounds for the target type (which includes negative numbers when decoding
// into an unsigned int).
//
// If an error occurs while decoding the content of the document, this function
// returns a toml.DecodeError, providing context about the issue. When using
// strict mode and a field is missing, a `toml.StrictMissingError` is
// returned. In any other case, this function returns a standard Go error.
//
// # Type mapping
//
// List of supported TOML types and their associated accepted Go types:
//
// String -> string
// Integer -> uint*, int*, depending on size
// Float -> float*, depending on size
// Boolean -> bool
// Offset Date-Time -> time.Time
// Local Date-time -> LocalDateTime, time.Time
// Local Date -> LocalDate, time.Time
// Local Time -> LocalTime, time.Time
// Array -> slice and array, depending on elements types
// Table -> map and struct
// Inline Table -> same as Table
// Array of Tables -> same as Array and Table
func (d *Decoder) Decode(v interface{}) error {
b, err := io.ReadAll(d.r)
if err != nil {
return fmt.Errorf("toml: %w", err)
}
dec := getDecoder(d.strict, d.unmarshalerInterface)
err = dec.unmarshal(b, v)
putDecoder(dec)
return err
}
// pathPart is one part of the key path leading to a value. Parts that come
// from the current table header only carry a name; parts that come from the
// key of the current key-value expression also carry the AST node, and their
// name is materialized lazily to avoid allocations.
type pathPart struct {
name string
node *unstable.Node
}
// bytes returns the raw bytes of the key part.
func (p *pathPart) bytes() []byte {
if p.node != nil {
return p.node.Data
}
return []byte(p.name)
}
// str returns the key part as a string, possibly allocating.
func (p *pathPart) str() string {
if p.node != nil {
return string(p.node.Data)
}
return p.name
}
// rawCapture accumulates the raw bytes fed to a type implementing
// unstable.Unmarshaler for a table target. The target is identified by the
// parts of its key and the array-table indexes in effect when the capture
// was created, so that it can be located again once the whole document has
// been processed (the address of the target may change as slices grow).
type rawCapture struct {
names []string
// indexes[i] is the index to use when reaching a slice or array right
// before consuming names[i]. indexes[len(names)] is the index of the
// element when the target is an element of an array table. -1 when not
// relevant.
indexes []int
buf []byte
}
type decoder struct {
p unstable.Parser
// strict mode
strict strict
// toggles unmarshaler interface
unmarshalerInterface bool
// tracks the duplicate and type consistency of the keys
seen tracker.SeenTracker
// path of the current table header, as copied strings
tableKey []string
// true when the expressions under the current table header cannot be
// stored anywhere and should be skipped
skipUntilTable bool
// scratch buffer for the key path of the current expression
path []pathPart
// raw captures for the unmarshaler interface, in order of first
// appearance. captureIdx is the index of the capture the current table
// belongs to, or -1.
captures []rawCapture
captureIdx int
// segIdx[i] records the array element index used when traversing a
// slice or array right before consuming the i-th part of the current
// table key. Reset for each table expression.
segIdx []int
// arrayCounts tracks the number of elements appended to fixed-size
// arrays used as array tables, keyed by the NUL-joined key parts.
// Values are pointer slots so that updating an existing path does not
// allocate a new key string.
arrayCounts map[string]*int
// Cached target of the current table, so that key-values do not need to
// walk the document structure from the root for every expression.
// tableFlush holds the write-backs to perform when leaving the table
// (for targets reached through map values, which are copies).
// tableParentSlot stores a replacement of the target itself (e.g. a nil
// map that was allocated) into its parent.
tableTarget reflect.Value
tableTargetValid bool
tableFlush []flushOp
tableParentSlot slotWriter
// strKey is a reusable string value used as map key, so that map
// operations with string keys do not need to allocate a boxed key for
// every access. It must be refreshed with stringMapKey immediately
// before each use: any recursive call may overwrite it.
strKey reflect.Value
// interned de-duplicates key strings: documents repeat the same keys
// over and over, and the table survives pooling, so repeated decodes
// of similar documents stop allocating key strings altogether.
interned map[string]string
// pathScratch is the buffer used by joinPath.
pathScratch []byte
// keyParts is the reusable buffer holding the decoded parts of the key of
// the current expression in the fused generic decode path.
keyParts [][]byte
}
// slotWriter remembers how to store a value at some location of the target
// structure. Implemented as a struct instead of a closure to avoid
// allocations.
type slotWriter struct {
kind uint8 // 0: none, 1: slot.Set, 2: m.SetMapIndex(k, ...), 3: m.SetMapIndex(string key ks, ...)
slot reflect.Value
m reflect.Value
k reflect.Value
ks string
}
func (d *decoder) storeSlot(s *slotWriter, nv reflect.Value) {
switch s.kind {
case 1:
if s.slot.CanSet() {
s.slot.Set(nv)
}
case 2:
s.m.SetMapIndex(s.k, nv)
case 3:
s.m.SetMapIndex(d.stringMapKey(s.ks), nv)
}
}
// flushOp stores val using w when the table is flushed.
type flushOp struct {
w slotWriter
val reflect.Value
}
// flushTable performs the pending write-backs of the cached table target, in
// reverse order so that inner copies land before their parents are stored.
func (d *decoder) flushTable() {
for i := len(d.tableFlush) - 1; i >= 0; i-- {
d.storeSlot(&d.tableFlush[i].w, d.tableFlush[i].val)
}
d.tableFlush = d.tableFlush[:0]
d.tableTargetValid = false
d.tableParentSlot = slotWriter{}
d.tableTarget = reflect.Value{}
}
// intern returns the string corresponding to the given bytes, reusing a
// previous allocation when the same key has been seen before.
func (d *decoder) intern(b []byte) string {
if s, ok := d.interned[string(b)]; ok { // does not allocate
return s
}
if d.interned == nil {
d.interned = make(map[string]string, 64)
} else if len(d.interned) >= 1<<14 {
// Safety valve for adversarial inputs: do not let the table grow
// without bounds.
for k := range d.interned {
delete(d.interned, k)
}
}
s := string(b)
d.interned[s] = s
return s
}
// partString returns the name of a path part, interning it when it comes
// from the document.
func (d *decoder) partString(p *pathPart) string {
if p.node != nil {
return d.intern(p.node.Data)
}
return p.name
}
// stringMapKey returns a reflect.Value holding the given string, reusing the
// same allocation every time. The result must be used (the map operation
// performed) before any recursive call, which may overwrite the buffer.
func (d *decoder) stringMapKey(s string) reflect.Value {
if !d.strKey.IsValid() {
d.strKey = reflect.New(stringType).Elem()
}
d.strKey.SetString(s)
return d.strKey
}
// joinPath builds the NUL-joined representation of a key path in the
// decoder's scratch buffer. The result is only valid until the next call.
func (d *decoder) joinPath(parts []string) []byte {
d.pathScratch = d.pathScratch[:0]
for i, p := range parts {
if i > 0 {
d.pathScratch = append(d.pathScratch, 0)
}
d.pathScratch = append(d.pathScratch, p...)
}
return d.pathScratch
}
// arrayCount returns the number of elements appended so far to the array
// table at the given path.
func (d *decoder) arrayCount(key []byte) int {
if d.arrayCounts == nil {
return 0
}
if p := d.arrayCounts[string(key)]; p != nil { // does not allocate
return *p
}
return 0
}
func (d *decoder) setArrayCount(key []byte, n int) {
if d.arrayCounts == nil {
d.arrayCounts = map[string]*int{}
}
if p := d.arrayCounts[string(key)]; p != nil { // does not allocate
*p = n
return
}
v := n
d.arrayCounts[string(key)] = &v
}
// resetChildArrayCounts forgets the counts of all the array tables under
// the given path, so that a new element starts fresh.
func (d *decoder) resetChildArrayCounts(key []byte) {
if len(d.arrayCounts) == 0 {
return
}
for k, p := range d.arrayCounts {
// Prefix match without building the prefix string: same bytes as
// key, followed by the NUL separator.
if len(k) > len(key) && k[len(key)] == 0 && k[:len(key)] == string(key) {
// Zero instead of delete: the next element of the parent table
// will reuse the slot without allocating a new key.
*p = 0
}
}
}
func (d *decoder) typeMismatchError(toml string, target reflect.Type, highlight []byte) error {
return &typeMismatchError{
toml: toml,
target: target,
highlight: highlight,
}
}
type typeMismatchError struct {
toml string
target reflect.Type
highlight []byte
// key is the TOML key being processed when the mismatch occurred. It is
// populated lazily as the error propagates back up to the key-value
// handler (see contextualizeError).
key Key
}
func (e *typeMismatchError) Error() string {
return fmt.Sprintf("cannot decode TOML %s into %s", e.toml, e.target)
}
// contextualizeError attaches the TOML key currently being processed to errors
// raised while decoding a key-value expression, so that DecodeError.Key()
// reports the offending key (e.g. on type mismatch errors). The current key is
// reconstructed from d.path; when the table target is cached, d.path holds only
// the key-value parts, so the table key prefix is prepended. This only runs on
// the error path and adds no cost to successful decodes.
func (d *decoder) contextualizeError(err error, withTableKey bool) error {
var mm *typeMismatchError
if errors.As(err, &mm) {
if mm.key == nil {
mm.key = d.currentKey(withTableKey)
}
return err
}
var perr *unstable.ParserError
if errors.As(err, &perr) {
if perr.Key == nil {
perr.Key = d.currentKey(withTableKey)
}
}
return err
}
// currentKey reconstructs the full TOML key being processed from the decoder's
// path. When withTableKey is true, d.path contains only the key-value parts
// (the table target is cached) and the table key is prepended.
func (d *decoder) currentKey(withTableKey bool) Key {
n := len(d.path)
if withTableKey {
n += len(d.tableKey)
}
key := make(Key, 0, n)
if withTableKey {
key = append(key, d.tableKey...)
}
for i := range d.path {
key = append(key, d.path[i].str())
}
return key
}
func (d *decoder) unmarshal(data []byte, v interface{}) error {
r := reflect.ValueOf(v)
if r.Kind() != reflect.Ptr {
return fmt.Errorf("toml: decoding can only be performed into a pointer, not %s", r.Kind())
}
if r.IsNil() {
return errors.New("toml: decoding pointer target cannot be nil")
}
root := r.Elem()
d.captureIdx = -1
d.p.Reset(data)
// Fully generic targets (interface{} or map[string]interface{}) are
// decoded straight into native Go maps and slices, with no reflection on
// the document structure at all. This covers the common "decode arbitrary
// TOML into a map" case, including every standard benchmark dataset.
if !d.unmarshalerInterface {
if k := root.Kind(); k == reflect.Interface || (k == reflect.Map && root.Type() == mapStringInterfaceType) {
return d.unmarshalFused(root, data)
}
}
// When the root target itself implements the Unmarshaler interface, the
// whole document decodes into it. Open a capture spanning the entire
// document up front: the top-level key-values then flow through the
// existing capture branch and any tables attach to it via resumeCapture,
// so UnmarshalTOML receives the assembled document exactly once.
if d.unmarshalerInterface && hasUnmarshaler(root) {
d.startRootCapture()
}
for d.p.NextExpression() {
err := d.handleRootExpression(d.p.Expression(), root)
if err != nil {
return d.wrapError(data, err)
}
}
if err := d.p.Error(); err != nil {
var perr *unstable.ParserError
if errors.As(err, &perr) {
return wrapDecodeError(data, perr)
}
return err
}
d.flushTable()
// Deliver the accumulated raw documents to the unmarshaler-interface
// targets.
for i := range d.captures {
nv, err := d.resolveCapture(root, &d.captures[i], 0, false)
if err != nil {
return err
}
if nv.IsValid() {
root.Set(nv)
}
}
// An empty document into a generic target still initializes it.
switch root.Kind() {
case reflect.Map:
if root.IsNil() {
root.Set(reflect.MakeMap(root.Type()))
}
case reflect.Interface:
if root.IsNil() {
root.Set(reflect.ValueOf(map[string]interface{}{}))
}
default:
}
return d.strict.Error(data)
}
// setAnyKey assigns the value of a key-value into the native map m, following
// the (possibly dotted) key and creating intermediate maps as needed.
func (d *decoder) setAnyKey(m map[string]interface{}, key unstable.Iterator, value *unstable.Node) error {
cur := m
for key.Next() {
name := d.intern(key.Node().Data)
if key.IsLast() {
av, err := d.decodeAny(value)
if err != nil {
return err
}
cur[name] = av
return nil
}
cur = d.anyChildTable(cur, name)
}
return nil
}
// anyChildTable returns the child table at name within cur, creating it if
// absent and descending into the current (last) element when an array table
// occupies the slot. A non-container in the slot cannot occur for a document
// the seen-tracker has accepted.
func (d *decoder) anyChildTable(cur map[string]interface{}, name string) map[string]interface{} {
switch v := cur[name].(type) {
case map[string]interface{}:
return v
case []interface{}:
if len(v) > 0 {
if last, ok := v[len(v)-1].(map[string]interface{}); ok {
return last
}
}
}
nm := map[string]interface{}{}
cur[name] = nm
return nm
}
// wrapError gives document context to errors generated while processing an
// expression.
func (d *decoder) wrapError(data []byte, err error) error {
var perr *unstable.ParserError
if errors.As(err, &perr) {
return wrapDecodeError(data, perr)
}
var mm *typeMismatchError
if errors.As(err, &mm) {
return wrapDecodeError(data, &unstable.ParserError{
Highlight: mm.highlight,
Message: mm.Error(),
Key: mm.key,
})
}
return err
}
// wrapSeenError turns an error returned by SeenTracker.CheckExpression into a
// ParserError carrying the position and key of the offending expression, so
// that redefinition and duplicate-key errors are reported as a DecodeError
// with context (see issue #668).
//
// The highlight spans the expression's key. Unlike Node.Raw, key nodes always
// carry a Raw range, so this works for tables and array tables too (whose own
// Raw range is not set by the parser). For a duplicate detected inside an
// inline table, node is the enclosing key-value expression, so the error
// points at that expression's key.
func (d *decoder) wrapSeenError(node *unstable.Node, err error) error {
if err == nil {
return nil
}
var key Key
var start, end unstable.Range
it := node.Key()
for it.Next() {
n := it.Node()
key = append(key, string(n.Data))
if len(key) == 1 {
start = n.Raw
}
end = n.Raw
}
var highlight []byte
if len(key) > 0 {
highlight = d.p.Raw(unstable.Range{
Offset: start.Offset,
Length: end.Offset + end.Length - start.Offset,
})
}
return &unstable.ParserError{
Highlight: highlight,
Message: strings.TrimPrefix(err.Error(), "toml: "),
Key: key,
}
}
func (d *decoder) handleRootExpression(expr *unstable.Node, root reflect.Value) error {
first, err := d.seen.CheckExpression(expr)
if err != nil {
return d.wrapSeenError(expr, err)
}
switch expr.Kind {
case unstable.KeyValue:
if d.skipUntilTable {
return nil
}
if d.captureIdx >= 0 {
d.captureKeyValue(expr)
return nil
}
return d.handleKeyValueExpression(expr, root)
case unstable.Table:
d.flushTable()
d.skipUntilTable = false
d.captureIdx = -1
d.strict.EnterTable(expr)
return d.handleTableExpression(expr, root, false, first)
case unstable.ArrayTable:
d.flushTable()
d.skipUntilTable = false
d.captureIdx = -1
d.strict.EnterTable(expr)
return d.handleTableExpression(expr, root, true, first)
default:
return unstable.NewParserError(expr.Data, "unsupported expression kind %s", expr.Kind)
}
}
// updateTableKey copies the parts of the key of a table expression into
// tableKey.
func (d *decoder) updateTableKey(expr *unstable.Node) {
d.tableKey = d.tableKey[:0]
it := expr.Key()
for it.Next() {
d.tableKey = append(d.tableKey, d.intern(it.Node().Data))
}
}
func (d *decoder) handleTableExpression(expr *unstable.Node, root reflect.Value, isArrayTable bool, first bool) error {
d.updateTableKey(expr)
// Check whether this table belongs to an exisiting raw capture (split
// tables, or children of a table assigned to an Unmarshaler).
if d.unmarshalerInterface {
if d.resumeCapture(expr) {
return nil
}
}
// Reset the per-segment array indexes.
d.segIdx = d.segIdx[:0]
for i := 0; i <= len(d.tableKey); i++ {
d.segIdx = append(d.segIdx, -1)
}
return d.walkTable(root, expr, isArrayTable, first)
}
// newContainerElem returns a fresh element for a slice of the given element
// type. Plain interface elements start out as an empty table.
func newContainerElem(et reflect.Type) reflect.Value {
if et == interfaceType {
return reflect.ValueOf(map[string]interface{}{})
}
return reflect.New(et).Elem()
}
// walkTable processes a [table] or [[array table]] header: it creates the
// intermediate containers, appends array-table elements, applies the strict
// policy, registers unmarshaler-interface captures, and caches the target
// container so that the key-values that follow are stored directly.
//
// Map values are not addressable: when one needs in-place mutations (struct
// or array values), a copy is made and registered to be stored back when the
// table changes (see flushTable). Maps and slices are references and are
// traversed without copies.
func (d *decoder) walkTable(root reflect.Value, expr *unstable.Node, isArrayTable bool, first bool) error {
v := root
pf := slotWriter{kind: 1, slot: root}
idx := 0
walk:
for {
// Dereference pointers in place.
for v.Kind() == reflect.Ptr {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
elem := v.Elem()
pf = slotWriter{kind: 1, slot: elem}
v = elem
}
// Tables assigned to a type implementing the unmarshaler interface
// are captured as raw bytes, delivered once the document is read.
if d.unmarshalerInterface && hasUnmarshaler(v) {
d.startCapture(idx, expr)
return nil
}
if idx >= len(d.tableKey) {
break walk
}
name := d.tableKey[idx]
switch v.Kind() {
case reflect.Interface:
if !v.IsNil() {
c := v.Elem()
if k := c.Kind(); k == reflect.Map || k == reflect.Slice {
// Reference types: mutations are visible through the
// existing interface value.
v = c
continue
}
}
// Anything else is replaced by a fresh generic map.
if !mapStringInterfaceType.AssignableTo(v.Type()) {
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot store a table in a %s", v.Type())
}
fresh := reflect.ValueOf(map[string]interface{}{})
d.storeSlot(&pf, fresh)
v = fresh
case reflect.Slice:
if v.Len() == 0 {
// Implicit creation of the first element: the array table
// that would create it has not been seen yet (issue 995).
if v.IsNil() {
v = reflect.MakeSlice(v.Type(), 0, 4)
}
v = reflect.Append(v, newContainerElem(v.Type().Elem()))
d.storeSlot(&pf, v)
}
n := v.Len() - 1
d.segIdx[idx] = n
elem := v.Index(n)
pf = slotWriter{kind: 1, slot: elem}
v = elem
case reflect.Array:
key := d.joinPath(d.tableKey[:idx])
cnt := d.arrayCount(key)
if cnt == 0 {
cnt = 1
d.setArrayCount(key, 1)
}
if cnt > v.Len() {
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot reach element %d of array of size %d", cnt-1, v.Len())
}
d.segIdx[idx] = cnt - 1
elem := v.Index(cnt - 1)
pf = slotWriter{kind: 1, slot: elem}
v = elem
case reflect.Map:
if v.IsNil() {
nm := reflect.MakeMap(v.Type())
d.storeSlot(&pf, nm)
v = nm
}
var key reflect.Value
var w slotWriter
if v.Type().Key() == stringType {
key = d.stringMapKey(name)
w = slotWriter{kind: 3, m: v, ks: name}
} else {
k, err := makeMapKey(v.Type().Key(), name)
if err != nil {
return err
}
key = k
w = slotWriter{kind: 2, m: v, k: k}
}
elem := v.MapIndex(key)
// The last part of an array table is finalized as a slice
// container: do not materialize a table for it.
if isArrayTable && idx == len(d.tableKey)-1 {
et := v.Type().Elem()
switch et.Kind() {
case reflect.Interface, reflect.Slice, reflect.Array:
if elem.IsValid() {
v = elem
} else {
v = reflect.Zero(et)
}
pf = w
idx++
continue
default:
}
}
if elem.IsValid() {
ce := elem
ceIface := false
if ce.Kind() == reflect.Interface {
ceIface = true
if !ce.IsNil() {
ce = ce.Elem()
}
}
switch ce.Kind() {
case reflect.Map, reflect.Slice:
pf = w
v = ce
case reflect.Ptr:
if ce.IsNil() {
np := reflect.New(ce.Type().Elem())
d.storeSlot(&w, np)
ce = np
}
pf = w
v = ce
case reflect.Struct, reflect.Array:
if ceIface {
// Interface-held non-generic content is replaced.
fresh := reflect.ValueOf(map[string]interface{}{})
d.storeSlot(&w, fresh)
pf = w
v = fresh
} else {
tmp := reflect.New(elem.Type()).Elem()
tmp.Set(elem)
d.tableFlush = append(d.tableFlush, flushOp{w: w, val: tmp})
pf = slotWriter{kind: 1, slot: tmp}
v = tmp
}
default:
if !ceIface {
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot store a table in a %s", ce.Type())
}
fresh := reflect.ValueOf(map[string]interface{}{})
d.storeSlot(&w, fresh)
pf = w
v = fresh
}
} else {
et := v.Type().Elem()
switch et.Kind() {
case reflect.Interface:
if !mapStringInterfaceType.AssignableTo(et) {
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot store a table in a %s", et)
}
fresh := reflect.ValueOf(map[string]interface{}{})
d.storeSlot(&w, fresh)
pf = w
v = fresh
case reflect.Map:
nm := reflect.MakeMap(et)
d.storeSlot(&w, nm)
pf = w
v = nm
case reflect.Ptr:
np := reflect.New(et.Elem())
d.storeSlot(&w, np)
pf = w
v = np
case reflect.Struct, reflect.Array, reflect.Slice:
tmp := reflect.New(et).Elem()
d.tableFlush = append(d.tableFlush, flushOp{w: w, val: tmp})
pf = slotWriter{kind: 1, slot: tmp}
v = tmp
default:
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot store a table in a %s", et)
}
}
idx++
case reflect.Struct:
plan := planForType(v.Type())
f, found := plan.lookup(name)
if !found {
d.strict.MissingTable(expr)
d.skipUntilTable = true
return nil
}
fv := fieldByIndexAlloc(v, f.index)
pf = slotWriter{kind: 1, slot: fv}
v = fv
idx++
default:
return unstable.NewParserError(d.p.Raw(expr.Raw), "cannot store a table in a %s", v.Kind())
}
}
if isArrayTable {
akey := d.joinPath(d.tableKey)
d.resetChildArrayCounts(akey)
// Unwrap an interface container.
if v.Kind() == reflect.Interface {
var slice []interface{}
if !v.IsNil() {
if s, ok := v.Elem().Interface().([]interface{}); ok {
slice = s
}
}
if first {
slice = slice[:0]
}
m := map[string]interface{}{}
slice = append(slice, m)
sv := reflect.ValueOf(slice)
d.storeSlot(&pf, sv)
d.setArrayCount(akey, len(slice))
d.segIdx[len(d.tableKey)] = len(slice) - 1
d.tableTarget = reflect.ValueOf(m)
d.tableParentSlot = slotWriter{kind: 1, slot: sv.Index(len(slice) - 1)}
d.tableTargetValid = true
return nil
}
switch v.Kind() {
case reflect.Slice:
if v.IsNil() {
v = reflect.MakeSlice(v.Type(), 0, 4)
} else if first {
v = v.Slice(0, 0)
}
v = reflect.Append(v, newContainerElem(v.Type().Elem()))
d.storeSlot(&pf, v)
n := v.Len() - 1
d.setArrayCount(akey, n+1)
d.segIdx[len(d.tableKey)] = n
elem := v.Index(n)
if d.unmarshalerInterface && hasUnmarshaler(elem) {
d.startCapture(len(d.tableKey), expr)
return nil
}
pf = slotWriter{kind: 1, slot: elem}
v = elem
case reflect.Array:
cnt := d.arrayCount(akey)
if first {
cnt = 0
}
if cnt >= v.Len() {
return unstable.NewParserError(d.p.Raw(expr.Raw), "array of size %d is too small to store this array table", v.Len())
}
v.Index(cnt).Set(reflect.Zero(v.Type().Elem()))
d.setArrayCount(akey, cnt+1)
d.segIdx[len(d.tableKey)] = cnt
elem := v.Index(cnt)
if d.unmarshalerInterface && hasUnmarshaler(elem) {
d.startCapture(len(d.tableKey), expr)
return nil
}
pf = slotWriter{kind: 1, slot: elem}
v = elem
default:
return fmt.Errorf("toml: cannot store an array table in a %s", v.Kind())
}
}
// Settle on the concrete container for the key-values that follow.
for {
switch v.Kind() {
case reflect.Ptr:
if v.IsNil() {
if !v.CanSet() {
return nil
}
v.Set(reflect.New(v.Type().Elem()))
}
elem := v.Elem()
pf = slotWriter{kind: 1, slot: elem}
v = elem
continue
case reflect.Interface:
if !v.IsNil() {
c := v.Elem()
if c.Type() == mapStringInterfaceType || c.Type() == sliceInterfaceType {
v = c
continue
}
}
if !mapStringInterfaceType.AssignableTo(v.Type()) {
return fmt.Errorf("toml: cannot store a table in a %s", v.Type())
}
fresh := reflect.ValueOf(map[string]interface{}{})
d.storeSlot(&pf, fresh)
v = fresh
continue
case reflect.Slice:
if v.Len() == 0 {
if v.IsNil() {
v = reflect.MakeSlice(v.Type(), 0, 4)
}
v = reflect.Append(v, newContainerElem(v.Type().Elem()))
d.storeSlot(&pf, v)
}
n := v.Len() - 1
d.segIdx[len(d.tableKey)] = n
elem := v.Index(n)
pf = slotWriter{kind: 1, slot: elem}
v = elem
continue
case reflect.Map, reflect.Struct:
d.tableTarget = v
d.tableParentSlot = pf
d.tableTargetValid = true
return nil
default:
return fmt.Errorf("toml: cannot store a table in a %s", v.Kind())
}
}
}
// resumeCapture looks for an existing capture this table expression belongs
// to. It returns true if the expression was consumed.
func (d *decoder) resumeCapture(expr *unstable.Node) bool {
// Iterate in reverse, so that tables attach to the latest element of
// array tables.
for i := len(d.captures) - 1; i >= 0; i-- {
c := &d.captures[i]
if len(d.tableKey) < len(c.names) {
continue
}
if expr.Kind == unstable.ArrayTable && len(d.tableKey) == len(c.names) {
// A new element of an array table is not part of the capture of
// the previous element.
continue
}
match := true
for j, p := range c.names {
if d.tableKey[j] != p {
match = false
break
}
}
if !match {
continue
}
d.captureIdx = i
if len(d.tableKey) > len(c.names) {
d.appendCaptureHeader(c, expr, len(c.names))
}
return true
}
return false
}
// appendCaptureHeader writes the table header of expr in the capture buffer,
// adjusted to be relative to the capture root.
func (d *decoder) appendCaptureHeader(c *rawCapture, expr *unstable.Node, skip int) {
c.buf = append(c.buf, '[')
if expr.Kind == unstable.ArrayTable {
c.buf = append(c.buf, '[')
}
c.buf = append(c.buf, d.rawKeySuffix(expr, skip)...)
c.buf = append(c.buf, ']')
if expr.Kind == unstable.ArrayTable {
c.buf = append(c.buf, ']')
}
c.buf = append(c.buf, '\n')
}
// rawKeySuffix returns the raw bytes of the key of the expression, skipping
// the first n parts.
func (d *decoder) rawKeySuffix(expr *unstable.Node, n int) []byte {
it := expr.Key()
idx := 0
var start, end unstable.Range
for it.Next() {
if idx >= n {
r := it.Node().Raw
if start.Length == 0 && start.Offset == 0 && idx == n {
start = r
}
end = r
}
idx++
}
return d.p.Data()[start.Offset : end.Offset+end.Length]
}
// startCapture registers a new capture for the table at the given path
// (prefix of tableKey).
func (d *decoder) startCapture(pathLen int, expr *unstable.Node) {
names := make([]string, pathLen)
copy(names, d.tableKey[:pathLen])
indexes := make([]int, pathLen+1)
copy(indexes, d.segIdx[:pathLen+1])
d.captures = append(d.captures, rawCapture{
names: names,
indexes: indexes,
})
d.captureIdx = len(d.captures) - 1
if pathLen < len(d.tableKey) {
d.appendCaptureHeader(&d.captures[d.captureIdx], expr, pathLen)
}
}
// startRootCapture opens a capture covering the entire document, used when the
// root target itself implements the Unmarshaler interface. It is opened before
// reading any expression: top-level key-values are then accumulated through the
// regular capture branch, and its empty name path matches every table in
// resumeCapture, so the whole document is handed to UnmarshalTOML once. The
// single index slot is -1 because the root is never reached through an array
// table.
func (d *decoder) startRootCapture() {
d.captures = append(d.captures, rawCapture{indexes: []int{-1}})
d.captureIdx = len(d.captures) - 1
}
// resolveCapture walks back to the target of a capture and delivers the
// accumulated raw bytes to its UnmarshalTOML implementation.
func (d *decoder) resolveCapture(v reflect.Value, c *rawCapture, idx int, indexed bool) (reflect.Value, error) {
if v.Kind() == reflect.Ptr {
if v.Type().Implements(unmarshalerType) && idx == len(c.names) {
u, _ := unmarshalerOf(v)
return v, u.UnmarshalTOML(c.buf)
}
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
nv, err := d.resolveCapture(v.Elem(), c, idx, indexed)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
v.Elem().Set(nv)
}
return v, nil
}
if !indexed && (v.Kind() == reflect.Slice || v.Kind() == reflect.Array) && c.indexes[idx] >= 0 {
i := c.indexes[idx]
if i >= v.Len() {
return reflect.Value{}, errors.New("toml: internal error: capture index out of range")
}
elem := v.Index(i)
nv, err := d.resolveCapture(elem, c, idx, true)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
elem.Set(nv)
}
return v, nil
}
if idx == len(c.names) {
u, ok := unmarshalerOf(v)
if !ok {
return reflect.Value{}, errors.New("toml: internal error: capture target does not implement UnmarshalTOML")
}
return v, u.UnmarshalTOML(c.buf)
}
name := c.names[idx]
switch v.Kind() {
case reflect.Struct:
plan := planForType(v.Type())
f, found := plan.lookup(name)
if !found {
return v, nil
}
fv := fieldByIndexAlloc(v, f.index)
nv, err := d.resolveCapture(fv, c, idx+1, false)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() && fv.CanSet() {
fv.Set(nv)
}
return v, nil
case reflect.Map:
key, err := makeMapKey(v.Type().Key(), name)
if err != nil {
return reflect.Value{}, err
}
if v.IsNil() {
v = reflect.MakeMap(v.Type())
}
elem := reflect.New(v.Type().Elem()).Elem()
if existing := v.MapIndex(key); existing.IsValid() {
elem.Set(existing)
}
nv, err := d.resolveCapture(elem, c, idx+1, false)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
v.SetMapIndex(key, nv)
}
return v, nil
case reflect.Interface:
elem := elemOrNewMap(v)
nv, err := d.resolveCapture(elem, c, idx, indexed)
if err != nil || !nv.IsValid() {
return reflect.Value{}, err
}
return nv, nil
default:
return reflect.Value{}, fmt.Errorf("toml: internal error: cannot resolve capture target through %s", v.Kind())
}
}
// captureKeyValue appends the raw bytes of a key-value expression to the
// current capture.
func (d *decoder) captureKeyValue(expr *unstable.Node) {
c := &d.captures[d.captureIdx]
c.buf = append(c.buf, d.p.Raw(expr.Raw)...)
c.buf = append(c.buf, '\n')
}
// hasUnmarshaler reports whether v can provide an unstable.Unmarshaler,
// without allocating anything.
func hasUnmarshaler(v reflect.Value) bool {
t := v.Type()
return t.Implements(unmarshalerType) || (v.CanAddr() && reflect.PtrTo(t).Implements(unmarshalerType))
}
// makeMapKey converts a TOML key into a value usable as the given map key
// type.
func makeMapKey(kt reflect.Type, name string) (reflect.Value, error) {
switch kt.Kind() {
case reflect.String:
return reflect.ValueOf(name).Convert(kt), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
i, err := strconv.ParseInt(name, 10, 64)
if err != nil {
return reflect.Value{}, fmt.Errorf("toml: cannot parse map key %q as %s: %w", name, kt, err)
}
k := reflect.New(kt).Elem()
if k.OverflowInt(i) {
return reflect.Value{}, fmt.Errorf("toml: map key %q overflows %s", name, kt)
}
k.SetInt(i)
return k, nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
u, err := strconv.ParseUint(name, 10, 64)
if err != nil {
return reflect.Value{}, fmt.Errorf("toml: cannot parse map key %q as %s: %w", name, kt, err)
}
k := reflect.New(kt).Elem()
if k.OverflowUint(u) {
return reflect.Value{}, fmt.Errorf("toml: map key %q overflows %s", name, kt)
}
k.SetUint(u)
return k, nil
case reflect.Float32, reflect.Float64:
f, err := strconv.ParseFloat(name, 64)
if err != nil {
return reflect.Value{}, fmt.Errorf("toml: cannot parse map key %q as %s: %w", name, kt, err)
}
k := reflect.New(kt).Elem()
k.SetFloat(f)
return k, nil
case reflect.Ptr:
if kt.Implements(textUnmarshalerType) {
k := reflect.New(kt.Elem())
err := k.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(name))
if err != nil {
return reflect.Value{}, fmt.Errorf("toml: error unmarshaling map key %q: %w", name, err)
}
return k, nil
}
default:
if reflect.PtrTo(kt).Implements(textUnmarshalerType) {
k := reflect.New(kt)
err := k.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(name))
if err != nil {
return reflect.Value{}, fmt.Errorf("toml: error unmarshaling map key %q: %w", name, err)
}
return k.Elem(), nil
}
}
return reflect.Value{}, fmt.Errorf("toml: cannot decode a key into a map with key type %s", kt)
}
// elemOrNewMap unwraps an interface value to descend into it. Contents that
// can hold a table (generic maps and slices) are kept; anything else is
// replaced by a fresh map[string]interface{}. Maps and slices are reference
// types: they are returned directly, not copied.
func elemOrNewMap(v reflect.Value) reflect.Value {
if !v.IsNil() {
concrete := v.Elem()
t := concrete.Type()
if t == mapStringInterfaceType || t == sliceInterfaceType {
return concrete
}
}
return reflect.ValueOf(map[string]interface{}{})
}
// handleKeyValueExpression stores the value of a top-level key-value
// expression, relative to the current table.
func (d *decoder) handleKeyValueExpression(expr *unstable.Node, root reflect.Value) error {
d.path = d.path[:0]
target := root
useCache := d.tableTargetValid && len(d.tableKey) > 0
if useCache {
target = d.tableTarget
} else {
for _, name := range d.tableKey {
d.path = append(d.path, pathPart{name: name})
}
}
it := expr.Key()
for it.Next() {
d.path = append(d.path, pathPart{node: it.Node()})
}
nv, err := d.descend(target, d.path, 0, expr, expr.Value())
if err != nil {
return d.contextualizeError(err, useCache)
}
if !nv.IsValid() {
return nil
}
if useCache {
// The target may have been replaced (e.g. a nil map allocated):
// re-link it into its parent.
if nv.Kind() == reflect.Map && nv.Pointer() != d.tableTarget.Pointer() {
d.storeSlot(&d.tableParentSlot, nv)
d.tableTarget = nv
}
} else {
if root.CanSet() {
root.Set(nv)
}
}
return nil
}
// descend walks the given key path into v, and assigns the value at the
// end. It returns the value to store back at this level. An invalid value
// means nothing should be stored (e.g. unknown field).
func (d *decoder) descend(v reflect.Value, path []pathPart, idx int, expr *unstable.Node, value *unstable.Node) (reflect.Value, error) {
if idx == len(path) {
return d.assignValue(v, expr, value)
}
if v.Kind() == reflect.Ptr {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
nv, err := d.descend(v.Elem(), path, idx, expr, value)
if err != nil || !nv.IsValid() {
return reflect.Value{}, err
}
v.Elem().Set(nv)
return v, nil
}
// A target implementing the unmarshaler interface consumes the value,
// whatever the remaining parts of the key are.
if d.unmarshalerInterface {
if u, ok := unmarshalerOf(v); ok {
return v, u.UnmarshalTOML(d.rawValue(expr, value))
}
}
part := path[idx]
switch v.Kind() {
case reflect.Map:
// Native fast path for the most common generic target: walk the
// remaining dotted-key path with plain Go map operations and decode
// the value directly, skipping the reflect.Value round-trips
// (stringMapKey, MapIndex, New, SetMapIndex) entirely.
if !d.unmarshalerInterface && v.Type() == mapStringInterfaceType {
return d.descendStrMap(v, path, idx, value)
}
var name string
var key reflect.Value
var err error
fastKey := v.Type().Key() == stringType
if fastKey {
name = d.partString(&part)
key = d.stringMapKey(name)
} else {
key, err = makeMapKey(v.Type().Key(), d.partString(&part))
if err != nil {
return reflect.Value{}, err
}
}
if v.IsNil() {
v = reflect.MakeMap(v.Type())
}
elemType := v.Type().Elem()
existing := v.MapIndex(key)
var elem reflect.Value
switch {
case existing.IsValid():
elem = reflect.New(elemType).Elem()
elem.Set(existing)
case idx+1 == len(path) && elemType.Kind() == reflect.Interface:
// Fast path: a fresh interface element does not need to be
// materialized, the assigned value is stored directly.
elem = reflect.Zero(elemType)
default:
elem = reflect.New(elemType).Elem()
}
nv, err := d.descend(elem, path, idx+1, expr, value)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
if fastKey {
// The recursion may have overwritten the key buffer.
key = d.stringMapKey(name)
}
v.SetMapIndex(key, nv)
}
return v, nil
case reflect.Struct:
plan := planForType(v.Type())
f, found := plan.lookupBytes(part.bytes())
if !found {
if part.node != nil {
d.strict.MissingField(expr)
}
return v, nil
}
fv := fieldByIndexAlloc(v, f.index)
var nv reflect.Value
var err error
if idx+1 == len(path) {
// Leaf field: assign directly. descend's first action for a
// fully-consumed path is exactly this call, so skipping the extra
// frame is equivalent and avoids a call per scalar field.
nv, err = d.assignValue(fv, expr, value)
} else {
nv, err = d.descend(fv, path, idx+1, expr, value)
}
if err != nil {
var mm *typeMismatchError
if errors.As(err, &mm) {
err = &unstable.ParserError{
Highlight: mm.highlight,
Message: fmt.Sprintf("cannot decode TOML %s into struct field %s.%s of type %s",
mm.toml, v.Type(), f.fieldName, mm.target),
}
}
return reflect.Value{}, err
}
if nv.IsValid() && fv.CanSet() {
fv.Set(nv)
}
return v, nil
case reflect.Interface:
elem := elemOrNewMap(v)
nv, err := d.descend(elem, path, idx, expr, value)
if err != nil || !nv.IsValid() {
return reflect.Value{}, err
}
return nv, nil
case reflect.Slice:
if v.Len() == 0 {
if v.IsNil() {
v = reflect.MakeSlice(v.Type(), 0, 4)
}
v = reflect.Append(v, reflect.New(v.Type().Elem()).Elem())
}
elem := v.Index(v.Len() - 1)
nv, err := d.descend(elem, path, idx, expr, value)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
elem.Set(nv)
}
return v, nil
case reflect.Array:
names := make([]string, idx)
for i := range names {
names[i] = path[i].str()
}
cnt := d.arrayCount(d.joinPath(names))
if cnt == 0 {
cnt = 1
}
elemIdx := cnt - 1
if elemIdx >= v.Len() {
return reflect.Value{}, unstable.NewParserError(keyHighlight(d.p.Data(), part.node),
"cannot reach element %d of array of size %d", elemIdx, v.Len())
}
elem := v.Index(elemIdx)
nv, err := d.descend(elem, path, idx, expr, value)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
elem.Set(nv)
}
return v, nil
default:
return reflect.Value{}, d.typeMismatchError("table", v.Type(), keyHighlight(d.p.Data(), part.node))
}
}
// descendStrMap assigns into a native map[string]interface{} target, following
// the remaining dotted-key parts with plain Go map operations and decoding the
// value with decodeAny. It returns the map to store back at this level: a new
// map when v was nil, otherwise v unchanged, since maps are reference types and
// are mutated in place.
func (d *decoder) descendStrMap(v reflect.Value, path []pathPart, idx int, value *unstable.Node) (reflect.Value, error) {
var m map[string]interface{}
if v.IsNil() {
m = make(map[string]interface{})
v = reflect.ValueOf(m)
} else {
m = v.Interface().(map[string]interface{})
}
// Walk intermediate parts, creating or reusing nested generic maps. A
// non-map value at an intermediate key can only occur in a document the
// seen-tracker has already rejected; replacing it mirrors the reflect
// path (elemOrNewMap).
for ; idx < len(path)-1; idx++ {
name := d.partString(&path[idx])
child, _ := m[name].(map[string]interface{})
if child == nil {
child = make(map[string]interface{})
m[name] = child
}
m = child
}
av, err := d.decodeAny(value)
if err != nil {
return reflect.Value{}, err
}
m[d.partString(&path[idx])] = av
return v, nil
}
// keyHighlight returns a highlight for the given key part node, falling back
// to the start of the document.
func keyHighlight(doc []byte, node *unstable.Node) []byte {
if node == nil {
return doc[0:0]
}
return doc[node.Raw.Offset : node.Raw.Offset+node.Raw.Length]
}
// rawValue returns the raw bytes of the value of a key-value expression.
func (d *decoder) rawValue(expr *unstable.Node, value *unstable.Node) []byte {
if value.Kind != unstable.InlineTable && value.Kind != unstable.Array {
return d.p.Raw(value.Raw)
}
if expr == nil || expr.Kind != unstable.KeyValue {
// Inline container nested in another container: best effort.
return d.p.Raw(value.Raw)
}
// Reconstruct the span of the value: it starts after the equal sign
// following the last part of the key, and stops at the end of the
// expression.
var last unstable.Range
it := expr.Key()
for it.Next() {
last = it.Node().Raw
}
doc := d.p.Data()
i := int(last.Offset + last.Length)
for i < len(doc) && (doc[i] == ' ' || doc[i] == '\t') {
i++
}
i++ // equal sign
for i < len(doc) && (doc[i] == ' ' || doc[i] == '\t') {
i++
}
end := int(expr.Raw.Offset + expr.Raw.Length)
return doc[i:end]
}
// unmarshalerOf returns the unstable.Unmarshaler implementation of v, if
// any. It allocates intermediate pointers as needed.
func unmarshalerOf(v reflect.Value) (unstable.Unmarshaler, bool) {
t := v.Type()
if t.Implements(unmarshalerType) {
if v.Kind() == reflect.Ptr && v.IsNil() {
v.Set(reflect.New(t.Elem()))
}
return v.Interface().(unstable.Unmarshaler), true
}
if v.CanAddr() && reflect.PtrTo(t).Implements(unmarshalerType) {
return v.Addr().Interface().(unstable.Unmarshaler), true
}
return nil, false
}
var unmarshalerType = reflect.TypeOf(new(unstable.Unmarshaler)).Elem()
// assignValue stores the TOML value carried by the node into v.
func (d *decoder) assignValue(v reflect.Value, expr *unstable.Node, value *unstable.Node) (reflect.Value, error) {
if v.Kind() == reflect.Ptr {
if d.unmarshalerInterface {
if u, ok := unmarshalerOf(v); ok {
return v, u.UnmarshalTOML(d.rawValue(expr, value))
}
}
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
nv, err := d.assignValue(v.Elem(), expr, value)
if err != nil || !nv.IsValid() {
return reflect.Value{}, err
}
v.Elem().Set(nv)
return v, nil
}
if d.unmarshalerInterface {
if u, ok := unmarshalerOf(v); ok {
return v, u.UnmarshalTOML(d.rawValue(expr, value))
}
}
switch value.Kind {
case unstable.String:
return d.assignString(v, value)
case unstable.Integer:
return d.assignInteger(v, value)
case unstable.Float:
return d.assignFloat(v, value)
case unstable.Bool:
return d.assignBool(v, value)
case unstable.DateTime:
return d.assignDateTime(v, value)
case unstable.LocalDateTime:
return d.assignLocalDateTime(v, value)
case unstable.LocalDate:
return d.assignLocalDate(v, value)
case unstable.LocalTime:
return d.assignLocalTime(v, value)
case unstable.Array:
return d.assignArray(v, expr, value)
case unstable.InlineTable:
return d.assignInlineTable(v, expr, value)
default:
return reflect.Value{}, unstable.NewParserError(value.Data, "unsupported value kind %s", value.Kind)
}
}
func (d *decoder) assignString(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
switch v.Kind() {
case reflect.String:
v.SetString(string(value.Data))
return v, nil
case reflect.Interface:
return boxInto(v, reflect.ValueOf(string(value.Data)))
default:
}
if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) {
err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(value.Data)
if err != nil {
return reflect.Value{}, unstable.NewParserError(d.p.Raw(value.Raw), "%s", err)
}
return v, nil
}
return reflect.Value{}, d.typeMismatchError("string", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignInteger(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
// Integer values targeting a float field are parsed as floats: they can
// represent (approximately) numbers beyond the int64 range.
if k := v.Kind(); k == reflect.Float32 || k == reflect.Float64 {
return d.assignFloat(v, value)
}
i, err := parseInteger(value.Data)
if err != nil {
return reflect.Value{}, err
}
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if v.OverflowInt(i) {
return reflect.Value{}, unstable.NewParserError(value.Data, "integer value %d cannot be stored in %s", i, v.Type())
}
v.SetInt(i)
return v, nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if i < 0 {
return reflect.Value{}, unstable.NewParserError(value.Data, "negative integer value %d cannot be stored in %s", i, v.Type())
}
if v.OverflowUint(uint64(i)) {
return reflect.Value{}, unstable.NewParserError(value.Data, "integer value %d cannot be stored in %s", i, v.Type())
}
v.SetUint(uint64(i))
return v, nil
case reflect.Interface:
return boxInto(v, reflect.ValueOf(i))
default:
}
if ok, err := tryTextUnmarshaler(v, value.Data); ok {
return v, err
}
return reflect.Value{}, d.typeMismatchError("integer", v.Type(), d.p.Raw(value.Raw))
}
// tryTextUnmarshaler attempts to deliver the raw text of a value to a target
// implementing encoding.TextUnmarshaler.
func tryTextUnmarshaler(v reflect.Value, text []byte) (bool, error) {
if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) {
return true, v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(text)
}
return false, nil
}
func (d *decoder) assignFloat(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
f, err := parseFloat(value.Data)
if err != nil {
return reflect.Value{}, err
}
switch v.Kind() {
case reflect.Float64:
v.SetFloat(f)
return v, nil
case reflect.Float32:
if !math.IsInf(f, 0) && math.Abs(f) > math.MaxFloat32 {
return reflect.Value{}, unstable.NewParserError(value.Data, "float value %f cannot be stored in float32", f)
}
v.SetFloat(f)
return v, nil
case reflect.Interface:
return boxInto(v, reflect.ValueOf(f))
default:
}
if ok, err := tryTextUnmarshaler(v, value.Data); ok {
return v, err
}
return reflect.Value{}, d.typeMismatchError("float", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignBool(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
b := value.Data[0] == 't'
switch v.Kind() {
case reflect.Bool:
v.SetBool(b)
return v, nil
case reflect.Interface:
return boxInto(v, reflect.ValueOf(b))
default:
}
if ok, err := tryTextUnmarshaler(v, value.Data); ok {
return v, err
}
return reflect.Value{}, d.typeMismatchError("boolean", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignDateTime(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
t, err := parseDateTime(value.Data)
if err != nil {
return reflect.Value{}, err
}
if v.Type() == timeType {
v.Set(reflect.ValueOf(t))
return v, nil
}
if v.Kind() == reflect.Interface {
return boxInto(v, reflect.ValueOf(t))
}
return reflect.Value{}, d.typeMismatchError("datetime", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignLocalDateTime(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
dt, rest, err := parseLocalDateTime(value.Data)
if err != nil {
return reflect.Value{}, err
}
if len(rest) > 0 {
return reflect.Value{}, unstable.NewParserError(rest, "extra characters at the end of a local date time")
}
switch v.Type() {
case localDateTimeType:
v.Set(reflect.ValueOf(dt))
return v, nil
case timeType:
v.Set(reflect.ValueOf(dt.AsTime(time.Local)))
return v, nil
}
if v.Kind() == reflect.Interface {
return boxInto(v, reflect.ValueOf(dt))
}
return reflect.Value{}, d.typeMismatchError("local datetime", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignLocalDate(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
date, err := parseLocalDate(value.Data)
if err != nil {
return reflect.Value{}, err
}
switch v.Type() {
case localDateType:
v.Set(reflect.ValueOf(date))
return v, nil
case timeType:
v.Set(reflect.ValueOf(date.AsTime(time.Local)))
return v, nil
}
if v.Kind() == reflect.Interface {
return boxInto(v, reflect.ValueOf(date))
}
return reflect.Value{}, d.typeMismatchError("local date", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignLocalTime(v reflect.Value, value *unstable.Node) (reflect.Value, error) {
t, rest, err := parseLocalTime(value.Data)
if err != nil {
return reflect.Value{}, err
}
if len(rest) > 0 {
return reflect.Value{}, unstable.NewParserError(rest, "extra characters at the end of a local time")
}
switch v.Type() {
case localTimeType:
v.Set(reflect.ValueOf(t))
return v, nil
case timeType:
v.Set(reflect.ValueOf(time.Date(0, 1, 1, t.Hour, t.Minute, t.Second, t.Nanosecond, time.Local)))
return v, nil
}
if v.Kind() == reflect.Interface {
return boxInto(v, reflect.ValueOf(t))
}
return reflect.Value{}, d.typeMismatchError("local time", v.Type(), d.p.Raw(value.Raw))
}
func (d *decoder) assignArray(v reflect.Value, expr *unstable.Node, value *unstable.Node) (reflect.Value, error) {
// Count the elements to allocate the target in one go.
count := 0
cit := value.Children()
for cit.Next() {
if cit.Node().Kind != unstable.Comment {
count++
}
}
switch v.Kind() {
case reflect.Slice:
// Allocate the backing array once at its final length and assign each
// element in place. This avoids a reflect.New allocation per element
// and the repeated growth checks of reflect.Append.
slice := reflect.MakeSlice(v.Type(), count, count)
i := 0
it := value.Children()
for it.Next() {
n := it.Node()
if n.Kind == unstable.Comment {
continue
}
elem := slice.Index(i)
nv, err := d.assignValue(elem, nil, n)
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
elem.Set(nv)
}
i++
}
return slice, nil
case reflect.Array:
it := value.Children()
i := 0
for it.Next() {
n := it.Node()
if n.Kind == unstable.Comment {
continue
}
if i >= v.Len() {
// Extra elements are dropped when the target array is too
// small.
break
}
elem := v.Index(i)
nv, err := d.assignValue(elem, nil, n)
if err != nil {
return reflect.Value{}, err
}
elem.Set(nv)
i++
}
return v, nil
case reflect.Interface:
// Build the []interface{} natively: each element is decoded straight
// into a Go value with no intermediate addressable reflect.Value and
// no reflect round-trip, and nested arrays recurse the same way.
slice := make([]interface{}, 0, count)
it := value.Children()
for it.Next() {
n := it.Node()
if n.Kind == unstable.Comment {
continue
}
ev, err := d.decodeAny(n)
if err != nil {
return reflect.Value{}, err
}
slice = append(slice, ev)
}
return boxInto(v, reflect.ValueOf(slice))
default:
}
return reflect.Value{}, d.typeMismatchError("array", v.Type(), d.rawValue(expr, value))
}
// decodeAny decodes a value node into a native Go value (the representation
// used for interface{} targets), without going through reflect. Scalars and
// arrays are handled directly; inline tables still defer to the reflect-based
// path so that their dotted-key merge semantics remain identical.
func (d *decoder) decodeAny(n *unstable.Node) (interface{}, error) {
switch n.Kind {
case unstable.String:
return string(n.Data), nil
case unstable.Integer:
i, err := parseInteger(n.Data)
return i, err
case unstable.Float:
f, err := parseFloat(n.Data)
return f, err
case unstable.Bool:
return n.Data[0] == 't', nil
case unstable.Array:
count := 0
cit := n.Children()
for cit.Next() {
if cit.Node().Kind != unstable.Comment {
count++
}
}
slice := make([]interface{}, 0, count)
it := n.Children()
for it.Next() {
c := it.Node()
if c.Kind == unstable.Comment {
continue
}
ev, err := d.decodeAny(c)
if err != nil {
return nil, err
}
slice = append(slice, ev)
}
return slice, nil
case unstable.InlineTable:
// Build the map natively: navigate each (possibly dotted) key with
// plain Go map operations and decode each value with decodeAny. The
// seen-tracker has already rejected duplicate or conflicting keys, so
// intermediate parts can be created/merged without revalidation.
count := 0
cit := n.Children()
for cit.Next() {
count++
}
m := make(map[string]interface{}, count)
it := n.Children()
for it.Next() {
kv := it.Node()
if err := d.setAnyKey(m, kv.Key(), kv.Value()); err != nil {
return nil, err
}
}
return m, nil
case unstable.DateTime:
t, err := parseDateTime(n.Data)
return t, err
case unstable.LocalDateTime:
dt, rest, err := parseLocalDateTime(n.Data)
if err != nil {
return nil, err
}
if len(rest) > 0 {
return nil, unstable.NewParserError(rest, "extra characters at the end of a local date time")
}
return dt, nil
case unstable.LocalDate:
date, err := parseLocalDate(n.Data)
return date, err
case unstable.LocalTime:
t, rest, err := parseLocalTime(n.Data)
if err != nil {
return nil, err
}
if len(rest) > 0 {
return nil, unstable.NewParserError(rest, "extra characters at the end of a local time")
}
return t, nil
default:
return nil, unstable.NewParserError(n.Data, "unsupported value kind %s", n.Kind)
}
}
func (d *decoder) assignInlineTable(v reflect.Value, expr *unstable.Node, value *unstable.Node) (reflect.Value, error) {
switch v.Kind() {
case reflect.Map:
// Inline tables are self-contained: they fully replace the target.
v = reflect.MakeMap(v.Type())
case reflect.Struct:
// fields are set in place
case reflect.Interface:
elem := reflect.ValueOf(map[string]interface{}{})
nv, err := d.assignInlineTable(elem, expr, value)
if err != nil {
return reflect.Value{}, err
}
return boxInto(v, nv)
default:
return reflect.Value{}, d.typeMismatchError("inline table", v.Type(), d.rawValue(expr, value))
}
it := value.Children()
for it.Next() {
kv := it.Node()
// Build the path from the key of this key-value. Keys of inline
// tables rarely have more than a few parts.
var pathBuf [4]pathPart
path := pathBuf[:0]
kit := kv.Key()
for kit.Next() {
path = append(path, pathPart{node: kit.Node()})
}
nv, err := d.descend(v, path, 0, kv, kv.Value())
if err != nil {
return reflect.Value{}, err
}
if nv.IsValid() {
v = nv
}
}
return v, nil
}
// boxInto returns the value to store in place of the interface value v. The
// caller stores the result in the slot v was found in, which performs the
// interface conversion, so the concrete value can be returned as-is.
func boxInto(v reflect.Value, c reflect.Value) (reflect.Value, error) {
if !c.Type().AssignableTo(v.Type()) {
return reflect.Value{}, fmt.Errorf("toml: cannot store %s into %s", c.Type(), v.Type())
}
return c, nil
}
var (
interfaceType = reflect.TypeOf(new(interface{})).Elem()
localDateType = reflect.TypeOf(LocalDate{})
localTimeType = reflect.TypeOf(LocalTime{})
localDateTimeType = reflect.TypeOf(LocalDateTime{})
)
// structPlan caches the mapping between TOML keys and the fields of a struct
// type. byFold, keyed by the lowercased name, resolves any key on its own when
// no two fields fold to the same name (the overwhelmingly common case, marked
// by hasCollision == false): TOML keys are usually lowercase and never match
// the exact (capitalized) Go field names, so the byName probe was always a
// wasted lookup. byName (the exact names) is only consulted, first, when
// fields do collide under folding, to preserve the exact-match-wins tiebreak.
type structPlan struct {
byName map[string]structField
byFold map[string]structField
hasCollision bool
}
type structField struct {
index []int
fieldName string
}
// foldBufSize bounds the stack buffer used to lowercase keys without
// allocating. Keys longer than this (extremely rare) take the strings.ToLower
// fallback.
const foldBufSize = 68
// lookup and lookupBytes keep the hot path to a single inlinable byFold lookup.
// byFold is indexed by both the exact field/tag names and their lowercased
// forms, so that lookup resolves the two common cases — a lowercase key, or a
// key matching the field's own casing — directly. byName is consulted first
// only for types whose fields collide under case-folding, to preserve the
// exact-match-wins tiebreak. The buffer-fold for other casings lives
// out-of-line so it does not bloat the hot path.
func (p *structPlan) lookup(name string) (structField, bool) {
if p.hasCollision {
if f, ok := p.byName[name]; ok {
return f, true
}
}
if f, ok := p.byFold[name]; ok {
return f, true
}
return p.lookupFoldStr(name)
}
func (p *structPlan) lookupBytes(name []byte) (structField, bool) {
if p.hasCollision {
if f, ok := p.byName[string(name)]; ok { // does not allocate
return f, true
}
}
if f, ok := p.byFold[string(name)]; ok { // does not allocate
return f, true
}
return p.lookupFold(name)
}
// lookupFold resolves keys whose casing matches neither the exact nor the
// lowercased index: it folds to lowercase (in a stack buffer for ASCII, so no
// allocation) and retries; only non-ASCII or oversized keys hit strings.ToLower.
func (p *structPlan) lookupFold(name []byte) (structField, bool) {
if len(name) <= foldBufSize {
// Fold into a stack buffer: len(name) <= cap(buf), so the append
// never reallocates and nothing escapes to the heap.
var buf [foldBufSize]byte
b := buf[:0]
ascii := true
for _, c := range name {
if c >= 0x80 {
ascii = false
break
}
if c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
b = append(b, c)
}
if ascii {
f, ok := p.byFold[string(b)] // does not allocate
return f, ok
}
}
f, ok := p.byFold[strings.ToLower(string(name))]
return f, ok
}
func (p *structPlan) lookupFoldStr(name string) (structField, bool) {
if len(name) <= foldBufSize {
// Fold into a stack buffer: len(name) <= cap(buf), so the append
// never reallocates and nothing escapes to the heap.
var buf [foldBufSize]byte
b := buf[:0]
ascii := true
for i := 0; i < len(name); i++ {
c := name[i]
if c >= 0x80 {
ascii = false
break
}
if c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
b = append(b, c)
}
if ascii {
f, ok := p.byFold[string(b)] // does not allocate
return f, ok
}
}
f, ok := p.byFold[strings.ToLower(name)]
return f, ok
}
var structPlans sync.Map // reflect.Type -> *structPlan
func planForType(t reflect.Type) *structPlan {
if plan, ok := structPlans.Load(t); ok {
return plan.(*structPlan)
}
plan := buildPlan(t)
structPlans.Store(t, plan)
return plan
}
func buildPlan(t reflect.Type) *structPlan {
plan := &structPlan{
byName: map[string]structField{},
byFold: map[string]structField{},
}
addFields(plan, t, nil)
return plan
}
func addFields(plan *structPlan, t reflect.Type, prefix []int) {
var embedded []reflect.StructField
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
tag, tagged := f.Tag.Lookup("toml")
name := f.Name
explicitName := ""
if tagged {
// A tag of exactly "-" drops the field. "-," names it "-".
if tag == "-" {
continue
}
parts := strings.SplitN(tag, ",", 2)
explicitName = parts[0]
if explicitName != "" {
name = explicitName
}
}
if f.Anonymous {
ft := f.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
if ft.Kind() != reflect.Struct {
// Embedded non-struct fields are not decoded into.
continue
}
if explicitName == "" {
// Embedded structs without an explicit tag name are flattened,
// even when their type is unexported: only their own exported
// fields are reachable. A tag that only sets options (e.g.
// `,inline`) still flattens, matching encoding/json and the
// encoder's behavior.
embedded = append(embedded, f)
continue
}
// An embedded struct given an explicit tag name acts as a regular
// named field.
} else if f.PkgPath != "" {
// unexported
continue
}
index := make([]int, 0, len(prefix)+1)
index = append(index, prefix...)
index = append(index, i)
sf := structField{index: index, fieldName: f.Name}
if _, ok := plan.byName[name]; !ok {
plan.byName[name] = sf
}
lower := strings.ToLower(name)
if _, ok := plan.byFold[lower]; !ok {
plan.byFold[lower] = sf
} else {
// Two distinct fields fold to the same name: case-insensitive
// matching is ambiguous, so lookups must consult byName first to
// keep the exact-match-wins tiebreak deterministic.
plan.hasCollision = true
}
// Index the exact (cased) name as well, so a key written with the
// field's own casing resolves in a single byFold lookup. Only fields
// whose name is not already lowercase need this extra entry. Any name
// that would conflict here also collides under folding (handled
// above), so byName-first preserves the exact tiebreak in that case.
if name != lower {
if _, ok := plan.byFold[name]; !ok {
plan.byFold[name] = sf
}
}
}
// Embedded structs are flattened after the regular fields, so that
// shallower fields win.
for _, f := range embedded {
ft := f.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
index := make([]int, 0, len(prefix)+1)
index = append(index, prefix...)
idx := f.Index[0]
index = append(index, idx)
addFields(plan, ft, index)
}
}
// fieldByIndexAlloc returns the field of v at the given index path,
// allocating intermediate embedded pointers as needed.
func fieldByIndexAlloc(v reflect.Value, index []int) reflect.Value {
// Fast path for non-embedded fields, which have a single-element index:
// no intermediate pointer dereferencing is possible.
if len(index) == 1 {
return v.Field(index[0])
}
for i, x := range index {
if i > 0 {
for v.Kind() == reflect.Ptr {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
}
}
v = v.Field(x)
}
return v
}
package unstable
import (
"errors"
"fmt"
)
// Iterator over a sequence of nodes.
//
// Starts uninitialized, you need to call Next() first.
//
// For example:
//
// it := n.Children()
// for it.Next() {
// n := it.Node()
// // do something with n
// }
type Iterator struct {
started bool
node *Node
}
// Next moves the iterator forward and returns true if points to a node, false
// otherwise.
func (c *Iterator) Next() bool {
if !c.started {
c.started = true
} else if c.node.Valid() {
c.node = c.node.Next()
}
return c.node.Valid()
}
// IsLast returns true if the current node of the iterator is the last
// one. Subsequent calls to Next() will return false.
func (c *Iterator) IsLast() bool {
return c.node.next == 0
}
// Node returns a pointer to the node pointed at by the iterator.
func (c *Iterator) Node() *Node {
return c.node
}
// Node in a TOML expression AST.
//
// Depending on Kind, its sequence of children should be interpreted
// differently.
//
// - Array have one child per element in the array.
// - InlineTable have one child per key-value in the table (each of kind
// InlineTable).
// - KeyValue have at least two children. The first one is the value. The
// rest make a potentially dotted key.
// - Table and ArrayTable's children represent a dotted key (same as
// KeyValue, but without the first node being the value).
//
// When relevant, Raw describes the range of bytes this node is referring to in
// the input document. Use Parser.Raw() to retrieve the actual bytes.
type Node struct {
Kind Kind
Raw Range // Raw bytes from the input.
Data []byte // Node value (either allocated or referencing the input).
// References to other nodes, as 1-based indexes into the parser's arena.
// 0 means no node.
parser *Parser
next int32
child int32
}
// Next returns a pointer to the next node, or nil if there is no next node.
func (n *Node) Next() *Node {
if n.next == 0 {
return nil
}
return &n.parser.nodes[n.next-1]
}
// Child returns a pointer to the first child node of this node. Other children
// can be accessed calling Next on the first child. Returns nil if there is no
// child node.
func (n *Node) Child() *Node {
if n.child == 0 {
return nil
}
return &n.parser.nodes[n.child-1]
}
// Valid returns true if the node's kind is set (not to Invalid).
func (n *Node) Valid() bool {
return n != nil && n.Kind != Invalid
}
// Key returns the children nodes making the Key on a supported node. Panics
// otherwise. They are guaranteed to be all be of the Kind Key. A simple key
// would return just one element.
func (n *Node) Key() Iterator {
switch n.Kind {
case KeyValue:
value := n.Child()
if !value.Valid() {
panic(errors.New("KeyValue should have at least two children"))
}
return Iterator{node: value.Next()}
case Table, ArrayTable:
return Iterator{node: n.Child()}
default:
panic(fmt.Errorf("Key() is not supported on a %s", n.Kind))
}
}
// Value returns a pointer to the value node of a KeyValue.
// Guaranteed to be non-nil. Panics if not called on a KeyValue node,
// or if the Children are malformed.
func (n *Node) Value() *Node {
return n.Child()
}
// Children returns an iterator over a node's children.
func (n *Node) Children() Iterator {
return Iterator{node: n.Child()}
}
package unstable
import "github.com/pelletier/go-toml/v2/internal/parserbridge"
// Expose the non-AST scanners to the root toml package without committing to
// them in the public API. See internal/parserbridge for the rationale.
//
//nolint:gochecknoinits // load-time wiring of an internal bridge (see internal/parserbridge)
func init() {
parserbridge.ScanScalar = func(p any, b []byte) (kind int, raw, value, rest []byte, err error) {
k, raw, value, rest, err := p.(*Parser).scanScalar(b)
return int(k), raw, value, rest, err
}
parserbridge.ScanKey = func(p any, b []byte, dst [][]byte) (parts [][]byte, raw, rest []byte, err error) {
return p.(*Parser).scanKey(b, dst)
}
parserbridge.ScanComment = scanComment
parserbridge.ParseValue = func(p any, b []byte) (node any, rest []byte, err error) {
return p.(*Parser).parseValue(b)
}
}
package unstable
import "fmt"
// Kind represents the type of TOML structure contained in a given Node.
type Kind int
const (
// Invalid represents an invalid meta node.
Invalid Kind = iota
// Comment represents a comment meta node.
Comment
// Key represents a key meta node.
Key
// Table represents a top-level table.
Table
// ArrayTable represents a top-level array table.
ArrayTable
// KeyValue represents a top-level key value.
KeyValue
// Array represents an array container value.
Array
// InlineTable represents an inline table container value.
InlineTable
// String represents a string value.
String
// Bool represents a boolean value.
Bool
// Float represents a floating point value.
Float
// Integer represents an integer value.
Integer
// LocalDate represents a a local date value.
LocalDate
// LocalTime represents a local time value.
LocalTime
// LocalDateTime represents a local date/time value.
LocalDateTime
// DateTime represents a data/time value.
DateTime
)
// String implementation of fmt.Stringer.
func (k Kind) String() string {
switch k {
case Invalid:
return "Invalid"
case Comment:
return "Comment"
case Key:
return "Key"
case Table:
return "Table"
case ArrayTable:
return "ArrayTable"
case KeyValue:
return "KeyValue"
case Array:
return "Array"
case InlineTable:
return "InlineTable"
case String:
return "String"
case Bool:
return "Bool"
case Float:
return "Float"
case Integer:
return "Integer"
case LocalDate:
return "LocalDate"
case LocalTime:
return "LocalTime"
case LocalDateTime:
return "LocalDateTime"
case DateTime:
return "DateTime"
}
panic(fmt.Errorf("Kind.String() not implemented for kind %d", int(k)))
}
package unstable
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"unicode/utf8"
)
// ParserError describes an error relative to the content of the document.
//
// It cannot outlive the instance of Parser it refers to, and may cause panics
// if the parser is reset.
type ParserError struct {
Highlight []byte
Message string
Key []string // optional
}
// Error is the implementation of the error interface.
func (e *ParserError) Error() string {
return e.Message
}
// NewParserError is a convenience function to create a ParserError
//
// Warning: Highlight needs to be a subslice of Parser.data, so only slices
// returned by Parser.Raw are valid candidates.
func NewParserError(highlight []byte, format string, args ...interface{}) error {
return &ParserError{
Highlight: highlight,
Message: fmt.Errorf(format, args...).Error(),
}
}
// Parser scans over a TOML-encoded document and generates an iterative AST.
//
// To prime the Parser, first reset it with the contents of a TOML document.
// Then, process all top-level expressions sequentially. See Example.
//
// Don't forget to check Error() after you're done parsing.
//
// Each top-level expression needs to be fully processed before calling
// NextExpression() again. Otherwise, calls to various Node methods may panic
// if the parser has moved on the next expression.
//
// For performance reasons, go-toml doesn't make a copy of the input bytes to
// the parser. Make sure to copy all the bytes you need to outlive the slice
// given to the parser.
type Parser struct {
KeepComments bool
data []byte
left []byte
nodes []Node
err error
}
// Data returns the slice provided to the last call to Reset.
func (p *Parser) Data() []byte {
return p.data
}
// Range returns a range description that corresponds to a given slice of the
// input. If the argument is not a subslice of the parser input, this function
// panics.
func (p *Parser) Range(b []byte) Range {
// b is a subslice of p.data if and only if they share the same backing
// array. In that case, because subslicing cannot extend capacity, the
// number of bytes between the start of b and the end of the backing array
// (its capacity) identifies the offset of b within data.
offset := cap(p.data) - cap(b)
if offset < 0 || offset+len(b) > len(p.data) {
panic(errors.New("not a slice of the data slice"))
}
return Range{
Offset: uint32(offset), //nolint:gosec // TOML documents are small
Length: uint32(len(b)), //nolint:gosec // TOML documents are small
}
}
// Raw returns the slice corresponding to the bytes in the given range.
func (p *Parser) Raw(raw Range) []byte {
return p.data[raw.Offset : raw.Offset+raw.Length]
}
// Reset brings the parser to its initial state for a given input. It wipes an
// reuses internal storage to reduce allocation.
func (p *Parser) Reset(b []byte) {
p.data = b
p.left = b
p.nodes = p.nodes[:0]
p.err = nil
}
// Error returns any error that has occurred during parsing.
func (p *Parser) Error() error {
return p.err
}
// Range of bytes in the document.
type Range struct {
Offset uint32
Length uint32
}
// Position describes a position in the input.
type Position struct {
// Number of bytes from the beginning of the input.
Offset int
// Line number, starting at 1.
Line int
// Column number, starting at 1.
Column int
}
// Shape describes the position of a range in the input.
type Shape struct {
Start Position
End Position
}
func (p *Parser) position(offset int) Position {
pos := Position{
Offset: offset,
Line: 1,
Column: 1,
}
b := p.data[:offset]
for {
idx := bytes.IndexByte(b, '\n')
if idx < 0 {
break
}
pos.Line++
b = b[idx+1:]
}
pos.Column = len(b) + 1
return pos
}
// Shape returns the shape of the given range in the input. Will
// panic if the range is not a subslice of the input.
func (p *Parser) Shape(r Range) Shape {
raw := p.Raw(r)
return Shape{
Start: p.position(int(r.Offset)),
End: p.position(int(r.Offset) + len(raw)),
}
}
// Expression returns a pointer to the node representing the last successfully
// parsed expression.
func (p *Parser) Expression() *Node {
if len(p.nodes) == 0 {
return nil
}
return &p.nodes[0]
}
// push appends a node to the arena and returns its handle (1-based index).
func (p *Parser) push(n Node) int32 {
if len(p.nodes) == cap(p.nodes) {
// Grow by 2x: large expressions (huge arrays) would otherwise grow
// the arena in small steps, copying it repeatedly.
newCap := 2 * cap(p.nodes)
if newCap < 64 {
newCap = 64
}
nodes := make([]Node, len(p.nodes), newCap)
copy(nodes, p.nodes)
p.nodes = nodes
}
n.parser = p
p.nodes = append(p.nodes, n)
return int32(len(p.nodes)) //nolint:gosec // node counts are bounded by document size
}
// at returns a pointer to the node with the given handle. Only valid until
// the next call to push.
func (p *Parser) at(handle int32) *Node {
return &p.nodes[handle-1]
}
// offsetOf returns the offset of b within the parser's data. b must be a
// subslice of p.data.
func (p *Parser) offsetOf(b []byte) int {
return cap(p.data) - cap(b)
}
// rangeFrom returns the Range covering bytes from the start of `from` to the
// start of `to`. Both must be subslices of p.data.
func (p *Parser) rangeFrom(from, to []byte) Range {
start := p.offsetOf(from)
end := p.offsetOf(to)
return Range{
Offset: uint32(start), //nolint:gosec // TOML documents are small
Length: uint32(end - start), //nolint:gosec // TOML documents are small
}
}
// NextExpression parses the next top-level expression. If an expression was
// successfully parsed, it returns true. If the parser is at the end of the
// document or an error occurred, it returns false.
//
// Retrieve the parsed expression with Expression().
func (p *Parser) NextExpression() bool {
if p.err != nil {
return false
}
p.nodes = p.nodes[:0]
for {
b := skipWhitespace(p.left)
if len(b) == 0 {
p.left = b
return false
}
var err error
switch b[0] {
case '\n':
p.left = b[1:]
continue
case '\r':
if len(b) > 1 && b[1] == '\n' {
p.left = b[2:]
continue
}
err = NewParserError(b[:1], "expected newline but got %#U", b[0])
case '#':
var comment, rest []byte
comment, rest, err = scanComment(b)
if err == nil {
rest, err = consumeEOL(rest)
}
if err == nil {
if p.KeepComments {
p.push(Node{
Kind: Comment,
Raw: p.Range(comment),
Data: comment,
})
p.left = rest
return true
}
p.left = rest
continue
}
case '[':
var rest []byte
rest, err = p.parseExprTable(b)
if err == nil {
p.left = rest
return true
}
default:
var rest []byte
rest, err = p.parseExprKeyval(b)
if err == nil {
p.left = rest
return true
}
}
// Errors at the end of the input have an empty highlight. Extend
// them to the last byte of the input so that they carry a usable
// position.
var perr *ParserError
if errors.As(err, &perr) && len(perr.Highlight) == 0 {
if offset := p.offsetOf(perr.Highlight); offset > 0 && offset == len(p.data) {
perr.Highlight = p.data[offset-1 : offset]
}
}
p.err = err
return false
}
}
// consumeEOL consumes a newline (LF or CRLF) or end of input.
func consumeEOL(b []byte) ([]byte, error) {
if len(b) == 0 {
return b, nil
}
switch b[0] {
case '\n':
return b[1:], nil
case '\r':
if len(b) > 1 && b[1] == '\n' {
return b[2:], nil
}
}
return nil, NewParserError(b[:1], "expected newline but got %#U", b[0])
}
// finishLine handles `ws [comment] (newline|eof)` after a top-level
// expression. If a comment is present and KeepComments is set, it is attached
// as the next sibling of the expression's root node.
func (p *Parser) finishLine(root int32, b []byte) ([]byte, error) {
b = skipWhitespace(b)
if len(b) > 0 && b[0] == '#' {
comment, rest, err := scanComment(b)
if err != nil {
return nil, err
}
if p.KeepComments {
h := p.push(Node{
Kind: Comment,
Raw: p.Range(comment),
Data: comment,
})
p.at(root).next = h
}
b = rest
}
return consumeEOL(b)
}
// parseExprKeyval parses a top-level `key = value` expression, including its
// line termination.
func (p *Parser) parseExprKeyval(b []byte) ([]byte, error) {
root, rest, err := p.parseKeyval(b)
if err != nil {
return nil, err
}
return p.finishLine(root, rest)
}
// parseExprTable parses a `[table]` or `[[array table]]` expression,
// including its line termination. b starts at '['.
func (p *Parser) parseExprTable(b []byte) ([]byte, error) {
var root int32
var err error
var rest []byte
if len(b) > 1 && b[1] == '[' {
root, rest, err = p.parseArrayTableHeader(b)
} else {
root, rest, err = p.parseTableHeader(b)
}
if err != nil {
return nil, err
}
return p.finishLine(root, rest)
}
// parseTableHeader parses `[ ws key ws ]`. b starts at '['.
func (p *Parser) parseTableHeader(b []byte) (int32, []byte, error) {
root := p.push(Node{Kind: Table})
first, b, err := p.parseKey(skipWhitespace(b[1:]))
if err != nil {
return 0, nil, err
}
p.at(root).child = first
if len(b) == 0 || b[0] != ']' {
return 0, nil, NewParserError(highlight1(b), "expected ']' to close table name")
}
return root, b[1:], nil
}
// parseArrayTableHeader parses `[[ ws key ws ]]`. b starts at '[['.
func (p *Parser) parseArrayTableHeader(b []byte) (int32, []byte, error) {
root := p.push(Node{Kind: ArrayTable})
first, b, err := p.parseKey(skipWhitespace(b[2:]))
if err != nil {
return 0, nil, err
}
p.at(root).child = first
if len(b) < 2 || b[0] != ']' || b[1] != ']' {
return 0, nil, NewParserError(highlight1(b), "expected ']]' to close array table name")
}
return root, b[2:], nil
}
// parseKeyval parses `key keyval-sep val`. Returns the handle to the KeyValue
// node.
func (p *Parser) parseKeyval(b []byte) (int32, []byte, error) {
root := p.push(Node{Kind: KeyValue})
start := b
firstKey, b, err := p.parseKey(b)
if err != nil {
return 0, nil, err
}
if len(b) == 0 || b[0] != '=' {
return 0, nil, NewParserError(highlight1(b), "expected '=' after key")
}
b = skipWhitespace(b[1:])
value, b, err := p.parseVal(b)
if err != nil {
return 0, nil, err
}
p.at(root).child = value
p.at(value).next = firstKey
p.at(root).Raw = p.rangeFrom(start, b)
return root, b, nil
}
// parseKey parses a potentially dotted key. It consumes the whitespace
// following the key, so that the caller can directly check for the next
// expected character ('=', ']', ...). Returns the handle of the first Key
// node; subsequent parts are chained via next.
func (p *Parser) parseKey(b []byte) (int32, []byte, error) {
var first, last int32
for {
h, rest, err := p.parseSimpleKey(b)
if err != nil {
return 0, nil, err
}
if first == 0 {
first = h
} else {
p.at(last).next = h
}
last = h
b = skipWhitespace(rest)
if len(b) > 0 && b[0] == '.' {
b = skipWhitespace(b[1:])
continue
}
return first, b, nil
}
}
func isUnquotedKeyChar(c byte) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_'
}
// parseSimpleKey parses one key part: either a bare key or a quoted key.
func (p *Parser) parseSimpleKey(b []byte) (int32, []byte, error) {
raw, value, rest, err := p.scanSimpleKey(b)
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: Key, Raw: p.Range(raw), Data: value})
return h, rest, nil
}
// scanSimpleKey scans one key part (bare or quoted) without building an AST
// node. It returns the raw bytes, the decoded key value, and the rest of the
// input.
func (p *Parser) scanSimpleKey(b []byte) (raw, value, rest []byte, err error) {
if len(b) == 0 {
return nil, nil, nil, NewParserError(b, "expected key but reached end of input")
}
switch b[0] {
case '\'':
return p.parseLiteralString(b)
case '"':
return p.parseBasicString(b)
default:
i := 0
for i < len(b) && isUnquotedKeyChar(b[i]) {
i++
}
if i == 0 {
return nil, nil, nil, NewParserError(b[:1], "invalid character at start of key: %#U", b[0])
}
return b[:i], b[:i], b[i:], nil
}
}
// parseVal parses a TOML value and returns the handle to its node.
func (p *Parser) parseVal(b []byte) (int32, []byte, error) {
if len(b) == 0 {
return 0, nil, NewParserError(b, "expected value, not end of input")
}
c := b[0]
switch {
case c == '"':
var raw, value, rest []byte
var err error
if len(b) > 2 && b[1] == '"' && b[2] == '"' {
raw, value, rest, err = p.parseMultilineBasicString(b)
} else {
raw, value, rest, err = p.parseBasicString(b)
}
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: String, Raw: p.Range(raw), Data: value})
return h, rest, nil
case c == '\'':
var raw, value, rest []byte
var err error
if len(b) > 2 && b[1] == '\'' && b[2] == '\'' {
raw, value, rest, err = p.parseMultilineLiteralString(b)
} else {
raw, value, rest, err = p.parseLiteralString(b)
}
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: String, Raw: p.Range(raw), Data: value})
return h, rest, nil
case c == 't':
return p.parseKeyword(b, "true", Bool)
case c == 'f':
return p.parseKeyword(b, "false", Bool)
case c == 'i':
return p.parseKeyword(b, "inf", Float)
case c == 'n':
return p.parseKeyword(b, "nan", Float)
case c == '[':
return p.parseValArray(b)
case c == '{':
return p.parseInlineTable(b)
case c == '+' || c == '-':
return p.parseIntOrFloat(b)
case c >= '0' && c <= '9':
if isDateTimeStart(b) {
return p.parseDateTime(b)
}
return p.parseIntOrFloat(b)
default:
return 0, nil, NewParserError(b[:1], "unexpected character %#U at start of value", c)
}
}
// scanScalar scans a single scalar TOML value (string, integer, float,
// boolean, or date/time) without building any AST node. It returns the kind of
// the value, its raw bytes, its decoded value bytes (for strings: quotes
// removed and escapes resolved; identical to raw for the other kinds), and the
// rest of the input. Arrays and inline tables are not scalars and produce an
// error: use parseValue for those.
//
// It is exposed to the root toml package through internal/parserbridge for the
// fused generic-decode path; it is not part of the public API.
func (p *Parser) scanScalar(b []byte) (kind Kind, raw, value, rest []byte, err error) {
if len(b) == 0 {
return Invalid, nil, nil, nil, NewParserError(b, "expected value, not end of input")
}
c := b[0]
switch {
case c == '"':
if len(b) > 2 && b[1] == '"' && b[2] == '"' {
raw, value, rest, err = p.parseMultilineBasicString(b)
} else {
raw, value, rest, err = p.parseBasicString(b)
}
return String, raw, value, rest, err
case c == '\'':
if len(b) > 2 && b[1] == '\'' && b[2] == '\'' {
raw, value, rest, err = p.parseMultilineLiteralString(b)
} else {
raw, value, rest, err = p.parseLiteralString(b)
}
return String, raw, value, rest, err
case c == 't':
return scanKeyword(b, "true", Bool)
case c == 'f':
return scanKeyword(b, "false", Bool)
case c == 'i':
return scanKeyword(b, "inf", Float)
case c == 'n':
return scanKeyword(b, "nan", Float)
case c == '+' || c == '-':
return scanIntOrFloat(b)
case c >= '0' && c <= '9':
if isDateTimeStart(b) {
return scanDateTime(b)
}
return scanIntOrFloat(b)
default:
return Invalid, nil, nil, nil, NewParserError(b[:1], "unexpected character %#U at start of value", c)
}
}
// scanKey scans a potentially dotted key without building AST nodes,
// appending the decoded value of each part to dst (pass dst[:0] to reuse a
// buffer). It consumes the whitespace following the key, so the caller can
// directly check for the next expected character ('=', ']', ...). It returns
// the parts, the raw bytes spanning the whole key (from the first part to the
// end of the last one, excluding trailing whitespace, usable as an error
// highlight), the rest of the input, and any error.
//
// It is exposed to the root toml package through internal/parserbridge for the
// fused generic-decode path; it is not part of the public API.
func (p *Parser) scanKey(b []byte, dst [][]byte) (parts [][]byte, raw, rest []byte, err error) {
parts = dst
start := b
for {
_, value, r, err := p.scanSimpleKey(b)
if err != nil {
return nil, nil, nil, err
}
parts = append(parts, value)
// r points just past the current part: the key spans from start to
// here, ignoring any whitespace that follows.
raw = start[:len(start)-len(r)]
b = skipWhitespace(r)
if len(b) > 0 && b[0] == '.' {
b = skipWhitespace(b[1:])
continue
}
return parts, raw, b, nil
}
}
// parseValue parses a single TOML value, which may be an array or inline table,
// into the parser's arena. It returns the root node of the value and the rest
// of the input. It resets the arena, so any node returned by a previous call to
// parseValue, Expression, or NextExpression is invalidated.
//
// It is exposed to the root toml package through internal/parserbridge for the
// fused generic-decode path; it is not part of the public API.
func (p *Parser) parseValue(b []byte) (*Node, []byte, error) {
p.nodes = p.nodes[:0]
h, rest, err := p.parseVal(b)
if err != nil {
return nil, nil, err
}
return &p.nodes[h-1], rest, nil
}
func (p *Parser) parseKeyword(b []byte, kw string, kind Kind) (int32, []byte, error) {
k, raw, _, rest, err := scanKeyword(b, kw, kind)
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: k, Raw: p.Range(raw), Data: raw})
return h, rest, nil
}
// scanKeyword scans a keyword value (true, false, inf, nan) without building
// an AST node. raw and value are identical (the keyword bytes).
func scanKeyword(b []byte, kw string, kind Kind) (Kind, []byte, []byte, []byte, error) {
if len(b) < len(kw) || string(b[:len(kw)]) != kw {
n := len(kw)
if len(b) < n {
n = len(b)
}
return Invalid, nil, nil, nil, NewParserError(b[:n], "expected keyword %q", kw)
}
return kind, b[:len(kw)], b[:len(kw)], b[len(kw):], nil
}
// parseValArray parses an array value. b starts at '['.
func (p *Parser) parseValArray(b []byte) (int32, []byte, error) {
arr := p.push(Node{Kind: Array})
b = b[1:]
var lastChild int32
appendChild := func(h int32) {
if lastChild == 0 {
p.at(arr).child = h
} else {
p.at(lastChild).next = h
}
lastChild = h
}
// Comments inside the array are attached as follows: the first comment
// of a "run" (consecutive comments with no value in between) becomes a
// child of the array, interleaved with values; subsequent comments of the
// run are attached as children of the first one.
var runFirst, runLast int32
// afterValue is true when a value has been parsed and a comma (or the
// closing bracket) is expected before the next one.
afterValue := false
for {
b = skipWhitespace(b)
if len(b) == 0 {
return 0, nil, NewParserError(b, "array is incomplete")
}
switch b[0] {
case ']':
return arr, b[1:], nil
case '\n':
b = b[1:]
continue
case '\r':
if len(b) > 1 && b[1] == '\n' {
b = b[2:]
continue
}
return 0, nil, NewParserError(b[:1], "expected newline but got %#U", b[0])
case '#':
comment, rest, err := scanComment(b)
if err != nil {
return 0, nil, err
}
if p.KeepComments {
h := p.push(Node{Kind: Comment, Raw: p.Range(comment), Data: comment})
switch {
case runFirst == 0:
appendChild(h)
runFirst = h
case runLast == runFirst:
p.at(runFirst).child = h
default:
p.at(runLast).next = h
}
runLast = h
}
b = rest
continue
case ',':
if !afterValue {
return 0, nil, NewParserError(b[:1], "expected value but got %#U", b[0])
}
afterValue = false
b = b[1:]
continue
default:
if afterValue {
return 0, nil, NewParserError(b[:1], "expected ',' or ']' after array value")
}
h, rest, err := p.parseVal(b)
if err != nil {
return 0, nil, err
}
appendChild(h)
afterValue = true
runFirst, runLast = 0, 0
b = rest
continue
}
}
}
// parseInlineTable parses an inline table value. b starts at '{'.
//
// Per TOML v1.1.0, inline tables may span multiple lines (whitespace,
// comments and newlines are allowed between elements) and may contain a
// trailing comma.
func (p *Parser) parseInlineTable(b []byte) (int32, []byte, error) {
tbl := p.push(Node{Kind: InlineTable, Raw: p.Range(b[:1])})
b = b[1:]
var lastChild int32
appendChild := func(h int32) {
if lastChild == 0 {
p.at(tbl).child = h
} else {
p.at(lastChild).next = h
}
lastChild = h
}
// Comments are attached as in arrays: the first comment of a "run"
// (consecutive comments with no key-value in between) becomes a child of
// the table, interleaved with key-values; subsequent comments of the run
// hang off the first one.
var runFirst, runLast int32
// afterValue is true when a key-value has been parsed and a comma (or the
// closing brace) is expected before the next one.
afterValue := false
for {
b = skipWhitespace(b)
if len(b) == 0 {
return 0, nil, NewParserError(b, "inline table is incomplete")
}
switch b[0] {
case '}':
return tbl, b[1:], nil
case '\n':
b = b[1:]
continue
case '\r':
if len(b) > 1 && b[1] == '\n' {
b = b[2:]
continue
}
return 0, nil, NewParserError(b[:1], "expected newline but got %#U", b[0])
case '#':
comment, rest, err := scanComment(b)
if err != nil {
return 0, nil, err
}
if p.KeepComments {
h := p.push(Node{Kind: Comment, Raw: p.Range(comment), Data: comment})
switch {
case runFirst == 0:
appendChild(h)
runFirst = h
case runLast == runFirst:
p.at(runFirst).child = h
default:
p.at(runLast).next = h
}
runLast = h
}
b = rest
continue
case ',':
if !afterValue {
return 0, nil, NewParserError(b[:1], "unexpected comma in inline table")
}
afterValue = false
b = b[1:]
continue
default:
if afterValue {
return 0, nil, NewParserError(b[:1], "expected ',' or '}' after inline table key-value")
}
h, rest, err := p.parseKeyval(b)
if err != nil {
return 0, nil, err
}
appendChild(h)
afterValue = true
runFirst, runLast = 0, 0
b = rest
continue
}
}
}
func isDigit(c byte) bool {
return c >= '0' && c <= '9'
}
func isHexDigit(c byte) bool {
return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
// isDateTimeStart reports whether b looks like the start of a date or time
// value instead of a number. Values starting with two digits followed by a
// colon are times; values starting with four digits followed by a dash are
// dates.
func isDateTimeStart(b []byte) bool {
if len(b) >= 3 && isDigit(b[1]) && b[2] == ':' {
return true
}
if len(b) >= 5 && isDigit(b[1]) && isDigit(b[2]) && isDigit(b[3]) && b[4] == '-' {
return true
}
return false
}
// expectDigits checks that the n first bytes of b are digits.
// parseDateTime parses date and/or time values. b starts with a digit.
//
// The parser is lenient: it scans the characters that can be part of a date
// and/or time value to delimit and classify the token, but leaves the
// validation of its contents to the document consumer. This keeps the value
// in one piece, so that errors about its content can point at the right
// place.
func (p *Parser) parseDateTime(b []byte) (int32, []byte, error) {
kind, raw, _, rest, err := scanDateTime(b)
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: kind, Raw: p.Range(raw), Data: raw})
return h, rest, nil
}
// scanDateTime classifies and delimits a date/time value without building an
// AST node. raw and value are identical (the token bytes).
func scanDateTime(b []byte) (Kind, []byte, []byte, []byte, error) {
// Greedily scan the characters that may compose a date/time value. A
// space is part of the value only when it serves as the delimiter
// between the date and the time, which is approximated by requiring a
// digit right after it.
i := 0
delim := -1
for i < len(b) {
c := b[i]
if isDigit(c) || c == ':' || c == '-' || c == '+' || c == '.' || c == 'Z' || c == 'z' {
i++
continue
}
if c == 'T' || c == 't' || (c == ' ' && i+1 < len(b) && isDigit(b[i+1])) {
if delim < 0 {
delim = i
}
i++
continue
}
break
}
tok := b[:i]
var kind Kind
switch {
case tok[2] == ':':
kind = LocalTime
case delim < 0:
kind = LocalDate
case bytes.ContainsAny(tok[delim+1:], "Zz+-"):
kind = DateTime
default:
kind = LocalDateTime
}
return kind, tok, tok, b[i:], nil
}
// scanDigitsWithUnderscores scans a run of digits potentially separated by
// underscores. b starts right after the first digit of the run. isInRange
// selects the kind of digits. Returns the index after the run.
func scanDigitsWithUnderscores(b []byte, i int, isInRange func(byte) bool) (int, error) {
for i < len(b) {
c := b[i]
if isInRange(c) {
i++
continue
}
if c == '_' {
if i+1 >= len(b) || !isInRange(b[i+1]) {
end := i + 2
if end > len(b) {
end = len(b)
}
return 0, NewParserError(b[i:end], "number must have at least one digit between underscores")
}
i += 2
continue
}
break
}
return i, nil
}
// parseIntOrFloat parses integer and float values, including the special
// values inf and nan with an optional sign.
func (p *Parser) parseIntOrFloat(b []byte) (int32, []byte, error) {
kind, raw, _, rest, err := scanIntOrFloat(b)
if err != nil {
return 0, nil, err
}
h := p.push(Node{Kind: kind, Raw: p.Range(raw), Data: raw})
return h, rest, nil
}
// scanIntOrFloat delimits and classifies an integer or float value (including
// the special floats inf and nan with an optional sign) without building an
// AST node. raw and value are identical (the token bytes).
func scanIntOrFloat(b []byte) (Kind, []byte, []byte, []byte, error) {
i := 0
if b[i] == '+' || b[i] == '-' {
i++
}
if i >= len(b) {
return Invalid, nil, nil, nil, NewParserError(b, "expected number after sign")
}
// special floats
if b[i] == 'i' || b[i] == 'n' {
kw := "inf"
if b[i] == 'n' {
kw = "nan"
}
if len(b) < i+3 || string(b[i:i+3]) != kw {
return Invalid, nil, nil, nil, NewParserError(b[i:i+1], "expected %q", kw)
}
i += 3
return Float, b[:i], b[:i], b[i:], nil
}
if !isDigit(b[i]) {
return Invalid, nil, nil, nil, NewParserError(b[i:i+1], "expected digit but got %#U", b[i])
}
// radix prefixes
if b[i] == '0' && i+1 < len(b) && (b[i+1] == 'x' || b[i+1] == 'o' || b[i+1] == 'b') {
if i != 0 {
return Invalid, nil, nil, nil, NewParserError(b[:2], "sign is not allowed on numbers with a radix prefix")
}
var isInRange func(byte) bool
switch b[1] {
case 'x':
isInRange = isHexDigit
case 'o':
isInRange = func(c byte) bool { return c >= '0' && c <= '7' }
case 'b':
isInRange = func(c byte) bool { return c == '0' || c == '1' }
}
i = 2
if i >= len(b) || !isInRange(b[i]) {
return Invalid, nil, nil, nil, NewParserError(b[:2], "radix prefix must be followed by at least one digit")
}
i++
var err error
i, err = scanDigitsWithUnderscores(b, i, isInRange)
if err != nil {
return Invalid, nil, nil, nil, err
}
return Integer, b[:i], b[:i], b[i:], nil
}
// decimal integer part
leadingZero := b[i] == '0'
digitsStart := i
i++
var err error
i, err = scanDigitsWithUnderscores(b, i, isDigit)
if err != nil {
return Invalid, nil, nil, nil, err
}
if leadingZero && i > digitsStart+1 {
return Invalid, nil, nil, nil, NewParserError(b[digitsStart:digitsStart+2], "integers cannot have leading zeroes")
}
kind := Integer
// fractional part
if i < len(b) && b[i] == '.' {
i++
if i >= len(b) || !isDigit(b[i]) {
return Invalid, nil, nil, nil, NewParserError(highlight1(b[i:]), "decimal point must be followed by a digit")
}
i++
i, err = scanDigitsWithUnderscores(b, i, isDigit)
if err != nil {
return Invalid, nil, nil, nil, err
}
kind = Float
}
// exponent
if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
i++
if i < len(b) && (b[i] == '+' || b[i] == '-') {
i++
}
if i >= len(b) || !isDigit(b[i]) {
return Invalid, nil, nil, nil, NewParserError(highlight1(b[i:]), "exponent must contain at least one digit")
}
i++
i, err = scanDigitsWithUnderscores(b, i, isDigit)
if err != nil {
return Invalid, nil, nil, nil, err
}
kind = Float
}
// A letter right after the number means it was meant to be a string that
// was left unquoted (e.g. "20s"). Report that instead of the misleading
// "expected newline" raised later (issue #413).
if i < len(b) {
if c := b[i]; (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
return Invalid, nil, nil, nil, NewParserError(b[i:i+1], "strings must be quoted")
}
}
return kind, b[:i], b[:i], b[i:], nil
}
// highlight1 returns a 1-byte highlight at the start of b, or b itself if it
// is empty.
func highlight1(b []byte) []byte {
if len(b) > 0 {
return b[:1]
}
return b
}
func skipWhitespace(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}
// Word-at-a-time byte scanning helpers. These detect, within an 8-byte word,
// the presence of bytes that need special handling, so that runs of plain
// ASCII characters can be skipped 8 bytes at a time.
const (
lsb = 0x0101010101010101
msb = 0x8080808080808080
)
// hasByteBelow reports whether any byte of the word x is strictly below n.
// Only meaningful when combined with a check that no byte has its high bit
// set.
func hasByteBelow(x uint64, n uint64) uint64 {
return (x - n*lsb) & ^x & msb
}
// hasByteEqual reports whether any byte of the word x equals c. Only
// meaningful for bytes without their high bit set.
func hasByteEqual(x uint64, c uint64) uint64 {
y := x ^ (c * lsb)
return (y - lsb) & ^y & msb
}
// scanComment parses a comment, starting at the '#' character. It returns the
// comment bytes (including '#', excluding the line ending) and the rest of
// the input.
func scanComment(b []byte) ([]byte, []byte, error) {
i := 1
for i < len(b) {
// Fast path: skip 8 bytes at a time as long as they are all plain
// printable ASCII.
for i+8 <= len(b) {
x := binary.LittleEndian.Uint64(b[i:])
if (x&msb)|hasByteBelow(x, 0x20)|hasByteEqual(x, 0x7f) != 0 {
break
}
i += 8
}
if i >= len(b) {
break
}
c := b[i]
if c >= 0x80 {
var ok bool
i, ok = scanUtf8Run(b, i)
if !ok {
return nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in comment")
}
continue
}
switch {
case c >= 0x20 && c < 0x7f:
i++
case c == '\n':
return b[:i], b[i:], nil
case c == '\r':
if i+1 < len(b) && b[i+1] == '\n' {
return b[:i], b[i:], nil
}
return nil, nil, NewParserError(b[i:i+1], "carriage returns are not allowed in comments")
case c == '\t':
i++
default:
return nil, nil, NewParserError(b[i:i+1], "control characters are not allowed in comments")
}
}
return b[:i], b[i:], nil
}
// parseLiteralString parses a single-line literal string, starting at the
// opening quote. Returns the raw bytes (with quotes), the string value
// (without quotes) and the rest of the input.
func (p *Parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) {
i := 1
for {
// Fast path over plain ASCII.
for i+8 <= len(b) {
x := binary.LittleEndian.Uint64(b[i:])
if (x&msb)|hasByteBelow(x, 0x20)|hasByteEqual(x, '\'')|hasByteEqual(x, 0x7f) != 0 {
break
}
i += 8
}
if i >= len(b) {
return nil, nil, nil, NewParserError(b[len(b):], "unterminated literal string")
}
c := b[i]
switch {
case c == '\'':
return b[:i+1], b[1:i], b[i+1:], nil
case c >= 0x20 && c < 0x7f:
i++
case c == '\t':
i++
case c == '\n' || c == '\r':
return nil, nil, nil, NewParserError(b[i:i+1], "literal strings cannot have new lines")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "literal strings cannot have control characters")
default:
var ok bool
i, ok = scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in literal string")
}
}
}
}
// parseMultilineLiteralString parses a multi-line literal string, starting at
// the opening triple quote.
func (p *Parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) {
i := 3
// trim the newline right after the opening delimiter
if i < len(b) && b[i] == '\n' {
i++
} else if i+1 < len(b) && b[i] == '\r' && b[i+1] == '\n' {
i += 2
}
contentStart := i
for i < len(b) {
c := b[i]
switch {
case c == '\'':
// count consecutive quotes
j := i
for j < len(b) && b[j] == '\'' {
j++
}
n := j - i
if n >= 3 {
if n > 5 {
return nil, nil, nil, NewParserError(b[i:j], "too many quotes at the end of a multiline literal string")
}
// n-3 quotes belong to the content; the last 3 close the
// string.
contentEnd := i + n - 3
return b[:j], b[contentStart:contentEnd], b[j:], nil
}
i = j
case c >= 0x20 && c < 0x7f:
i++
case c == '\t' || c == '\n':
i++
case c == '\r':
if i+1 < len(b) && b[i+1] == '\n' {
i += 2
continue
}
return nil, nil, nil, NewParserError(b[i:i+1], "carriage returns must be followed by a newline character")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "multiline literal strings cannot have control characters")
default:
var ok bool
i, ok = scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in multiline literal string")
}
}
}
return nil, nil, nil, NewParserError(b[len(b):], "multiline literal string not terminated by '''")
}
// parseBasicString parses a single-line basic string, starting at the opening
// quote. The value is a subslice of the input if the string contains no
// escape sequence, or a new allocation otherwise.
func (p *Parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
i := 1
// First pass: handle strings without escape sequences without allocating.
for {
for i+8 <= len(b) {
x := binary.LittleEndian.Uint64(b[i:])
if (x&msb)|hasByteBelow(x, 0x20)|hasByteEqual(x, '"')|hasByteEqual(x, '\\')|hasByteEqual(x, 0x7f) != 0 {
break
}
i += 8
}
if i >= len(b) {
return nil, nil, nil, NewParserError(b[len(b):], "unterminated basic string")
}
c := b[i]
switch {
case c == '"':
return b[:i+1], b[1:i], b[i+1:], nil
case c == '\\':
// switch to the escape-aware parser, copying what has been
// scanned so far
return p.parseBasicStringEscaped(b, i)
case c >= 0x20 && c < 0x7f:
i++
case c == '\t':
i++
case c == '\n' || c == '\r':
return nil, nil, nil, NewParserError(b[i:i+1], "basic strings cannot have new lines")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "basic strings cannot have control characters")
default:
var ok bool
i, ok = scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in basic string")
}
}
}
}
// findBasicStringEnd returns the index of the quote closing a basic string,
// or -1 if the string is not terminated. i is the index of the first
// character after the opening quote. It does not validate the content: it
// only skips over escape sequences so that escaped quotes do not terminate
// the string.
func findBasicStringEnd(b []byte, i int) int {
for i < len(b) {
switch b[i] {
case '"':
return i
case '\\':
i += 2
default:
i++
}
}
return -1
}
// parseBasicStringEscaped continues parsing a basic string that contains
// escape sequences. i is the index of the first backslash.
func (p *Parser) parseBasicStringEscaped(b []byte, i int) ([]byte, []byte, []byte, error) {
// Escape sequences only ever shrink, so the content length before
// unescaping is enough to never reallocate.
bufCap := len(b) - 1
if end := findBasicStringEnd(b, i); end >= 0 {
bufCap = end - 1
}
buf := make([]byte, i-1, bufCap)
copy(buf, b[1:i])
for i < len(b) {
c := b[i]
switch {
case c == '"':
return b[:i+1], buf, b[i+1:], nil
case c == '\\':
i++
if i >= len(b) {
return nil, nil, nil, NewParserError(b[i-1:], `need a character after \`)
}
var err error
buf, i, err = unescape(buf, b, i)
if err != nil {
return nil, nil, nil, err
}
case c >= 0x20 && c < 0x7f:
buf = append(buf, c)
i++
case c == '\t':
buf = append(buf, c)
i++
case c == '\n' || c == '\r':
return nil, nil, nil, NewParserError(b[i:i+1], "basic strings cannot have new lines")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "basic strings cannot have control characters")
default:
j, ok := scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in basic string")
}
buf = append(buf, b[i:j]...)
i = j
}
}
return nil, nil, nil, NewParserError(b[len(b):], "unterminated basic string")
}
// unescape processes one escape sequence. i is the index of the character
// right after the backslash. It returns the updated buffer and index.
func unescape(buf []byte, b []byte, i int) ([]byte, int, error) {
c := b[i]
switch c {
case '"':
return append(buf, '"'), i + 1, nil
case '\\':
return append(buf, '\\'), i + 1, nil
case 'b':
return append(buf, '\b'), i + 1, nil
case 'f':
return append(buf, '\f'), i + 1, nil
case 'n':
return append(buf, '\n'), i + 1, nil
case 'r':
return append(buf, '\r'), i + 1, nil
case 't':
return append(buf, '\t'), i + 1, nil
case 'e':
// TOML v1.1.0: \e is the escape character (U+001B).
return append(buf, 0x1B), i + 1, nil
case 'x':
// TOML v1.1.0: \xHH is a two-digit hexadecimal code point.
return unescapeUnicode(buf, b, i+1, 2)
case 'u':
return unescapeUnicode(buf, b, i+1, 4)
case 'U':
return unescapeUnicode(buf, b, i+1, 8)
default:
return nil, 0, NewParserError(b[i-1:i+1], "invalid escape character %#U", c)
}
}
// unescapeUnicode handles \uXXXX and \UXXXXXXXX escape sequences. i is the
// index of the first hex digit.
func unescapeUnicode(buf []byte, b []byte, i int, n int) ([]byte, int, error) {
if i+n > len(b) {
return nil, 0, NewParserError(b[i-2:], "unicode escape sequence is too short")
}
var r uint32
for k := 0; k < n; k++ {
c := b[i+k]
var d uint32
switch {
case c >= '0' && c <= '9':
d = uint32(c - '0')
case c >= 'a' && c <= 'f':
d = uint32(c-'a') + 10
case c >= 'A' && c <= 'F':
d = uint32(c-'A') + 10
default:
return nil, 0, NewParserError(b[i+k:i+k+1], "invalid hexadecimal digit in unicode escape sequence")
}
r = r<<4 | d
}
if r > utf8.MaxRune || (r >= 0xD800 && r <= 0xDFFF) {
return nil, 0, NewParserError(b[i-2:i+n], "escape sequence is not a valid unicode code point")
}
return utf8.AppendRune(buf, rune(r)), i + n, nil
}
// parseMultilineBasicString parses a multi-line basic string, starting at the
// opening triple quote.
func (p *Parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, error) {
i := 3
// trim the newline right after the opening delimiter
if i < len(b) && b[i] == '\n' {
i++
} else if i+1 < len(b) && b[i] == '\r' && b[i+1] == '\n' {
i += 2
}
contentStart := i
// First pass without allocating, until an escape sequence is found.
for i < len(b) {
c := b[i]
switch {
case c == '"':
j := i
for j < len(b) && b[j] == '"' {
j++
}
n := j - i
if n >= 3 {
if n > 5 {
return nil, nil, nil, NewParserError(b[i:j], "too many quotes at the end of a multiline basic string")
}
contentEnd := i + n - 3
return b[:j], b[contentStart:contentEnd], b[j:], nil
}
i = j
case c == '\\':
return p.parseMultilineBasicStringEscaped(b, contentStart, i)
case c >= 0x20 && c < 0x7f:
i++
case c == '\t' || c == '\n':
i++
case c == '\r':
if i+1 < len(b) && b[i+1] == '\n' {
i += 2
continue
}
return nil, nil, nil, NewParserError(b[i:i+1], "carriage returns must be followed by a newline character")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "multiline basic strings cannot have control characters")
default:
var ok bool
i, ok = scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in multiline basic string")
}
}
}
return nil, nil, nil, NewParserError(b[len(b):], `multiline basic string not terminated by """`)
}
// findMultilineBasicStringEnd returns the index of the first quote of the
// run of quotes closing a multi-line basic string, or -1 if the string is
// not terminated. It does not validate the content: it only skips over
// escape sequences so that escaped quotes do not terminate the string.
func findMultilineBasicStringEnd(b []byte, i int) int {
for {
j := bytes.IndexAny(b[i:], "\"\\")
if j < 0 {
return -1
}
i += j
if b[i] == '\\' {
i += 2
if i > len(b) {
return -1
}
continue
}
j = i
for j < len(b) && b[j] == '"' {
j++
}
if j-i >= 3 {
return i
}
i = j
}
}
// parseMultilineBasicStringEscaped continues parsing a multi-line basic
// string that contains escape sequences. i is the index of the first
// backslash; content starts at contentStart.
func (p *Parser) parseMultilineBasicStringEscaped(b []byte, contentStart, i int) ([]byte, []byte, []byte, error) {
// Escape sequences only ever shrink, so the content length before
// unescaping is enough to never reallocate. The closing run of quotes
// can lend up to two quotes to the content.
bufCap := len(b) - contentStart
if end := findMultilineBasicStringEnd(b, i); end >= 0 {
bufCap = end + 2 - contentStart
}
buf := make([]byte, i-contentStart, bufCap)
copy(buf, b[contentStart:i])
for i < len(b) {
c := b[i]
switch {
case c == '"':
j := i
for j < len(b) && b[j] == '"' {
j++
}
n := j - i
if n >= 3 {
if n > 5 {
return nil, nil, nil, NewParserError(b[i:j], "too many quotes at the end of a multiline basic string")
}
buf = append(buf, b[i:i+n-3]...)
return b[:j], buf, b[j:], nil
}
buf = append(buf, b[i:j]...)
i = j
case c == '\\':
i++
if i >= len(b) {
return nil, nil, nil, NewParserError(b[i-1:], `need a character after \`)
}
// Escaped newline: backslash, optional whitespace, newline,
// then all following whitespace and newlines are trimmed.
if b[i] == ' ' || b[i] == '\t' || b[i] == '\n' || b[i] == '\r' {
j := i
for j < len(b) && (b[j] == ' ' || b[j] == '\t') {
j++
}
if j < len(b) && b[j] == '\r' {
j++
}
if j >= len(b) || b[j] != '\n' {
return nil, nil, nil, NewParserError(b[i-1:i+1], "invalid escape character %#U", b[i])
}
j++
for j < len(b) && (b[j] == ' ' || b[j] == '\t' || b[j] == '\n' || b[j] == '\r') {
// note: a lone \r not followed by \n will be caught on
// the next iteration of the outer loop.
if b[j] == '\r' {
if j+1 >= len(b) || b[j+1] != '\n' {
break
}
j++
}
j++
}
i = j
continue
}
var err error
buf, i, err = unescape(buf, b, i)
if err != nil {
return nil, nil, nil, err
}
case c >= 0x20 && c < 0x7f:
buf = append(buf, c)
i++
case c == '\t' || c == '\n':
buf = append(buf, c)
i++
case c == '\r':
if i+1 < len(b) && b[i+1] == '\n' {
buf = append(buf, '\r', '\n')
i += 2
continue
}
return nil, nil, nil, NewParserError(b[i:i+1], "carriage returns must be followed by a newline character")
case c < 0x80:
return nil, nil, nil, NewParserError(b[i:i+1], "multiline basic strings cannot have control characters")
default:
j, ok := scanUtf8Run(b, i)
if !ok {
return nil, nil, nil, NewParserError(b[i:i+1], "invalid UTF-8 character in multiline basic string")
}
buf = append(buf, b[i:j]...)
i = j
}
}
return nil, nil, nil, NewParserError(b[len(b):], `multiline basic string not terminated by """`)
}
// scanUtf8Run consumes a run of valid non-ASCII UTF-8 runes starting at
// b[i]. It returns the index of the first byte after the run, and whether
// the run was entirely valid. Processing whole runs amortizes the cost of
// the call compared to validating rune by rune.
func scanUtf8Run(b []byte, i int) (int, bool) {
for i < len(b) {
c := b[i]
switch {
case c < 0x80:
return i, true
case c < 0xC2:
return i, false
case c < 0xE0:
if i+1 >= len(b) || b[i+1]&0xC0 != 0x80 {
return i, false
}
i += 2
case c < 0xF0:
if i+2 >= len(b) || b[i+2]&0xC0 != 0x80 {
return i, false
}
b1 := b[i+1]
switch c {
case 0xE0:
if b1 < 0xA0 || b1 > 0xBF {
return i, false
}
case 0xED:
// exclude surrogates
if b1 < 0x80 || b1 > 0x9F {
return i, false
}
default:
if b1&0xC0 != 0x80 {
return i, false
}
}
i += 3
case c < 0xF5:
if i+3 >= len(b) || b[i+2]&0xC0 != 0x80 || b[i+3]&0xC0 != 0x80 {
return i, false
}
b1 := b[i+1]
switch c {
case 0xF0:
if b1 < 0x90 || b1 > 0xBF {
return i, false
}
case 0xF4:
if b1 < 0x80 || b1 > 0x8F {
return i, false
}
default:
if b1&0xC0 != 0x80 {
return i, false
}
}
i += 4
default:
return i, false
}
}
return i, true
}
package unstable
// Unmarshaler is implemented by types that can unmarshal a TOML description
// of themselves. The input is a valid TOML document containing the relevant
// portion of the parsed document.
//
// For tables (including split tables defined in multiple places), the data
// contains the raw key-value bytes from the original document with adjusted
// table headers to be relative to the unmarshaling target.
//
// When the decoding target itself implements this interface, it receives the
// whole document — every top-level key-value as well as every table and array
// table — assembled into a single valid TOML document and delivered once.
type Unmarshaler interface {
UnmarshalTOML(data []byte) error
}
// RawMessage is a raw encoded TOML value. It implements Unmarshaler and can
// be used to delay TOML decoding or capture raw content.
//
// Example usage:
//
// type Config struct {
// Plugin RawMessage `toml:"plugin"`
// }
//
// var cfg Config
// toml.NewDecoder(r).EnableUnmarshalerInterface().Decode(&cfg)
// // cfg.Plugin now contains the raw TOML bytes for [plugin]
type RawMessage []byte
// UnmarshalTOML implements Unmarshaler.
func (m *RawMessage) UnmarshalTOML(data []byte) error {
*m = append((*m)[:0], data...)
return nil
}