TomlParser.java
package tools.jackson.dataformat.toml;
import java.io.IOException;
import java.io.Reader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.time.*;
import java.time.format.DateTimeParseException;
import java.time.temporal.Temporal;
import tools.jackson.core.io.IOContext;
import tools.jackson.core.io.NumberInput;
import tools.jackson.core.StreamReadConstraints;
import tools.jackson.core.StreamReadFeature;
import tools.jackson.core.exc.StreamConstraintsException;
import tools.jackson.core.util.VersionUtil;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.node.*;
class TomlParser {
private static final JsonNodeFactory factory = new JsonNodeFactoryImpl();
static final int MAX_CHARS_TO_REPORT = 1000;
private final TomlFactory tomlFactory;
/**
* Constraints to enforce: read from {@link IOContext} (and not from
* {@link #tomlFactory}) so that the same settings apply as for the
* document-length check the lexer does.
*
* @since 3.1.7
*/
private final StreamReadConstraints streamReadConstraints;
private final TomlStreamReadException.ErrorContext errorContext;
private final int options;
private final Lexer lexer;
private TomlToken next;
private TomlParser(
TomlFactory tomlFactory,
IOContext ioContext,
TomlStreamReadException.ErrorContext errorContext,
int options,
Reader reader
) {
this.tomlFactory = tomlFactory;
this.streamReadConstraints = ioContext.streamReadConstraints();
this.errorContext = errorContext;
this.options = options;
this.lexer = new Lexer(reader, ioContext, errorContext);
lexer.prohibitInternalBufferAllocate = (options & TomlWriteFeature.INTERNAL_PROHIBIT_INTERNAL_BUFFER_ALLOCATE) != 0;
// NOTE: lexing of the first token is deliberately NOT done here but in
// `readFirstToken()`, so that it happens inside the caller's try/finally
// and cannot leak the buffers `Lexer` just took from the recycler
}
/**
* Reads the first token; separate from constructor since the `Lexer` has already
* allocated recycled buffers by then, and failure here must not bypass
* {@link Lexer#releaseBuffers()}.
*/
private void readFirstToken() throws IOException {
this.next = lexer.yylex();
}
/**
* @param tomlFactory factory with configuration
* @param ioContext I/O context
* @param reader character stream
* @return parsed <code>ObjectNode</code>
* @throws IOException if there are I/O issues
*/
public static ObjectNode parse(
final TomlFactory tomlFactory,
final IOContext ioContext,
final int formatReadFeatures,
final Reader reader
) throws IOException {
final TomlFactory factory = tomlFactory == null ? new TomlFactory() : tomlFactory;
TomlStreamReadException.ErrorContext errorCtxt = new TomlStreamReadException.ErrorContext(ioContext.contentReference(), null);
TomlParser parser = new TomlParser(factory, ioContext, errorCtxt,
formatReadFeatures, reader);
try {
parser.readFirstToken();
return parser.parse();
} finally {
parser.lexer.releaseBuffers();
}
}
private TomlToken peek() throws TomlStreamReadException {
TomlToken here = this.next;
if (here == null) throw errorContext.atPosition(lexer).generic("Premature end of file");
return here;
}
/**
* Note: Polling also lexes the next token, so methods like {@link Lexer#yytext()} will not work afterwards
*/
private TomlToken poll(int nextState) throws IOException {
TomlToken here = peek();
lexer.yybegin(nextState);
next = lexer.yylex();
return here;
}
private void pollExpected(TomlToken expected, int nextState) throws IOException {
TomlToken actual = poll(nextState);
if (actual != expected) {
throw errorContext.atPosition(lexer).unexpectedToken(actual, expected.toString());
}
}
public ObjectNode parse() throws IOException {
TomlObjectNode root = (TomlObjectNode) factory.objectNode();
TomlObjectNode currentTable = root;
int currentTableDepth = 1;
while (next != null) {
TomlToken token = peek();
if (token == TomlToken.UNQUOTED_KEY || token == TomlToken.STRING) {
parseKeyVal(currentTable, Lexer.EXPECT_EOL, currentTableDepth);
} else if (token == TomlToken.STD_TABLE_OPEN) {
pollExpected(TomlToken.STD_TABLE_OPEN, Lexer.EXPECT_INLINE_KEY);
FieldRef fieldRef = parseAndEnterKey(root, true, 1);
currentTableDepth = fieldRef.objectDepth + 1;
currentTable = getOrCreateObject(fieldRef.object, fieldRef.key, currentTableDepth);
if (currentTable.defined) {
throw errorContext.atPosition(lexer).generic("Table redefined");
}
currentTable.defined = true;
currentTable.explicitlyDefined = true;
pollExpected(TomlToken.STD_TABLE_CLOSE, Lexer.EXPECT_EOL);
} else if (token == TomlToken.ARRAY_TABLE_OPEN) {
pollExpected(TomlToken.ARRAY_TABLE_OPEN, Lexer.EXPECT_INLINE_KEY);
FieldRef fieldRef = parseAndEnterKey(root, true, 1);
int arrayDepth = fieldRef.objectDepth + 1;
TomlArrayNode array = getOrCreateArray(fieldRef.object, fieldRef.key, arrayDepth);
if (array.closed) {
throw errorContext.atPosition(lexer).generic("Array already finished");
}
currentTableDepth = arrayDepth + 1;
streamReadConstraints.validateNestingDepth(currentTableDepth);
currentTable = (TomlObjectNode) array.addObject();
currentTable.explicitlyDefined = true;
pollExpected(TomlToken.ARRAY_TABLE_CLOSE, Lexer.EXPECT_EOL);
} else {
throw errorContext.atPosition(lexer).unexpectedToken(token, "key or table");
}
}
assert lexer.yyatEOF();
int eofState = lexer.yystate();
if (eofState != Lexer.EXPECT_EXPRESSION && eofState != Lexer.EXPECT_EOL) {
throw errorContext.atPosition(lexer).generic("EOF in wrong state");
}
return root;
}
private FieldRef parseAndEnterKey(
TomlObjectNode outer,
boolean forTable,
int outerDepth
) throws IOException {
TomlObjectNode node = outer;
int nodeDepth = outerDepth;
while (true) {
if (node.closed) {
throw errorContext.atPosition(lexer).generic("Object already closed");
}
TomlToken partToken = peek();
String part;
if (partToken == TomlToken.STRING) {
part = lexer.textBuffer.contentsAsString();
} else if (partToken == TomlToken.UNQUOTED_KEY) {
part = lexer.yytext();
} else {
throw errorContext.atPosition(lexer).unexpectedToken(partToken, "quoted or unquoted key");
}
// [dataformats-text#430]: each key part becomes a property name
streamReadConstraints.validateNameLength(part.length());
pollExpected(partToken, Lexer.EXPECT_INLINE_KEY);
if (peek() != TomlToken.DOT_SEP) {
return new FieldRef(node, part, nodeDepth);
}
pollExpected(TomlToken.DOT_SEP, Lexer.EXPECT_INLINE_KEY);
int childDepth = nodeDepth + 1;
streamReadConstraints.validateNestingDepth(childDepth);
JsonNode existing = node.get(part);
if (existing == null) {
node = (TomlObjectNode) node.putObject(part);
if (!forTable) {
/* "Dotted keys create and define a table for each key part before
* the last one, provided that such tables were not previously created."
*/
node.defined = true;
}
nodeDepth = childDepth;
} else if (existing.isObject()) {
node = (TomlObjectNode) existing;
if (!forTable && node.explicitlyDefined) {
throw errorContext.atPosition(lexer).generic("Dotted key cannot extend explicitly defined table");
}
nodeDepth = childDepth;
} else if (existing.isArray()) {
/* "Any reference to an array of tables points to the most recently defined table element of the array.
* This allows you to define sub-tables, and even sub-arrays of tables, inside the most recent table."
*
* I interpret this somewhat broadly for table headers: I accept such references even if there were
* unrelated tables in between. This case doesn't seem to be covered by the specification.
*/
TomlArrayNode array = (TomlArrayNode) existing;
if (array.closed) {
throw errorContext.atPosition(lexer).generic("Array already closed");
}
if (!forTable) {
throw errorContext.atPosition(lexer).generic("Dotted key cannot extend array of tables");
}
// Only arrays declared by array tables are not closed, and those are always arrays of objects.
int tableDepth = childDepth + 1;
streamReadConstraints.validateNestingDepth(tableDepth);
node = (TomlObjectNode) array.get(array.size() - 1);
nodeDepth = tableDepth;
} else {
throw errorContext.atPosition(lexer).generic("Path into existing non-object value of type " + existing.getNodeType());
}
}
}
private JsonNode parseValue(int nextState, int valueDepth) throws IOException {
TomlToken firstToken = peek();
switch (firstToken) {
case STRING:
String text = lexer.textBuffer.contentsAsString();
pollExpected(TomlToken.STRING, nextState);
return factory.stringNode(text);
case TRUE:
pollExpected(TomlToken.TRUE, nextState);
return factory.booleanNode(true);
case FALSE:
pollExpected(TomlToken.FALSE, nextState);
return factory.booleanNode(false);
case OFFSET_DATE_TIME:
case LOCAL_DATE_TIME:
case LOCAL_DATE:
case LOCAL_TIME:
return parseDateTime(nextState);
case FLOAT:
return parseFloat(nextState);
case INTEGER:
return parseInt(nextState);
case ARRAY_OPEN:
streamReadConstraints.validateNestingDepth(valueDepth);
return parseArray(nextState, valueDepth);
case INLINE_TABLE_OPEN:
streamReadConstraints.validateNestingDepth(valueDepth);
return parseInlineTable(nextState, valueDepth);
default:
throw errorContext.atPosition(lexer).unexpectedToken(firstToken, "value");
}
}
// Helper method for ensuring text content included in exception is
// of bound length, to avoid excessive messages.
//
// @since 3.2
private String reportText(String text) {
if (text.length() <= MAX_CHARS_TO_REPORT) {
return text;
}
return text.substring(0, MAX_CHARS_TO_REPORT) + " [truncated]";
}
private JsonNode parseDateTime(int nextState) throws IOException {
String originalText = lexer.yytext();
String text = originalText;
TomlToken token = peek();
// The time delimiter can be [Tt ]. java.time supports [Tt], and TOML accepts lowercase z.
if (token == TomlToken.LOCAL_TIME || token == TomlToken.LOCAL_DATE_TIME || token == TomlToken.OFFSET_DATE_TIME) {
StringBuilder normalized = null;
int timeStart = token == TomlToken.LOCAL_TIME ? 0 : 11;
int timeEnd = text.length();
if (token == TomlToken.OFFSET_DATE_TIME) {
char last = text.charAt(text.length() - 1);
if (last == 'z' || last == 'Z') {
timeEnd--;
} else {
int plus = text.indexOf('+', timeStart);
int minus = text.indexOf('-', timeStart);
int offsetStart = plus < 0 ? minus : (minus < 0 ? plus : Math.min(plus, minus));
if (offsetStart >= 0) {
timeEnd = offsetStart;
}
}
}
int secondsColon = text.indexOf(':', timeStart + 3);
if (secondsColon < 0 || secondsColon >= timeEnd) {
normalized = new StringBuilder(text);
normalized.insert(timeEnd, ":00");
}
if (token != TomlToken.LOCAL_TIME && text.charAt(10) == ' ') {
if (normalized == null) {
normalized = new StringBuilder(text);
}
normalized.setCharAt(10, 'T');
}
if (token == TomlToken.OFFSET_DATE_TIME && text.charAt(text.length() - 1) == 'z') {
if (normalized == null) {
normalized = new StringBuilder(text);
}
normalized.setCharAt(normalized.length() - 1, 'Z');
}
if (normalized != null) {
text = normalized.toString();
}
}
boolean parseJavaTime = TomlReadFeature.PARSE_JAVA_TIME.enabledIn(options);
if (parseJavaTime || TomlReadFeature.VALIDATE_DATE_TIME.enabledIn(options)) {
Temporal value;
try {
if (token == TomlToken.LOCAL_DATE) {
value = LocalDate.parse(text);
} else if (token == TomlToken.LOCAL_TIME) {
value = LocalTime.parse(text);
} else if (token == TomlToken.LOCAL_DATE_TIME) {
value = LocalDateTime.parse(text);
} else if (token == TomlToken.OFFSET_DATE_TIME) {
value = OffsetDateTime.parse(text);
} else {
VersionUtil.throwInternal();
throw new AssertionError();
}
} catch (DateTimeParseException e) {
throw errorContext.atPosition(lexer).invalidDateTime(e, reportText(originalText));
}
pollExpected(token, nextState);
if (parseJavaTime) {
return factory.pojoNode(value);
}
return factory.stringNode(text);
}
pollExpected(token, nextState);
return factory.stringNode(text);
}
private JsonNode parseInt(int nextState) throws IOException {
char[] buffer = lexer.getTextBuffer();
int start = lexer.getTextBufferStart();
int length = lexer.getTextBufferEnd() - lexer.getTextBufferStart();
for (int i = 0; i < length; i++) {
if (buffer[start + i] == '_') {
// slow path to remove underscores: compact into the already-owned
// buffer itself (chars before the first '_' are already in place),
// avoiding a second array allocation.
// NOTE: leaves stale chars after the compacted region, so this is
// only safe as long as nothing re-reads `lexer.yytext()` for this token
int pos = start + i;
for (int j = pos + 1; j < start + length; j++) {
char c = buffer[j];
if (c != '_') {
buffer[pos++] = c;
}
}
length = pos - start;
break;
}
}
ValueNode node = parseIntFromBuffer(buffer, start, length);
pollExpected(TomlToken.INTEGER, nextState);
return node;
}
private ValueNode parseIntFromBuffer(char[] buffer, int start, int length) throws TomlStreamReadException {
if (length > 2) {
char baseChar = buffer[start + 1];
if (baseChar == 'x' || baseChar == 'o' || baseChar == 'b') {
try {
streamReadConstraints.validateIntegerLength(length);
} catch (StreamConstraintsException e) {
final String reportNum = length <= MAX_CHARS_TO_REPORT
? new String(buffer, start, length)
: new String(buffer, start, MAX_CHARS_TO_REPORT) + " [truncated]";
throw errorContext.atPosition(lexer).invalidNumber(e, reportNum);
}
start += 2;
length -= 2;
String text = new String(buffer, start, length);
try {
// note: we parse all these as unsigned. Hence the weird int limits.
// hex
if (baseChar == 'x') {
if (length <= 31 / 4) {
return factory.numberNode(Integer.parseInt(text, 16));
} else if (length <= 63 / 4) {
return factory.numberNode(Long.parseLong(text, 16));
} else {
return factory.numberNode(NumberInput.parseBigIntegerWithRadix(
text, 16, tomlFactory.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER)));
}
}
// octal
if (baseChar == 'o') {
// this is a bit conservative, but who uses octal anyway?
if (length <= 31 / 3) {
return factory.numberNode(Integer.parseInt(text, 8));
} else if (text.length() <= 63 / 3) {
return factory.numberNode(Long.parseLong(text, 8));
} else {
return factory.numberNode(NumberInput.parseBigIntegerWithRadix(
text, 8, tomlFactory.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER)));
}
}
// binary
assert baseChar == 'b';
if (length <= 31) {
return factory.numberNode(Integer.parseUnsignedInt(text, 2));
} else if (length <= 63) {
return factory.numberNode(Long.parseUnsignedLong(text, 2));
} else {
return factory.numberNode(NumberInput.parseBigIntegerWithRadix(
text, 2, tomlFactory.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER)));
}
} catch (NumberFormatException e) {
throw errorContext.atPosition(lexer).invalidNumber(e, text);
}
}
}
// decimal
boolean negative;
if (buffer[start] == '-') {
start++;
length--;
negative = true;
} else if (buffer[start] == '+') {
start++;
length--;
negative = false;
} else {
negative = false;
}
// adapted from JsonParserBase
if (length <= 9) {
int v = NumberInput.parseInt(buffer, start, length);
if (negative) v = -v;
return factory.numberNode(v);
}
if (length <= 18 || NumberInput.inLongRange(buffer, start, length, negative)) {
long v;
if (length <= 18) {
v = NumberInput.parseLong(buffer, start, length);
if (negative) {
v = -v;
}
} else {
// [dataformats-text#663]: 19-digit values need separate handling
v = NumberInput.parseLong19(buffer, start, negative);
}
// Might still fit in int, need to check
if ((int) v == v) {
return factory.numberNode((int) v);
} else {
return factory.numberNode(v);
}
}
try {
streamReadConstraints.validateIntegerLength(length);
} catch (StreamConstraintsException e) {
final String reportNum = length <= MAX_CHARS_TO_REPORT
? new String(buffer, start, length)
: new String(buffer, start, MAX_CHARS_TO_REPORT) + " [truncated]";
throw errorContext.atPosition(lexer).invalidNumber(e, reportNum);
}
BigInteger big = NumberInput.parseBigInteger(
new String(buffer, start, length),
tomlFactory.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER));
// sign was skipped above, need to re-apply
return factory.numberNode(negative ? big.negate() : big);
}
private JsonNode parseFloat(int nextState) throws IOException {
String rawText = lexer.yytext();
final String text = rawText.indexOf('_') >= 0 ? rawText.replace("_", "") : rawText;
pollExpected(TomlToken.FLOAT, nextState);
if (text.endsWith("nan")) {
return factory.numberNode(Double.NaN);
} else if (text.endsWith("inf")) {
return factory.numberNode(text.startsWith("-") ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY);
} else {
try {
// Related to [databind#4250], need to pre-validate
if (!NumberInput.looksLikeValidNumber(text)) {
throw new NumberFormatException("Not a valid Number representation");
}
streamReadConstraints.validateFPLength(text.length());
BigDecimal dec = NumberInput.parseBigDecimal(
text, tomlFactory.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER));
return factory.numberNode(dec);
} catch (NumberFormatException | StreamConstraintsException e) {
final String reportNum = text.length() <= MAX_CHARS_TO_REPORT ?
text :
text.substring(0, MAX_CHARS_TO_REPORT) + " [truncated]";
throw errorContext.atPosition(lexer).invalidNumber(e, reportNum);
}
}
}
private ObjectNode parseInlineTable(int nextState, int tableDepth) throws IOException {
// inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close
// inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
pollExpected(TomlToken.INLINE_TABLE_OPEN, Lexer.EXPECT_INLINE_KEY);
TomlObjectNode node = (TomlObjectNode) factory.objectNode();
while (true) {
TomlToken token = peek();
if (token == TomlToken.INLINE_TABLE_CLOSE) {
break;
}
parseKeyVal(node, Lexer.EXPECT_TABLE_SEP, tableDepth);
TomlToken sepToken = peek();
if (sepToken == TomlToken.INLINE_TABLE_CLOSE) {
break;
} else if (sepToken == TomlToken.COMMA) {
pollExpected(TomlToken.COMMA, Lexer.EXPECT_INLINE_KEY);
} else {
throw errorContext.atPosition(lexer).unexpectedToken(sepToken, "comma or table end");
}
}
pollExpected(TomlToken.INLINE_TABLE_CLOSE, nextState);
node.closed = true;
node.defined = true;
return node;
}
private ArrayNode parseArray(int nextState, int arrayDepth) throws IOException {
// array = array-open [ array-values ] ws-comment-newline array-close
// array-values = ws-comment-newline val ws-comment-newline array-sep array-values
// array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ]
pollExpected(TomlToken.ARRAY_OPEN, Lexer.EXPECT_VALUE);
TomlArrayNode node = (TomlArrayNode) factory.arrayNode();
while (true) {
TomlToken token = peek();
if (token == TomlToken.ARRAY_CLOSE) {
break;
}
JsonNode value = parseValue(Lexer.EXPECT_ARRAY_SEP, arrayDepth + 1);
node.add(value);
TomlToken sepToken = peek();
if (sepToken == TomlToken.ARRAY_CLOSE) {
break;
} else if (sepToken == TomlToken.COMMA) {
pollExpected(TomlToken.COMMA, Lexer.EXPECT_VALUE);
} else {
throw errorContext.atPosition(lexer).unexpectedToken(sepToken, "comma or array end");
}
}
pollExpected(TomlToken.ARRAY_CLOSE, nextState);
node.closed = true;
return node;
}
private void parseKeyVal(TomlObjectNode target, int nextState, int targetDepth) throws IOException {
// keyval = key keyval-sep val
FieldRef fieldRef = parseAndEnterKey(target, false, targetDepth);
pollExpected(TomlToken.KEY_VAL_SEP, Lexer.EXPECT_VALUE);
JsonNode value = parseValue(nextState, fieldRef.objectDepth + 1);
if (fieldRef.object.has(fieldRef.key)) {
throw errorContext.atPosition(lexer).generic("Duplicate key");
}
fieldRef.object.set(fieldRef.key, value);
}
private TomlObjectNode getOrCreateObject(ObjectNode node, String field, int childDepth) throws IOException {
streamReadConstraints.validateNestingDepth(childDepth);
JsonNode existing = node.get(field);
if (existing == null) {
return (TomlObjectNode) node.putObject(field);
} else if (existing.isObject()) {
return (TomlObjectNode) existing;
} else {
throw errorContext.atPosition(lexer).generic("Path into existing non-object value of type " + existing.getNodeType());
}
}
private TomlArrayNode getOrCreateArray(ObjectNode node, String field, int childDepth) throws IOException {
streamReadConstraints.validateNestingDepth(childDepth);
JsonNode existing = node.get(field);
if (existing == null) {
return (TomlArrayNode) node.putArray(field);
} else if (existing.isArray()) {
return (TomlArrayNode) existing;
} else {
throw errorContext.atPosition(lexer).generic("Path into existing non-array value of type " + existing.getNodeType());
}
}
private static class FieldRef {
final TomlObjectNode object;
final String key;
final int objectDepth;
FieldRef(TomlObjectNode object, String key, int objectDepth) {
this.object = object;
this.key = key;
this.objectDepth = objectDepth;
}
}
@SuppressWarnings("serial") // only used internally, no need to be JDK serializable
private static class TomlObjectNode extends ObjectNode {
boolean closed = false;
// True for both explicit headers and dotted-key super-tables; gates "Table redefined".
boolean defined = false;
// Subset of `defined`: set only by [table] or [[array-of-tables]] headers,
// not by dotted-key super-tables.
// Gates rejection of dotted-key extensions of an already-explicit table.
boolean explicitlyDefined = false;
TomlObjectNode(JsonNodeFactory nc) {
super(nc);
}
}
@SuppressWarnings("serial") // only used internally, no need to be JDK serializable
private static class TomlArrayNode extends ArrayNode {
boolean closed = false;
TomlArrayNode(JsonNodeFactory nf) {
super(nf);
}
TomlArrayNode(JsonNodeFactory nf, int capacity) {
super(nf, capacity);
}
}
@SuppressWarnings("serial") // only used internally, no need to be JDK serializable
private static class JsonNodeFactoryImpl extends JsonNodeFactory {
public JsonNodeFactoryImpl() {
super();
}
@Override
public ArrayNode arrayNode() {
return new TomlArrayNode(this);
}
@Override
public ArrayNode arrayNode(int capacity) {
return new TomlArrayNode(this, capacity);
}
@Override
public ObjectNode objectNode() {
return new TomlObjectNode(this);
}
}
}