IonTokenReader.java
/*
* Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.ion.impl;
import static com.amazon.ion.util.IonTextUtils.isDigit;
import static com.amazon.ion.util.IonTextUtils.isOperatorPart;
import static com.amazon.ion.util.IonTextUtils.isWhitespace;
import static com.amazon.ion.util.IonTextUtils.printCodePointAsString;
import com.amazon.ion.Decimal;
import com.amazon.ion.IonException;
import com.amazon.ion.Timestamp;
import com.amazon.ion.UnexpectedEofException;
import com.amazon.ion.impl._Private_IonConstants.HighNibble;
import com.amazon.ion.util.IonTextUtils;
import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Stack;
/**
* This class is responsible for breaking the input stream into Tokens.
* It does both the token recognition (aka the scanner) and the state
* management needed to use the value underlying some of these tokens.
*/
final class IonTokenReader
{
// TODO clean up, many of these are unused.
public static int isPunctuation = 0x0001;
public static int isKeyword = 0x0002;
public static int isTypeName = 0x0004;
public static int isConstant = 0x0008;
private static int isPosInt = 0x0010;
private static int isNegInt = 0x0020;
public static int isFloat = 0x0040;
public static int isDecimal = 0x0080;
public static int isTag = 0x0100;
static public enum Type {
eof (isPunctuation, "<eof>" ),
tOpenParen (isPunctuation, "(" ),
tCloseParen (isPunctuation, ")" ),
tOpenSquare (isPunctuation, "[" ),
tCloseSquare (isPunctuation, "[" ),
tOpenCurly (isPunctuation, "{" ),
tCloseCurly (isPunctuation, "}" ),
tOpenDoubleCurly (isPunctuation, "{{" ),
//tCloseDoubleCurly (isPunctuation, "}}" ), // only valid at end of blob, not recognized in token read
tSingleQuote (isPunctuation, "'" ),
tDoubleQuote (isPunctuation, "\"" ),
//tColon (isPunctuation, ":" ), // filled in during identifier scan with lookahead
//tDoubleColon (isPunctuation, "::" ), // ditto
tComma (isPunctuation, "," ),
kwTrue ((isConstant + isTag + isKeyword), "true", HighNibble.hnBoolean),
kwFalse ((isConstant + isTag + isKeyword), "false", HighNibble.hnBoolean),
kwNull ((isConstant + isTag + isKeyword), "null", HighNibble.hnNull),
kwNullNull ((isConstant + isTag + isKeyword), "null.null", HighNibble.hnNull),
kwNullInt ((isConstant + isTag + isKeyword), "null.int", HighNibble.hnPosInt),
kwNullList ((isConstant + isTag + isKeyword), "null.list", HighNibble.hnList),
kwNullSexp ((isConstant + isTag + isKeyword), "null.sexp", HighNibble.hnSexp),
kwNullFloat ((isConstant + isTag + isKeyword), "null.float", HighNibble.hnFloat),
kwNullBlob ((isConstant + isTag + isKeyword), "null.blob", HighNibble.hnBlob),
kwNullClob ((isConstant + isTag + isKeyword), "null.clob", HighNibble.hnClob),
kwNullString ((isConstant + isTag + isKeyword), "null.string", HighNibble.hnString),
kwNullStruct ((isConstant + isTag + isKeyword), "null.struct", HighNibble.hnStruct), // LN
kwNullSymbol ((isConstant + isTag + isKeyword), "null.symbol", HighNibble.hnSymbol),
kwNullBoolean ((isConstant + isTag + isKeyword), "null.bool", HighNibble.hnBoolean),
kwNullDecimal ((isConstant + isTag + isKeyword), "null.decimal", HighNibble.hnDecimal),
kwNullTimestamp ((isConstant + isTag + isKeyword), "null.timestamp", HighNibble.hnTimestamp),
kwNan ((isConstant + isKeyword), "nan", HighNibble.hnFloat),
kwPosInf ((isConstant + isKeyword), "+inf", HighNibble.hnFloat),
kwNegInf ((isConstant + isKeyword), "-inf", HighNibble.hnFloat),
constNegInt ((isConstant + isNegInt), "cNegInt", HighNibble.hnNegInt),
constPosInt ((isConstant + isPosInt), "cPosInt", HighNibble.hnPosInt),
constFloat ((isConstant + isFloat), "cFloat", HighNibble.hnFloat),
constDecimal ((isConstant + isDecimal), "cDec", HighNibble.hnDecimal),
constTime ((isConstant), "cTime", HighNibble.hnTimestamp),
constString ((isConstant + isTag), "cString", HighNibble.hnString),
constSymbol ((isConstant + isTag), "cSymbol", HighNibble.hnSymbol),
constMemberName ((isConstant + isTag), "cMemberName", HighNibble.hnSymbol),
constUserTypeDecl ((isConstant + isTag), "cUserTypeDecl", HighNibble.hnSymbol),
none(0);
private int flags;
private String image;
private HighNibble highNibble;
Type() {}
Type(int v) {
this.flags = v;
}
Type(int v, String name) {
flags = v;
image = name;
}
Type(int v, String name, HighNibble ln) {
flags = v;
image = name;
highNibble = ln;
}
/**
* TODO why this class exists at all. We should store
* this stuff directly in IonTimestampImpl as a BigDecimal (time) and
* int (offset, -1==unknown) and avoid constructing more objects.
* <p>
* Also, we probably don't need the DateFormats. The parser already
* matches against a regex, so we should be able to pull the data
* straight from the string to compute the value.
*/
public static class timeinfo { // TODO remove vestigial class timeinfo
static public Timestamp parse(String s) {
Timestamp t = null;
s = s.trim(); // TODO why is this necessary?
try {
t = Timestamp.valueOf(s); // TODO should Timestamp just throw an IonException?
}
catch (IllegalArgumentException e) {
throw new IonException(e);
}
return t;
}
}
// TODO move out of this class into TR.
public Type setNumericValue(IonTokenReader tr, String s) {
switch (this) {
case kwNan:
tr.doubleValue = Double.NaN;
return this;
case kwPosInf:
tr.doubleValue = Double.POSITIVE_INFINITY;
return this;
case kwNegInf:
tr.doubleValue = Double.NEGATIVE_INFINITY;
return this;
case constNegInt:
case constPosInt:
if (NumberType.NT_HEX.equals(tr.numberType)) {
tr.intValue = new BigInteger(s, 16);
// In hex case we've discarded the prefix [+-]?0x so
// reconstruct the sign
if (this == constNegInt) tr.intValue = tr.intValue.negate();
}
else {
tr.intValue = new BigInteger(s, 10);
// Make sure that sign aligns with type.
// Note that this allows negative zero.
assert (BigInteger.ZERO.equals(tr.intValue)
? true
: this == (tr.intValue.signum() < 0 ? constNegInt : constPosInt));
}
return this;
case constFloat:
tr.doubleValue = Double.parseDouble(s);
return this;
case constDecimal:
// BigDecimal parses using e instead of d or D
String eFormat = s.replace('d', 'e');
if (eFormat == s) {
// No match for 'd' but look for 'D'
eFormat = s.replace('D', 'e');
}
tr.decimalValue = Decimal.valueOf(eFormat);
return this;
case constTime:
tr.dateValue = timeinfo.parse(s);
return this;
default:
throw new AssertionError("Unknown op for numeric case: " + this);
}
}
public boolean isKeyword() { return ((flags & isKeyword) != 0); }
public boolean isConstant() { return ((flags & isConstant) != 0); }
public boolean isNumeric() { return ((flags & (isPosInt + isNegInt + isFloat + isDecimal)) != 0); }
public String getImage() { return (image == null) ? this.name() : image; }
public HighNibble getHighNibble() { return highNibble; }
@Override
public String toString() {
if (this.getImage() != null) return this.getImage();
return super.toString();
}
}
//
// this is a temp reader we use when converting blob's into
// blob values
//
static class LocalReader extends Reader {
IonTokenReader _tr;
int _sboffset;
int _sbavailable;
LocalReader(IonTokenReader tr) {
_tr = tr;
}
@Override
public void close() throws IOException {
_tr = null;
return;
}
@Override
public void reset() {
_sboffset = 0;
_sbavailable = _tr.value.length();
}
@Override
public int read() throws IOException {
int c = -1;
if (_sbavailable > 0) {
c = _tr.value.charAt(_sboffset++);
_sbavailable--;
}
else {
c = _tr.read();
}
return c;
}
@Override
public int read(char[] dst, int dstoffset, int len) throws IOException {
int needed = len;
while (needed-- > 0) {
int c = this.read();
if (c < 0) break;
dst[dstoffset++] = (char)c;
}
return len - needed;
}
}
// easy to use number types
static enum NumberType {
NT_POSINT,
NT_NEGINT,
NT_HEX, // pos or neg
NT_FLOAT,
NT_DECIMAL,
NT_DECIMAL_NEGATIVE_ZERO
}
/**
* Magic "character" to represent an escape sequence with an empty expansion.
* (or, in other words, a non-eof character that should be ignored)
*/
static final int EMPTY_ESCAPE_SEQUENCE = -2;
//--------------------------------------------------------------------
//
// this is the parsing context to manage the in list, in struct,
// waiting for identifier tag (usertypedecl or member name) vs value
//
//
static public enum Context {
NONE,
STRING,
BLOB,
CLOB,
EXPRESSION,
DATALIST,
STRUCT;
}
public Stack<Context> contextStack = new Stack<Context>();
public Context context = Context.NONE;
public void pushContext(Context newcontext) {
contextStack.push(newcontext);
context = newcontext;
}
public Context popContext() {
context = contextStack.pop();
return context;
}
//--------------------------------------------------------------------
//
// state that keeps us on track ...
//
private IonCharacterReader in;
private LocalReader localReader;
private PushbackReader pushbackReader;
public boolean inQuotedContent;
public boolean isIncomplete;
public boolean isLongString;
public boolean quotedIdentifier;
public int embeddedSlash;
public int endquote;
public Type t = Type.none;
public Type keyword = Type.none;
public StringBuilder value = new StringBuilder();
public String stringValue;
public Double doubleValue;
public BigInteger intValue;
public Timestamp dateValue;
public BigDecimal decimalValue;
public boolean isNegative;
public NumberType numberType;
public IonTokenReader(final Reader r) {
this.in = new IonCharacterReader( r );
}
public long getConsumedAmount() {
return in.getConsumedAmount();
}
public int getLineNumber() {
return in.getLineNumber();
}
public int getColumn() {
return in.getColumn();
}
public String position() {
return "line " + this.getLineNumber() + " column " + this.getColumn();
}
public String getValueString(boolean is_in_expression) throws IOException {
if (this.isIncomplete) {
finishScanString(is_in_expression);
stringValue = value.toString(); // TODO combine with below?
this.inQuotedContent = false;
}
else if (stringValue == null) {
stringValue = value.toString();
}
return stringValue;
}
void resetValue() {
isIncomplete = false;
stringValue = null;
doubleValue = null;
intValue = null;
dateValue = null;
decimalValue = null;
isNegative = false;
numberType = null;
t = null;
value.setLength(0);
}
public PushbackReader getPushbackReader() {
if (localReader == null) {
localReader = new LocalReader(this);
pushbackReader =
new PushbackReader(localReader,
_Private_Utils.MAX_LOOKAHEAD_UTF16);
}
localReader.reset();
return pushbackReader ;
}
/**
* Reads the next character using the underlying stream.
* @return -1 on end of stream.
* @throws IOException
*/
final int read() throws IOException {
final int ch = in.read();
assert ch != '\r';
return ch;
}
int readIgnoreWhitespace() throws IOException {
assert !inQuotedContent;
int c;
// read through comments - this detects and rejects double slash
// and slash star style comments and their contents
for (;;) {
c = read();
if (c == '/') { // possibly a start of a comment
int c2 = read();
if (c2 == '/') {
// we have a // comment, scan for the terminating new line
while (c2 != '\n' && c2 != -1) {
c2 = read();
}
c = c2;
}
else if (c2 == '*') {
// we have a /* */ comment scan for closing */
scancomment:
while (c2 != -1) {
c2 = read();
if (c2 == '*') {
c2 = read();
if (c2 == '/') {
break scancomment;
}
unread(c2); // in case this was the '*' we're looking for
}
}
c = read();
}
else {
// it wasn't a comment start, throw it back
unread(c2);
}
}
if (/* inContent ||*/ !isWhitespace(c)) {
break;
}
}
return c;
}
void unread(int c) throws IOException {
this.in.unread(c);
}
// TODO clone the body of next(c) here, later , for perf, we
// don't really want TWO methods calls per character, or
// mostly we don't want an extra one. (and I doubt that
// Java handles the tail optimization)
public Type next(boolean is_in_expression) throws IOException {
inQuotedContent = false;
int c = this.readIgnoreWhitespace();
return next(c, is_in_expression);
}
private Type next(int c, boolean is_in_expression) throws IOException {
int c2;
t = Type.none;
isIncomplete = false;
switch (c) {
case -1:
return (t = Type.eof);
case '{':
c2 = read();
if (c2 == '{') {
return (t = Type.tOpenDoubleCurly);
}
unread(c2);
return (t = Type.tOpenCurly);
case '}':
return (t = Type.tCloseCurly);
case '[':
return (t = Type.tOpenSquare);
case ']':
return (t = Type.tCloseSquare);
case '(':
return (t = Type.tOpenParen);
case ')':
return (t = Type.tCloseParen);
case ',':
return (t = Type.tComma);
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return readNumber(c);
case '\"':
inQuotedContent = true;
this.keyword = Type.none; // anything in quotes isn't a keyword
return scanString(c, _Private_IonConstants.lnIsVarLen - 1);
case '\'':
c2 = read();
if (c2 == '\'') {
c2 = read();
if (c2 == '\'') {
return scanLongString();
}
this.unread(c2);
c2 = '\''; // restore c2 for the next unread
}
this.unread(c2);
inQuotedContent = true;
return scanIdentifier(c);
case '+':
c2 = read();
if (c2 == 'i') {
c2 = read();
if (c2 == 'n') {
c2 = read();
if (c2 == 'f') {
return (t = Type.kwPosInf);
}
this.unread(c2);
c2 = 'n';
}
this.unread(c2);
c2 = 'i';
}
this.unread(c2);
if (is_in_expression) {
return scanOperator(c);
}
break; // break to error
case '-':
c2 = read();
if (c2 >= '0' && c2 <= '9') {
this.unread(c2);
return readNumber(c);
}
if (c2 == 'i') {
c2 = read();
if (c2 == 'n') {
c2 = read();
if (c2 == 'f') {
return (t = Type.kwNegInf);
}
this.unread(c2);
c2 = 'n';
}
this.unread(c2);
c2 = 'i';
}
this.unread(c2);
if (is_in_expression) {
return scanOperator(c);
}
break; // break to error
default:
if (IonTextUtils.isIdentifierStart(c)) {
return scanIdentifier(c);
}
if (is_in_expression && isOperatorPart(c)) {
return scanOperator(c);
}
}
String message =
"Unexpected character " + printCodePointAsString(c) +
" encountered at line " + this.getLineNumber() +
" column " + this.getColumn();
throw new IonException(message);
}
public Type scanIdentifier(int c) throws IOException
{
// reset our local value buffer
resetValue();
// we don't have an identifier type any longer, just string
this.t = Type.constSymbol;
// some strings are keywords, most are not
// mostly we'll guess that it's not a keyword
this.keyword = null;
// first we read the identifier content into our value buffer
// if the value is quoted, use the escape char loop, otherwise
// look for the non-identifier character (and throw it back)
if (!readIdentifierContents(c)) { // not quoted (true is quoted)
// anything in quotes (even single quotes) is not a keyword
// and here we're not in quoted content (that is handled above)
// so see if it's also a keyword
this.keyword = IonTokenReader.matchKeyword(value, 0, value.length());
if (this.keyword != null) {
if (this.keyword == Type.kwNull) {
c = this.read();
if (c == '.') {
int dot = value.length();
value.append((char)c);
c = this.read();
int added = readIdentifierContents(c, IonTokenConstsX.TN_MAX_NAME_LENGTH + 1); // +1 is "enough roap" so if there are extra letters at the end of the keyword we keep at least 1
int kw = IonTokenConstsX.keyword(value, dot+1, dot+added+1);
switch (kw) {
case IonTokenConstsX.KEYWORD_NULL:
case IonTokenConstsX.KEYWORD_BOOL:
case IonTokenConstsX.KEYWORD_INT:
case IonTokenConstsX.KEYWORD_FLOAT:
case IonTokenConstsX.KEYWORD_DECIMAL:
case IonTokenConstsX.KEYWORD_TIMESTAMP:
case IonTokenConstsX.KEYWORD_SYMBOL:
case IonTokenConstsX.KEYWORD_STRING:
case IonTokenConstsX.KEYWORD_BLOB:
case IonTokenConstsX.KEYWORD_CLOB:
case IonTokenConstsX.KEYWORD_LIST:
case IonTokenConstsX.KEYWORD_SEXP:
case IonTokenConstsX.KEYWORD_STRUCT:
this.keyword = setNullType(value, dot + 1, value.length() - dot - 1);
break;
default:
// not a valid type name - so we have to unread back to the dot
for (int ii=value.length(); ii>dot; ) {
ii--;
char uc = value.charAt(ii);
unread(uc);
}
String message =
position()
+ ": Expected Ion type after 'null.' but found: "
+ value;
throw new IonException(message);
}
}
else {
unread(c);
}
}
this.t = this.keyword;
return this.t;
}
}
// see if we're a user type of a member name
c = this.readIgnoreWhitespace();
if (c != ':') {
unread(c);
}
else {
c = read();
if (c != ':') {
unread(c);
this.t = Type.constMemberName;
}
else {
this.t = Type.constUserTypeDecl;
}
}
return this.t;
}
boolean readIdentifierContents(int c) throws IOException {
int quote = c;
inQuotedContent = (quote == '\'' || quote == '\"');
if ((quotedIdentifier = inQuotedContent) == true) {
for (;;) {
c = read();
if (c < 0 || c == quote) break;
if (c == '\\') {
c = readEscapedCharacter(this.in, false /*not clob*/);
}
if (c != EMPTY_ESCAPE_SEQUENCE) {
value.appendCodePoint(c);
}
}
if (c == -1) { // TODO throw UnexpectedEofException
throw new IonException("end encountered before closing quote '\\" + (char)endquote+ "'");
}
// c, at this point is a single quote, which we don't append
inQuotedContent = false;
}
else {
value.append((char)c);
for (;;) {
c = read();
if (!IonTextUtils.isIdentifierPart(c)) {
break;
}
value.append((char)c);
}
unread(c); // we throw back our terminator here
}
return quotedIdentifier;
}
int readIdentifierContents(int c, int max_length) throws IOException {
assert(c != '\'' && c != '\"');
value.append((char)c);
int count = 1;
while (count < max_length) {
c = read();
if (!IonTextUtils.isIdentifierPart(c)) {
unread(c); // we throw back our terminator here
break;
}
value.append((char)c);
count++;
}
return count;
}
static Type matchKeyword(StringBuilder sb, int pos, int valuelen) throws IOException
{
Type keyword = null;
switch (sb.charAt(pos++)) { // there has to be at least 1 chacter or we wouldn't be here
case 'f':
if (valuelen == 5 // "f"
&& sb.charAt(pos++) == 'a'
&& sb.charAt(pos++) == 'l'
&& sb.charAt(pos++) == 's'
&& sb.charAt(pos++) == 'e'
) {
keyword = Type.kwFalse;
}
break;
case 'n':
if (valuelen == 4 // 'n'
&& sb.charAt(pos++) == 'u'
&& sb.charAt(pos++) == 'l'
&& sb.charAt(pos++) == 'l'
) {
keyword = Type.kwNull;
}
else if (valuelen == 3 // 'n'
&& sb.charAt(pos++) == 'a'
&& sb.charAt(pos++) == 'n'
) {
keyword = Type.kwNan;
}
break;
case 't':
if (valuelen == 4 // "t"
&& sb.charAt(pos++) == 'r'
&& sb.charAt(pos++) == 'u'
&& sb.charAt(pos++) == 'e'
) {
keyword = Type.kwTrue;
}
break;
}
return keyword;
}
public Type setNullType(StringBuilder sb, int pos, int valuelen)
{
switch (valuelen) {
case 3:
//int
if (sb.charAt(pos++) == 'i'
&& sb.charAt(pos++) == 'n'
&& sb.charAt(pos++) == 't'
) {
return Type.kwNullInt;
}
break;
case 4:
//bool, blob, list, null, clob, sexp
switch (sb.charAt(pos++)) {
case 'b':
//bool, blob
switch (sb.charAt(pos++)) {
case 'o':
// bool
if (sb.charAt(pos++) == 'o'
&& sb.charAt(pos++) == 'l'
) {
return Type.kwNullBoolean;
}
break;
case 'l':
//blob
if (sb.charAt(pos++) == 'o'
&& sb.charAt(pos++) == 'b'
) {
return Type.kwNullBlob;
}
break;
}
break;
case 'l':
if (sb.charAt(pos++) == 'i'
&& sb.charAt(pos++) == 's'
&& sb.charAt(pos++) == 't'
) {
return Type.kwNullList;
}
break;
case 'n':
if (sb.charAt(pos++) == 'u'
&& sb.charAt(pos++) == 'l'
&& sb.charAt(pos++) == 'l'
) {
return Type.kwNullNull;
}
break;
case 'c':
//clob
if (sb.charAt(pos++) == 'l'
&& sb.charAt(pos++) == 'o'
&& sb.charAt(pos++) == 'b'
) {
return Type.kwNullClob;
}
break;
case 's':
//sexp
if (sb.charAt(pos++) == 'e'
&& sb.charAt(pos++) == 'x'
&& sb.charAt(pos++) == 'p'
) {
return Type.kwNullSexp;
}
break;
}
break;
case 5:
//float
if (sb.charAt(pos++) == 'f'
&& sb.charAt(pos++) == 'l'
&& sb.charAt(pos++) == 'o'
&& sb.charAt(pos++) == 'a'
&& sb.charAt(pos++) == 't'
) {
return Type.kwNullFloat;
}
break;
case 6:
switch (sb.charAt(pos++)) {
case 's':
//string
//struct
//symbol
switch(sb.charAt(pos++)) {
case 't':
if (sb.charAt(pos++) == 'r') {
switch (sb.charAt(pos++)) {
case 'i':
if (sb.charAt(pos++) == 'n' && sb.charAt(pos++) == 'g') {
return Type.kwNullString;
}
break;
case 'u':
if (sb.charAt(pos++) == 'c' && sb.charAt(pos++) == 't') {
return Type.kwNullStruct;
}
break;
}
}
break;
case 'y':
if (sb.charAt(pos++) == 'm'
&& sb.charAt(pos++) == 'b'
&& sb.charAt(pos++) == 'o'
&& sb.charAt(pos++) == 'l'
) {
return Type.kwNullSymbol;
}
break;
}
}
break;
case 7:
//decimal
if (sb.charAt(pos++) == 'd'
&& sb.charAt(pos++) == 'e'
&& sb.charAt(pos++) == 'c'
&& sb.charAt(pos++) == 'i'
&& sb.charAt(pos++) == 'm'
&& sb.charAt(pos++) == 'a'
&& sb.charAt(pos++) == 'l'
) {
return Type.kwNullDecimal;
}
break;
case 9:
//timestamp
if (sb.charAt(pos++) == 't'
&& sb.charAt(pos++) == 'i'
&& sb.charAt(pos++) == 'm'
&& sb.charAt(pos++) == 'e'
&& sb.charAt(pos++) == 's'
&& sb.charAt(pos++) == 't'
&& sb.charAt(pos++) == 'a'
&& sb.charAt(pos++) == 'm'
&& sb.charAt(pos++) == 'p'
) {
return Type.kwNullTimestamp;
}
break;
}
String nullimage = sb.toString();
throw new IonException("invalid null value '"+nullimage+"' at " + this.position());
}
public Type scanOperator(int c) throws IOException
{
// reset our local value buffer
resetValue();
// we don't have an identifier type any longer, just string
this.t = Type.constSymbol;
// some strings are keywords, most are not
// mostly we'll guess that it's not a keyword
this.keyword = null;
// first we read the identifier content into our value buffer
// if the value is quoted, use the escape char loop, otherwise
// look for the non-identifier character (and throw it back)
value.append((char)c);
for (;;) {
c = read();
if (!IonTextUtils.isOperatorPart(c)) {
break;
}
value.append((char)c);
}
unread(c); // we throw back our terminator here
return this.t;
}
public Type scanString(int c, int maxlookahead) throws IOException
{
// reset out local value buffer
resetValue();
if (c != '\"') {
throw new IonException("Programmer error! Only a quote should get you here.");
}
endquote = '\"';
sizedloop:
while (maxlookahead-- > 0) {
switch ((c = this.read())) {
case -1: break sizedloop; // TODO deoptimize, throw exception
case '\"': break sizedloop;
case '\n':
throw new IonException("unexpected line terminator encountered in quoted string");
case '\\':
c = readEscapedCharacter(this.in, false /*not clob*/);
// throws UnexpectedEofException on EOF
if (c != EMPTY_ESCAPE_SEQUENCE) {
value.appendCodePoint(c);
}
break;
default:
// c could be part of a surrogate pair so we can't use
// appendCodePoint()
value.append((char)c);
break;
}
}
if (maxlookahead != -1 && c == '\"') {
// this is the normal, non-longline case so we're just done
closeString();
}
else {
// we're not at the closing quote, so this is an incomplete
// string which will have to be read to the end ... later
leaveOpenString(c, false);
}
return Type.constString;
}
void leaveOpenString(int c, boolean longstring) {
if (c == -1) {
throw new UnexpectedEofException();
}
this.isIncomplete = true;
this.inQuotedContent = true;
this.isLongString = longstring;
}
/**
* Scans the rest of the string through {@link #endquote}.
* <p>
XXX WARNING XXX
* Almost identical logic is found in
* {@link IonBinary#appendToLongValue(int, boolean, boolean, PushbackReader)}
*
* @param is_in_expression
* @throws UnexpectedEofException if we hit EOF before the endquote.
*/
void finishScanString(boolean is_in_expression) throws IOException
{
assert isIncomplete;
assert inQuotedContent;
for (;;)
{
int c = this.read();
if (c == -1) {
throw new UnexpectedEofException();
}
if (c == endquote) break; // endquote == -1 during long strings
if (c == '\\') {
c = readEscapedCharacter(this.in, false /*not clob*/);
if (c != EMPTY_ESCAPE_SEQUENCE) {
value.appendCodePoint(c);
}
}
else if (c == '\'' && isLongString) {
// This happens while handling triple-quote field names
if (twoMoreSingleQuotes()) {
// At the end of a long string. Look for another one.
inQuotedContent = false;
c = readIgnoreWhitespace();
if (c == '\'' && twoMoreSingleQuotes()) {
// Yep, another triple-quote. We've consumed all three
// so just keep reading.
inQuotedContent = true;
}
else {
// Something else, we are done.
unread(c);
break;
}
}
else {
// Still inside the long string
value.append((char)c);
}
}
else if (!isLongString && (c == '\n')) {
throw new IonException("unexpected line terminator encountered in quoted string");
}
else {
value.append((char)c);
}
}
return;
}
/**
* If two single quotes are next on the input, consume them and return
* true. Otherwise, leave them on the input and return false.
* @return true when there are two pending single quotes.
* @throws IOException
*/
private boolean twoMoreSingleQuotes() throws IOException
{
int c = read();
if (c == '\'') {
int c2 = read();
if (c2 == '\'') {
return true;
}
unread(c2);
}
unread(c);
return false;
}
void closeString() {
this.isIncomplete = false;
this.inQuotedContent = false;
this.isLongString = false;
}
public Type scanLongString() throws IOException
{
// reset out local value buffer
resetValue();
endquote = -1;
// we're not at the closing quote, so this is an incomplete
// string which will have to be read to the end ... later
leaveOpenString(EMPTY_ESCAPE_SEQUENCE, true);
return Type.constString;
}
/**
* Closes parsing of a triple-quote string, remembering that there may be
* more to come.
*/
void closeLongString() {
this.isIncomplete = true;
this.inQuotedContent = false;
this.isLongString = true;
}
/**
* @return the translated escape sequence. Will not return -1 since EOF is
* not legal in this context. May return {@link #EMPTY_ESCAPE_SEQUENCE} to
* indicate a zero-length sequence such as BS-NL.
* @throws UnexpectedEofException if the EOF is encountered in the middle
* of the escape sequence.
* */
public static int readEscapedCharacter(PushbackReader r, boolean inClob)
throws IOException, UnexpectedEofException
{
// \\ The backslash character
// \0 The character 0 (nul terminator char, for some)
// \xhh The character with hexadecimal value 0xhh
// \ u hhhh The character with hexadecimal value 0xhhhh
// \t The tab character ('\u0009')
// \n The newline (line feed) character ('\ u 000A')
// \v The vertical tab character ('\u000B')
// \r The carriage-return character ('\ u 000D')
// \f The form-feed character ('\ u 000C')
// \a The alert (bell) character ('\ u 0007')
// \" The double quote character
// \' The single quote character
// \? The question mark character
int c2 = 0, c = r.read();
switch (c) {
case -1:
throw new UnexpectedEofException();
case 't': return '\t';
case 'n': return '\n';
case 'v': return '\u000B';
case 'r': return '\r';
case 'f': return '\f';
case 'b': return '\u0008';
case 'a': return '\u0007';
case '\\': return '\\';
case '\"': return '\"';
case '\'': return '\'';
case '/': return '/';
case '?': return '?';
case 'U':
// Expecting 8 hex digits
if (inClob) {
throw new IonException("Unicode escapes \\U not allowed in clob");
}
c = readDigit(r, 16, true);
if (c < 0) break; // TODO throw UnexpectedEofException
c2 = c << 28;
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 24;
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 20;
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 16;
// ... fall through...
case 'u':
// Expecting 4 hex digits
if (inClob) {
throw new IonException("Unicode escapes \\u not allowed in clob");
}
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 12;
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 8;
// ... fall through...
case 'x':
// Expecting 2 hex digits
c = readDigit(r, 16, true);
if (c < 0) break;
c2 += c << 4;
c = readDigit(r, 16, true);
if (c < 0) break;
return c2 + c;
case '0':
return 0;
case '\n':
return EMPTY_ESCAPE_SEQUENCE;
default:
break;
}
throw new IonException("invalid escape sequence \"\\"
+ (char) c + "\" [" + c + "]");
}
public static int readDigit(/*EscapedCharacterReader*/PushbackReader r, int radix, boolean isRequired) throws IOException {
int c = r.read();
if (c < 0) {
r.unread(c);
return -1;
}
int c2 = Character.digit(c, radix);
if (c2 < 0) {
if (isRequired) {
throw new IonException("bad digit in escaped character '"+((char)c)+"'");
}
// if it's not a required digit, we just throw it back
r.unread(c);
return -1;
}
return c2;
}
private void checkAndUnreadNumericStopper(int c) throws IOException
{
// Ignore EOF, so we don't "unread" it - EOF is a terminator
if (c != -1) {
if ( ! this.isValueTerminatingCharacter(c) ) {
final String message =
position() + ": Numeric value followed by invalid character "
+ printCodePointAsString(c);
throw new IonException(message);
}
this.unread(c);
}
}
private final boolean isValueTerminatingCharacter(int c) throws IOException
{
boolean isTerminator;
if (c == '/') {
// this is terminating only if it starts a comment of some sort
c = this.read();
this.unread(c); // we never "keep" this character
isTerminator = (c == '/' || c == '*');
}
else {
isTerminator = IonTextUtils.isNumericStop(c);
}
return isTerminator;
}
public Type readNumber(int c) throws IOException {
// clear out our string buffer
resetValue();
// process the initial sign character, if its there.
boolean explicitPlusSign = false;
switch (c) {
case '-':
value.append((char)c);
c = this.read();
t = Type.constNegInt;
this.numberType = NumberType.NT_NEGINT;
this.isNegative = true;
break;
case '+':
// we eat the plus sign, but remember we saw one
explicitPlusSign = true;
c = this.read();
t = Type.constPosInt;
this.isNegative = false;
this.numberType = NumberType.NT_POSINT;
break;
default:
t = Type.constPosInt;
this.numberType = NumberType.NT_POSINT;
this.isNegative = false;
break;
}
// process the initial digit. Caller has checked that it's a digit.
assert isDigit(c, 10);
value.append((char)c);
boolean isZero = (c == '0');
boolean leadingZero = isZero;
// process the next character after the initial digit.
switch((c = this.read())) {
case 'x':
case 'X':
if (!isZero) {
throw new IonException("badly formed number encountered at " + position());
}
this.numberType = NumberType.NT_HEX;
// We don't need to append the char because we're going to wipe
// the value anyway to accumulate just the hex digits.
return scanHexNumber();
case '.':
case 'd':
case 'D':
t = Type.constDecimal;
this.numberType = NumberType.NT_DECIMAL;
value.append((char)c);
break;
case 'e':
case 'E':
t = Type.constFloat;
this.numberType = NumberType.NT_FLOAT;
value.append((char)c);
break;
default:
if (!isDigit(c, 10)) {
checkAndUnreadNumericStopper(c);
if (isZero && NumberType.NT_NEGINT.equals(this.numberType)) {
t = Type.constPosInt;
this.numberType = NumberType.NT_POSINT;
}
return t;
}
value.append((char)c);
isZero &= (c == '0');
break;
}
// see if see can continue find leading digits
if (NumberType.NT_NEGINT.equals(numberType)
|| NumberType.NT_POSINT.equals(numberType)
) {
// We've now scanned at least two digits.
// read in the remaining whole number digits
for (;;) {
c = this.read();
if (!isDigit(c, 10)) break;
value.append((char)c);
isZero &= (c == '0');
}
// and see what we ran aground on ..
switch (c) {
case '.':
case 'd':
case 'D':
if (leadingZero) {
throw new IonException(position() + ": Invalid leading zero on numeric");
}
t = Type.constDecimal;
this.numberType = NumberType.NT_DECIMAL;
value.append((char)c);
break;
case 'e':
case 'E':
if (leadingZero) {
throw new IonException(position() + ": Invalid leading zero on numeric");
}
t = Type.constFloat;
this.numberType = NumberType.NT_FLOAT;
value.append((char)c);
break;
case 'T': // same as '-' it's a timestamp
case '-':
if (NumberType.NT_POSINT.equals(this.numberType) && !explicitPlusSign) {
return scanTimestamp(c);
}
// otherwise fall through to "all other" processing
default:
// Hit the end of our integer. Make sure its a valid stopper.
checkAndUnreadNumericStopper(c);
if (leadingZero && !isZero) {
throw new IonException(position() + ": Invalid leading zero on numeric");
}
if (isZero && NumberType.NT_NEGINT.equals(this.numberType)) {
t = Type.constPosInt;
this.numberType = NumberType.NT_POSINT;
}
return t;
}
}
// now we must have just decimal or float
// if it's decimal we ended on a DOT, so get the trailing
// (fractional) digits
if (NumberType.NT_DECIMAL.equals(this.numberType)) {
for (;;) {
c = this.read();
if (!isDigit(c, 10)) break;
value.append((char)c);
}
// and see what we ran aground on this time..
switch (c) {
case '-':
case '+':
// We'll handle exponent below.
this.unread(c);
break;
case 'e':
case 'E':
// this is the only viable option now
t = Type.constFloat;
this.numberType = NumberType.NT_FLOAT;
value.append((char)c);
break;
case 'd':
case 'D':
// this is the only viable option now
assert t == Type.constDecimal;
assert NumberType.NT_DECIMAL.equals(this.numberType);
value.append((char)c);
break;
default:
checkAndUnreadNumericStopper(c);
return t;
}
}
// now we have the exponent part to read
// reading the first character of the exponent, it can be
// a sign character or a digit (this is the only place
// the sign is valid) and we're for sure a float at this point
switch ((c = this.read())) {
case '-':
value.append((char)c);
break;
case '+':
// we eat the plus sign
break;
default:
if (!isDigit(c, 10)) {
// this they said 'e' then they better put something valid after it!
throw new IonException("badly formed number encountered at " + position());
}
this.unread(c);
// we'll re-read this in just a bit, but we had to check
break;
}
// read in the remaining whole number digits and then quit
for (;;) {
c = this.read();
if (!isDigit(c, 10)) break;
value.append((char)c);
}
checkAndUnreadNumericStopper(c);
return t;
}
Type scanHexNumber() throws IOException
{
boolean anydigits = false;
int c;
boolean isZero = true;
// Leave the 0x in the buffer since we'll need it to parse the content.
// first get clear out our string buffer, we're starting over
value.setLength(0); // resetValue() wipes out too much
// read in the remaining whole number digits and then quit
for (;;) {
c = this.read();
if (!isDigit(c, 16)) break;
anydigits = true;
isZero &= (c == '0');
value.append((char)c);
}
if (!anydigits) {
throw new IonException("badly formed hexadecimal number encountered at " + position());
}
checkAndUnreadNumericStopper(c);
if (isZero && t == Type.constNegInt) {
t = Type.constPosInt;
}
// as long as we saw any digit, we're good
return t;
}
/**
* Scans a timestamp after reading <code>yyyy-</code>.
*
* We can be a little lenient here since the result will be reparsed and
* validated more thoroughly by {@link Timestamp#valueOf(CharSequence)}.
*
* @param c the last character scanned; must be <code>'-'</code>.
* @return {@link Type#constTime}
*/
Type scanTimestamp(int c) throws IOException {
endofdate:
for (;;) { // fake for loop to create a label we can jump out of,
// because 4 or 5 levels of nested if's is just ugly
// at this point we will have read leading digits and exactly 1 dash
// in other words, we'll have read the year
if (c == 'T') {
// yearT is a valid timestamp value
value.append((char)c);
c = this.read(); // because we'll unread it before we return
break endofdate;
}
if (c != '-') {
// not a dash or a T after the year - so this is a bad value
throw new IllegalStateException("invalid timestamp, expecting a dash here at " + this.position());
}
// append the dash and then read the month field
value.append((char)c); // so append it, because we haven't already
c = readDigits(2, "month");
if (c == 'T') {
// year-monthT is a valid timestamp value
value.append((char)c);
c = this.read(); // because we'll unread it before we return
break endofdate;
}
if (c != '-') {
// if the month isn't followed by a dash or a T it's an invalid month
throw new IonException("invalid timestamp, expecting month at " + this.position());
}
// append the dash and read the day (or day-of-month) field
value.append((char)c);
c = readDigits(2, "day of month");
if (c == 'T') {
check4timezone:
for (;;) { // another fake label/ for=goto
// attach the 'T' to the value we're collecting
value.append((char)c);
// we're going to "watch" how many digits we read in the hours
// field. It's 0 that's actually ok, since we can end at the
// 'T' we just read
int length_before_reading_hours = value.length();
// so read the hours
c = readDigits(2, "hours");
if (length_before_reading_hours == value.length()) {
// FIXME I don't think there should be a timezone here
break check4timezone;
}
if (c != ':') {
throw new IonException("invalid timestamp, expecting hours at " + this.position());
}
value.append((char)c);
// so read the minutes
c = readDigits(2, "minutes");
if (c != ':') {
if (c == '-' || c == '+' || c == 'Z') {
break check4timezone;
}
break endofdate;
}
value.append((char)c);
// so read the seconds
c = readDigits(2, "seconds");
if (c != '.') {
if (c == '-' || c == '+' || c == 'Z') {
break check4timezone;
}
break endofdate;
}
value.append((char)c);
// so read the fractional seconds
c = readDigits(32, "fractional seconds");
break check4timezone;
}//check4timezone
// now check to see if it's a timezone offset we're looking at
if (c == '-' || c == '+') {
value.append((char)c);
// so read the timezone offset
c = readDigits(2, "timezone offset");
if (c != ':') break endofdate;
value.append((char)c);
c = readDigits(2, "timezone offset");
}
else if (c == 'Z') {
value.append((char)c);
c = this.read(); // because we'll unread it before we return
}
}
break endofdate;
}//endofdate
checkAndUnreadNumericStopper(c);
return Type.constTime;
}
int readDigits(int limit, String field) throws IOException {
int c, len = 0;
// read in the remaining whole number digits and then quit
for (;;) {
c = this.read();
if (!isDigit(c, 10)) break;
len++;
if (len > limit) {
throw new IonException("invalid format " + field + " too long at " + this.position());
}
value.append((char)c);
}
return c;
}
public boolean makeValidNumeric(Type castto) throws IOException {
String s = getValueString(false);
try
{
this.t = castto.setNumericValue(this, s);
}
catch (NumberFormatException e)
{
throw new IonException(this.position() + ": invalid numeric value " + s, e);
}
return this.t.isNumeric();
}
}