IonBinary.java
/*
* Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.ion.impl;
import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_SIZE;
import static com.amazon.ion.impl._Private_Utils.EMPTY_BYTE_ARRAY;
import static com.amazon.ion.impl._Private_Utils.readFully;
import static com.amazon.ion.util.IonStreamUtils.isIonBinary;
import com.amazon.ion.Decimal;
import com.amazon.ion.IonException;
import com.amazon.ion.SymbolTable;
import com.amazon.ion.SymbolToken;
import com.amazon.ion.Timestamp;
import com.amazon.ion.Timestamp.Precision;
import com.amazon.ion.UnexpectedEofException;
import com.amazon.ion.impl._Private_IonConstants.HighNibble;
import com.amazon.ion.util.IonTextUtils;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PushbackReader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.util.ArrayList;
import java.util.Stack;
final class IonBinary
{
static boolean debugValidation = false;
private static final BigInteger MAX_LONG_VALUE = new BigInteger(Long.toString(Long.MAX_VALUE));
private static final int SIZE_OF_LONG = 8;
final static int _ib_TOKEN_LEN = 1;
final static int _ib_VAR_INT32_LEN_MAX = 5; // 31 bits (java limit) / 7 bits per byte = 5 bytes
final static int _ib_VAR_INT64_LEN_MAX = 10; // 63 bits (java limit) / 7 bits per byte = 10 bytes
final static int _ib_INT64_LEN_MAX = 8;
final static int _ib_FLOAT64_LEN = 8;
private static final Double DOUBLE_POS_ZERO = Double.valueOf(0.0);
static final int ZERO_DECIMAL_TYPEDESC =
_Private_IonConstants.makeTypeDescriptor(_Private_IonConstants.tidDecimal,
_Private_IonConstants.lnNumericZero);
static final int NULL_DECIMAL_TYPEDESC =
_Private_IonConstants.makeTypeDescriptor(_Private_IonConstants.tidDecimal,
_Private_IonConstants.lnIsNullAtom);
private IonBinary() { }
/**
* Verifies that a reader starts with a valid Ion cookie, throwing an
* exception if it does not.
*
* @param reader must not be null.
* @throws IonException if there's a problem reading the cookie, or if the
* data does not start with {@link _Private_IonConstants#BINARY_VERSION_MARKER_1_0}.
*/
public static void verifyBinaryVersionMarker(Reader reader)
throws IonException
{
try
{
int pos = reader.position();
//reader.sync();
//reader.setPosition(0);
byte[] bvm = new byte[BINARY_VERSION_MARKER_SIZE];
int len = readFully(reader, bvm);
if (len < BINARY_VERSION_MARKER_SIZE)
{
String message =
"Binary data is too short: at least " +
BINARY_VERSION_MARKER_SIZE +
" bytes are required, but only " + len + " were found.";
throw new IonException(message);
}
if (! isIonBinary(bvm))
{
StringBuilder buf = new StringBuilder();
buf.append("Binary data has unrecognized header");
for (int i = 0; i < bvm.length; i++)
{
int b = bvm[i] & 0xFF;
buf.append(" 0x");
buf.append(Integer.toHexString(b).toUpperCase());
}
throw new IonException(buf.toString());
}
//reader.setPosition(0); // cas 19 apr 2008
reader.setPosition(pos); // cas 5 may 2009 :)
}
catch (IOException e)
{
throw new IonException(e);
}
}
/* imported from SimpleByteBuffer.SimpleByteWriter */
static public int writeTypeDescWithLength(OutputStream userstream, int typeid, int lenOfLength, int valueLength) throws IOException
{
int written_len = 1;
int td = ((typeid & 0xf) << 4);
if (valueLength >= _Private_IonConstants.lnIsVarLen) {
td |= _Private_IonConstants.lnIsVarLen;
userstream.write((byte)(td & 0xff));
written_len += writeVarUInt(userstream, (long)valueLength, lenOfLength, true);
}
else {
td |= (valueLength & 0xf);
userstream.write((byte)(td & 0xff));
}
return written_len;
}
static public int writeTypeDescWithLength(OutputStream userstream, int typeid, int valueLength) throws IOException
{
int written_len = 1;
int td = ((typeid & 0xf) << 4);
if (valueLength >= _Private_IonConstants.lnIsVarLen) {
td |= _Private_IonConstants.lnIsVarLen;
userstream.write((byte)(td & 0xff));
int lenOfLength = IonBinary.lenVarUInt(valueLength);
written_len += writeVarUInt(userstream, (long)valueLength, lenOfLength, true);
}
else {
td |= (valueLength & 0xf);
userstream.write((byte)(td & 0xff));
}
return written_len;
}
static public int writeIonInt(OutputStream userstream, long value, int len) throws IOException
{
// we shouldn't be writing out 0's as an Ion int value
if (value == 0) {
assert len == 0;
return len; // aka 0
}
// figure out how many we have bytes we have to write out
long mask = 0xffL;
boolean is_negative = (value < 0);
assert len == IonBinary.lenIonInt(value);
if (is_negative) {
value = -value;
// note for Long.MIN_VALUE the negation returns
// itself as a value, but that's also the correct
// "positive" value to write out anyway, so it
// all works out
}
// write the rest
switch (len) { // we already wrote 1 byte
case 8: userstream.write((byte)((value >> (8*7)) & mask));
case 7: userstream.write((byte)((value >> (8*6)) & mask));
case 6: userstream.write((byte)((value >> (8*5)) & mask));
case 5: userstream.write((byte)((value >> (8*4)) & mask));
case 4: userstream.write((byte)((value >> (8*3)) & mask));
case 3: userstream.write((byte)((value >> (8*2)) & mask));
case 2: userstream.write((byte)((value >> (8*1)) & mask));
case 1: userstream.write((byte)(value & mask));
}
return len;
}
static public int writeVarUInt(OutputStream userstream, long value)
throws IOException
{
int len = IonBinary.lenVarUInt(value);
writeVarUInt(userstream, value, len, false);
return len;
}
static public int writeVarUInt(OutputStream userstream, long value, int len,
boolean force_zero_write)
throws IOException
{
int mask = 0x7F;
assert len == IonBinary.lenVarUInt(value);
assert value >= 0;
switch (len - 1) {
case 9: userstream.write((byte)((value >> (7*9)) & mask));
case 8: userstream.write((byte)((value >> (7*8)) & mask));
case 7: userstream.write((byte)((value >> (7*7)) & mask));
case 6: userstream.write((byte)((value >> (7*6)) & mask));
case 5: userstream.write((byte)((value >> (7*5)) & mask));
case 4: userstream.write((byte)((value >> (7*4)) & mask));
case 3: userstream.write((byte)((value >> (7*3)) & mask));
case 2: userstream.write((byte)((value >> (7*2)) & mask));
case 1: userstream.write((byte)((value >> (7*1)) & mask));
case 0: userstream.write((byte)((value & mask) | 0x80L));
break;
case -1: // or len == 0
if (force_zero_write) {
userstream.write((byte)0x80);
assert len == 1;
}
else {
assert len == 0;
}
break;
}
return len;
}
static public int writeString(OutputStream userstream, String value)
throws IOException
{
int len = 0;
for (int ii=0; ii<value.length(); ii++) {
int c = value.charAt(ii);
if (c < 128) {
userstream.write((byte)c);
++len;
continue;
}
// multi-byte utf8
if (c >= 0xD800 && c <= 0xDFFF) {
if (_Private_IonConstants.isHighSurrogate(c)) {
// houston we have a high surrogate (let's hope it has a partner
if (++ii >= value.length()) {
throw new IllegalArgumentException("invalid string, unpaired high surrogate character");
}
int c2 = value.charAt(ii);
if (!_Private_IonConstants.isLowSurrogate(c2)) {
throw new IllegalArgumentException("invalid string, unpaired high surrogate character");
}
c = _Private_IonConstants.makeUnicodeScalar(c, c2);
}
else if (_Private_IonConstants.isLowSurrogate(c)) {
// it's a loner low surrogate - that's an error
throw new IllegalArgumentException("invalid string, unpaired low surrogate character");
}
}
for (c = makeUTF8IntFromScalar(c); (c & 0xffffff00) != 0; ++len) {
userstream.write((byte)(c & 0xff));
c = c >>> 8;
}
}
return len;
}
// TODO: move this to IonConstants or IonUTF8
static final public int makeUTF8IntFromScalar(int c) throws IOException
{
// TO DO: check this encoding, it is from:
// http://en.wikipedia.org/wiki/UTF-8
// we probably should use some sort of Java supported
// library for this. this class might be of interest:
// CharsetDecoder(Charset cs, float averageCharsPerByte, float maxCharsPerByte)
// in: java.nio.charset.CharsetDecoder
int value = 0;
// first the quick, easy and common case - ascii
if (c < 0x80) {
value = (0xff & c );
}
else if (c < 0x800) {
// 2 bytes characters from 0x000080 to 0x0007FF
value = ( 0xff & (0xC0 | (c >> 6) ) );
value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 8;
}
else if (c < 0x10000) {
// 3 byte characters from 0x800 to 0xFFFF
// but only 0x800...0xD7FF and 0xE000...0xFFFF are valid
if (c > 0xD7FF && c < 0xE000) {
throwUTF8Exception();
}
value = ( 0xff & (0xE0 | (c >> 12) ) );
value |= ( 0xff & (0x80 | ((c >> 6) & 0x3F)) ) << 8;
value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 16;
}
else if (c <= 0x10FFFF) {
// 4 byte characters 0x010000 to 0x10FFFF
// these are are valid
value = ( 0xff & (0xF0 | (c >> 18)) );
value |= ( 0xff & (0x80 | ((c >> 12) & 0x3F)) ) << 8;
value |= ( 0xff & (0x80 | ((c >> 6) & 0x3F)) ) << 16;
value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 24;
}
else {
throwUTF8Exception();
}
return value;
}
static void throwUTF8Exception() throws IOException
{
throw new IOException("Invalid UTF-8 character encountered");
}
public static class BufferManager
{
BlockedBuffer _buf;
IonBinary.Reader _reader;
IonBinary.Writer _writer;
public BufferManager() {
_buf = new BlockedBuffer();
this.openReader();
this.openWriter();
}
public BufferManager(BlockedBuffer buf) {
_buf = buf;
this.openReader();
this.openWriter();
}
/**
* Creates a new buffer containing the entire content of an
* {@link InputStream}.
*
* @param bytestream a stream interface the byte image to buffer
*
* @throws IonException wrapping any {@link IOException}s thrown by the
* stream.
*/
public BufferManager(InputStream bytestream)
{
this(); // this will create a fresh buffer
// as well as a reader and writer
// now we move the data from the input stream to the buffer
// more or less as fast as we can. I am (perhaps foolishly)
// assuming the "available()" is a useful size.
try
{
_writer.write(bytestream);
}
catch (IOException e)
{
throw new IonException(e);
}
}
/**
* Creates a new buffer containing the entire content of an
* {@link InputStream}.
*
* @param bytestream a stream interface the byte image to buffer
*
* @throws IonException wrapping any {@link IOException}s thrown by the
* stream.
*/
public BufferManager(InputStream bytestream, int len)
{
this(); // this will create a fresh buffer
// as well as a reader and writer
// now we move the data from the input stream to the buffer
// more or less as fast as we can. I am (perhaps foolishly)
// assuming the "available()" is a useful size.
try
{
_writer.write(bytestream, len);
}
catch (IOException e)
{
throw new IonException(e);
}
}
@Override
public BufferManager clone() throws CloneNotSupportedException
{
BlockedBuffer buffer_clone = this._buf.clone();
BufferManager clone = new BufferManager(buffer_clone);
return clone;
}
public IonBinary.Reader openReader() {
if (_reader == null) {
_reader = new IonBinary.Reader(_buf);
}
return _reader;
}
public IonBinary.Writer openWriter() {
if (_writer == null) {
_writer = new IonBinary.Writer(_buf);
}
return _writer;
}
public BlockedBuffer buffer() { return _buf; }
public IonBinary.Reader reader() { return _reader; }
public IonBinary.Writer writer() { return _writer; }
public IonBinary.Reader reader(int pos) throws IOException
{
_reader.setPosition(pos);
return _reader;
}
public IonBinary.Writer writer(int pos) throws IOException
{
_writer.setPosition(pos);
return _writer;
}
public static BufferManager makeReadManager(BlockedBuffer buf) {
BufferManager bufmngr = new BufferManager(buf);
bufmngr.openReader();
return bufmngr;
}
public static BufferManager makeReadWriteManager(BlockedBuffer buf) {
BufferManager bufmngr = new BufferManager(buf);
bufmngr.openReader();
bufmngr.openWriter();
return bufmngr;
}
}
/**
* Variable-length, high-bit-terminating integer, 7 data bits per byte.
*/
public static int lenVarUInt(long longVal) {
assert longVal >= 0;
if (longVal < (1L << (7 * 1))) return 1; // 7 bits
if (longVal < (1L << (7 * 2))) return 2; // 14 bits
if (longVal < (1L << (7 * 3))) return 3; // 21 bits
if (longVal < (1L << (7 * 4))) return 4; // 28 bits
if (longVal < (1L << (7 * 5))) return 5; // 35 bits
if (longVal < (1L << (7 * 6))) return 6; // 42 bits
if (longVal < (1L << (7 * 7))) return 7; // 49 bits
if (longVal < (1L << (7 * 8))) return 8; // 56 bits
if (longVal < (1L << (7 * 9))) return 9; // 63 bits
return 10;
}
// TODO maybe add lenVarInt(int) to micro-optimize, or?
public static int lenVarInt(long longVal) {
if (longVal == 0) {
return 0;
}
if (longVal < 0) longVal = -longVal;
if (longVal < (1L << (7 * 1 - 1))) return 1; // 6 bits
if (longVal < (1L << (7 * 2 - 1))) return 2; // 13 bits
if (longVal < (1L << (7 * 3 - 1))) return 3; // 20 bits
if (longVal < (1L << (7 * 4 - 1))) return 4; // 27 bits
if (longVal < (1L << (7 * 5 - 1))) return 5; // 34 bits
if (longVal < (1L << (7 * 6 - 1))) return 6; // 41 bits
if (longVal < (1L << (7 * 7 - 1))) return 7; // 48 bits
if (longVal < (1L << (7 * 8 - 1))) return 8; // 55 bits
if (longVal < (1L << (7 * 9 - 1))) return 9; // 62 bits
return 10;
}
/**
* @return zero if input is zero
*/
public static int lenUInt(long longVal) {
if (longVal == 0) {
return 0;
}
if (longVal < 0) {
throw new BlockedBuffer.BlockedBufferException("fatal signed long where unsigned was promised");
}
if (longVal < (1L << (8 * 1))) return 1; // 8 bits
if (longVal < (1L << (8 * 2))) return 2; // 16 bits
if (longVal < (1L << (8 * 3))) return 3; // 24 bits
if (longVal < (1L << (8 * 4))) return 4; // 32 bits
if (longVal < (1L << (8 * 5))) return 5; // 40 bits
if (longVal < (1L << (8 * 6))) return 6; // 48 bits
if (longVal < (1L << (8 * 7))) return 7; // 56 bits
return 8;
}
public static int lenUInt(BigInteger bigVal)
{
if (bigVal.signum() < 0) {
throw new IllegalArgumentException("lenUInt expects a non-negative a value");
}
final int bits = bigVal.bitLength();
// determine the number of octets needed to represent this bit pattern
// (div 8)
int bytes = bits >> 3;
// if we have a bit more than what can fit in an octet, we need to add
// an extra octet (mod 8)
if ((bits & 0x7) != 0)
{
bytes++;
}
return bytes;
}
// TODO maybe add lenInt(int) to micro-optimize, or?
public static int lenInt(long longVal) {
if (longVal != 0) {
return 0;
}
if (longVal < 0) longVal = -longVal;
if (longVal < (1L << (8 * 1 - 1))) return 1; // 7 bits
if (longVal < (1L << (8 * 2 - 1))) return 2; // 15 bits
if (longVal < (1L << (8 * 3 - 1))) return 3; // 23 bits
if (longVal < (1L << (8 * 4 - 1))) return 4; // 31 bits
if (longVal < (1L << (8 * 5 - 1))) return 5; // 39 bits
if (longVal < (1L << (8 * 6 - 1))) return 6; // 47 bits
if (longVal < (1L << (8 * 7 - 1))) return 7; // 55 bits
if (longVal == Long.MIN_VALUE) return 9;
return 8;
}
public static int lenInt(BigInteger bi, boolean force_zero_writes)
{
int len = bi.abs().bitLength() + 1; // add 1 for the sign bit (which must always be present)
int bytelen = 0;
switch (bi.signum()) {
case 0:
bytelen = force_zero_writes ? 1 : 0;
break;
case 1:
case -1:
bytelen = ((len-1) / 8) + 1;
break;
}
return bytelen;
}
public static int lenIonInt(long v) {
if (v < 0) {
// note that for Long.MIN_VALUE (0x8000000000000000) the negative
// is the same, but that's also the bit pattern we need to
// write out, but the UInt method won't like it, so we just
// return then value that we actually know.
if (v == Long.MIN_VALUE) return SIZE_OF_LONG;
return IonBinary.lenUInt(-v);
}
else if (v > 0) {
return IonBinary.lenUInt(v);
}
return 0; // CAS UPDATE, was 1
}
public static int lenIonInt(BigInteger v)
{
if (v.signum() < 0)
{
v = v.negate();
}
int len = lenUInt(v);
return len;
}
/**
* The size of length value when short lengths are recorded in the
* typedesc low-nibble.
* @param valuelen
* @return zero if valuelen < 14
*/
public static int lenLenFieldWithOptionalNibble(int valuelen) {
if (valuelen < _Private_IonConstants.lnIsVarLen) {
return 0;
}
return lenVarUInt(valuelen);
}
public static int lenTypeDescWithAppropriateLenField(int type, int valuelen)
{
switch (type) {
case _Private_IonConstants.tidNull: // null(0)
case _Private_IonConstants.tidBoolean: // boolean(1)
return _Private_IonConstants.BB_TOKEN_LEN;
case _Private_IonConstants.tidPosInt: // 2
case _Private_IonConstants.tidNegInt: // 3
case _Private_IonConstants.tidFloat: // float(4)
case _Private_IonConstants.tidDecimal: // decimal(5)
case _Private_IonConstants.tidTimestamp: // timestamp(6)
case _Private_IonConstants.tidSymbol: // symbol(7)
case _Private_IonConstants.tidString: // string (8)
case _Private_IonConstants.tidClob: // clob(9)
case _Private_IonConstants.tidBlob: // blob(10)
case _Private_IonConstants.tidList: // 11 -- cas mar 6 2008, moved containers
case _Private_IonConstants.tidSexp: // 12 -- up heresince they now use the same
case _Private_IonConstants.tidStruct: // 13 -- length encodings as scalars
case _Private_IonConstants.tidTypedecl: // 14
if (valuelen < _Private_IonConstants.lnIsVarLen) {
return _Private_IonConstants.BB_TOKEN_LEN;
}
return _Private_IonConstants.BB_TOKEN_LEN + lenVarUInt(valuelen);
case _Private_IonConstants.tidUnused: // unused(15)
default:
throw new IonException("invalid type");
}
}
public static int lenIonFloat(double value) {
if (Double.valueOf(value).equals(DOUBLE_POS_ZERO))
{
// pos zero special case
return 0;
}
// always 8-bytes for IEEE-754 64-bit
return _ib_FLOAT64_LEN;
}
/**
* Would this value have a zero length-nibble? That is: is it 0d0 ?
*/
public static boolean isNibbleZero(BigDecimal bd)
{
if (Decimal.isNegativeZero(bd)) return false;
if (bd.signum() != 0) return false;
int scale = bd.scale();
return (scale == 0);
}
/**
* @param bd must not be null.
*/
public static int lenIonDecimal(BigDecimal bd)
{
// first check for the special cases of null
// and 0d0 which are encoded in the nibble
if (bd == null) return 0;
if (isNibbleZero(bd)) return 0;
// otherwise do this the hard way
BigInteger mantissa = bd.unscaledValue();
// negative zero mantissa must be written, positive zero does not
boolean forceMantissa = Decimal.isNegativeZero(bd);
int mantissaByteCount = lenInt(mantissa, forceMantissa);
// We really need the length of the exponent (-scale) but in our
// representation the length is the same regardless of sign.
int scale = bd.scale();
int exponentByteCount = lenVarInt(scale);
// Exponent is always at least one byte.
if (exponentByteCount == 0) exponentByteCount = 1;
return exponentByteCount + mantissaByteCount;
}
/**
* this method computes the output length of this timestamp value
* in the Ion binary format. It does not include the length of
* the typedesc byte that preceeds the actual value. The output
* length of a null value is 0, as a result this this.
* @param di may be null
*/
public static int lenIonTimestamp(Timestamp di)
{
if (di == null) return 0;
int len = 0;
switch (di.getPrecision()) {
case FRACTION:
case SECOND:
{
BigDecimal fraction = di.getFractionalSecond();
if (fraction != null)
{
assert fraction.signum() >=0 && ! fraction.equals(BigDecimal.ZERO)
: "Bad timestamp fraction: " + fraction;
// Since the fraction is not 0d0, at least one subfield of the
// exponent and mantissa is non-zero, so this will always write at
// least one byte.
int fracLen = IonBinary.lenIonDecimal(fraction);
assert fracLen > 0;
len += fracLen;
}
len++; // len of seconds < 60
}
case MINUTE:
len += 2; // len of hour and minutes (both < 127)
case DAY:
len += 1; // len of month and day (both < 127)
case MONTH:
len += 1; // len of month and day (both < 127)
case YEAR:
len += IonBinary.lenVarUInt(di.getZYear());
}
Integer offset = di.getLocalOffset();
if (offset == null) {
len++; // room for the -0 (i.e. offset is "no specified offset")
}
else if (offset == 0) {
len++;
}
else {
len += IonBinary.lenVarInt(offset.longValue());
}
return len;
}
/**
* @param v may be null.
* @throws IllegalArgumentException if the text contains bad UTF-16 data.
*/
public static int lenIonString(String v)
{
if (v == null) return 0;
int len = 0;
for (int ii=0; ii<v.length(); ii++) {
int c = v.charAt(ii);
// handle the cheap characters quickly
if (c < 128) {
len++;
continue;
}
// multi-byte utf8
if (c >= 0xD800 && c <= 0xDFFF) {
// look for surrogate pairs and merge them (and throw on bad data)
if (_Private_IonConstants.isHighSurrogate(c)) {
ii++;
if (ii >= v.length()) {
String message =
"Text ends with unmatched UTF-16 surrogate " +
IonTextUtils.printCodePointAsString(c);
throw new IllegalArgumentException(message);
}
int c2 = v.charAt(ii);
if (!_Private_IonConstants.isLowSurrogate(c2)) {
String message =
"Text contains unmatched UTF-16 high surrogate " +
IonTextUtils.printCodePointAsString(c) +
" at index " + (ii-1);
throw new IllegalArgumentException(message);
}
c = _Private_IonConstants.makeUnicodeScalar(c, c2);
}
else if (_Private_IonConstants.isLowSurrogate(c)) {
String message =
"Text contains unmatched UTF-16 low surrogate " +
IonTextUtils.printCodePointAsString(c) +
" at index " + ii;
throw new IllegalArgumentException(message);
}
}
// no need to check the 0x10FFFF overflow as it is checked in lenUnicodeScalarAsUTF8
// and now figure out how long this "complicated" (non-ascii) character is
if (c < 0x80) {
++len;
}
else if (c < 0x800) {
len += 2;
}
else if (c < 0x10000) {
len += 3;
}
else if (c <= 0x10FFFF) { // just about as cheap as & == 0 and checks for some out of range values
len += 4;
} else {
// TODO how is this possible?
throw new IllegalArgumentException("invalid string, illegal Unicode scalar (character) encountered");
}
}
return len;
}
public static int lenAnnotationListWithLen(String[] annotations,
SymbolTable symbolTable)
{
int annotationLen = 0;
if (annotations != null) {
// add up the length of the encoded symbols
for (int ii=0; ii<annotations.length; ii++) {
int symid = symbolTable.findSymbol(annotations[ii]);
assert symid > 0; // TODO amazon-ion/ion-java/issues/12
annotationLen += IonBinary.lenVarUInt(symid);
}
// now the len of the list
annotationLen += IonBinary.lenVarUInt(annotationLen);
}
return annotationLen;
}
public static int lenAnnotationListWithLen(ArrayList<Integer> annotations)
{
int annotationLen = 0;
// add up the length of the encoded symbols
for (Integer ii : annotations) {
int symid = ii.intValue();
annotationLen += IonBinary.lenVarUInt(symid);
}
// now the len of the list
annotationLen += IonBinary.lenVarUInt(annotationLen);
return annotationLen;
}
public static int lenIonNullWithTypeDesc() {
return _ib_TOKEN_LEN;
}
public static int lenIonBooleanWithTypeDesc(Boolean v) {
return _ib_TOKEN_LEN;
}
public static int lenIonIntWithTypeDesc(Long v) {
int len = 0;
if (v != null) {
long vl = v.longValue();
int vlen = lenIonInt(vl);
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
public static int lenIonFloatWithTypeDesc(Double v) {
int len = 0;
if (v != null) {
int vlen = lenIonFloat(v);
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
public static int lenIonDecimalWithTypeDesc(BigDecimal v) {
int len = 0;
if (v != null) {
int vlen = lenIonDecimal(v);
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
public static int lenIonTimestampWithTypeDesc(Timestamp di) {
int len = 0;
if (di != null) {
int vlen = IonBinary.lenIonTimestamp(di);
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
public static int lenIonStringWithTypeDesc(String v) {
int len = 0;
if (v != null) {
int vlen = lenIonString(v);
if (vlen < 0) return -1;
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
public static int lenIonClobWithTypeDesc(String v) {
return lenIonStringWithTypeDesc(v);
}
public static int lenIonBlobWithTypeDesc(byte[] v) {
int len = 0;
if (v != null) {
int vlen = v.length;
len += vlen;
len += lenLenFieldWithOptionalNibble(vlen);
}
return len + _ib_TOKEN_LEN;
}
/** Utility method to convert an unsigned magnitude stored as a long to a {@link BigInteger}. */
public static BigInteger unsignedLongToBigInteger(int signum, long val)
{
byte[] magnitude = {
(byte) ((val >> 56) & 0xFF),
(byte) ((val >> 48) & 0xFF),
(byte) ((val >> 40) & 0xFF),
(byte) ((val >> 32) & 0xFF),
(byte) ((val >> 24) & 0xFF),
(byte) ((val >> 16) & 0xFF),
(byte) ((val >> 8) & 0xFF),
(byte) (val & 0xFF),
};
return new BigInteger(signum, magnitude);
}
/**
* Uses {@link InputStream#read(byte[], int, int)} until the entire length is read.
* This method will block until the request is satisfied.
*
* @param in The stream to read from.
* @param buf The buffer to read to.
* @param offset The offset of the buffer to read from.
* @param len The length of the data to read.
*/
public static void readAll(InputStream in, byte[] buf, int offset, int len) throws IOException
{
int rem = len;
while (rem > 0)
{
int amount = in.read(buf, offset, rem);
if (amount <= 0)
{
// try to throw a useful exception
if (in instanceof Reader)
{
((Reader) in).throwUnexpectedEOFException();
}
// defer to a plain exception
throw new IonException("Unexpected EOF");
}
rem -= amount;
offset += amount;
}
}
public static final class Reader
extends BlockedBuffer.BlockedByteInputStream
{
/**
* @param bb blocked buffer to read from
*/
public Reader(BlockedBuffer bb)
{
super(bb);
}
/**
* @param bb blocked buffer to read from
* @param pos initial offset to read
*/
public Reader(BlockedBuffer bb, int pos)
{
super(bb, pos);
}
/**
* return the underlying bytes as a single buffer
*
* @return bytes[]
* @throws IOException
* @throws UnexpectedEofException if end of file is hit.
* @throws IOException if there's other problems reading input.
*/
public byte[] getBytes() throws IOException {
if (this._buf == null) return null;
this.sync();
this.setPosition(0);
int len = _buf.size();
byte[] buf = new byte[len];
if (readFully(this, buf) != len) {
throw new UnexpectedEofException();
}
return buf;
}
/**
* Read exactly one byte of input.
*
* @return 0x00 through 0xFF as a positive int.
* @throws UnexpectedEofException if end of file is hit.
* @throws IOException if there's other problems reading input.
*/
public int readToken() throws UnexpectedEofException, IOException
{
int c = read();
if (c < 0) throwUnexpectedEOFException();
return c;
}
public int readActualTypeDesc() throws IOException
{
int c = read();
if (c < 0) throwUnexpectedEOFException();
int typeid = _Private_IonConstants.getTypeCode(c);
if (typeid == _Private_IonConstants.tidTypedecl) {
int lownibble = _Private_IonConstants.getLowNibble(c);
if (lownibble == 0) {
// 0xE0 is the first byte of the IonVersionMarker
// so we'll return it as is, the caller has to
// verify the remaining bytes and handle them
// appropriately - so here we do nothing
}
else {
this.readLength(typeid, lownibble);
int alen = this.readVarIntAsInt();
// TODO add skip(int) method instead of this loop.
while (alen > 0) {
if (this.read() < 0) throwUnexpectedEOFException();
alen--;
}
c = read();
if (c < 0) throwUnexpectedEOFException();
}
}
return c;
}
public int[] readAnnotations() throws IOException
{
int[] annotations = null;
int annotationLen = this.readVarUIntAsInt();
int annotationPos = this.position(); // pos at the first ann sid
int annotationEnd = annotationPos + annotationLen;
int annotationCount = 0;
// first we read through and count
while(this.position() < annotationEnd) {
// read the annotation symbol id itself
// and for this first pass we just throw that
// value away, since we're just counting
this.readVarUIntAsInt();
annotationCount++;
}
if (annotationCount > 0) {
// then, if there are any there, we
// allocate the array, and re-read the sids
// look them up and fill in the array
annotations = new int[annotationCount];
int annotationIdx = 0;
this.setPosition(annotationPos);
while(this.position() < annotationEnd) {
// read the annotation symbol id itself
int sid = this.readVarUIntAsInt();
annotations[annotationIdx++] = sid;
}
}
return annotations;
}
public int readLength(int td, int ln) throws IOException
{
// TODO check for invalid lownibbles
switch (td) {
case _Private_IonConstants.tidNull: // null(0)
case _Private_IonConstants.tidBoolean: // boolean(1)
return 0;
case _Private_IonConstants.tidPosInt: // 2
case _Private_IonConstants.tidNegInt: // 3
case _Private_IonConstants.tidFloat: // float(4)
case _Private_IonConstants.tidDecimal: // decimal(5)
case _Private_IonConstants.tidTimestamp: // timestamp(6)
case _Private_IonConstants.tidSymbol: // symbol(7)
case _Private_IonConstants.tidString: // string (8)
case _Private_IonConstants.tidClob: // clob(9)
case _Private_IonConstants.tidBlob: // blob(10)
case _Private_IonConstants.tidList: // 11
case _Private_IonConstants.tidSexp: // 12
case _Private_IonConstants.tidTypedecl: // 14
switch (ln) {
case 0:
case _Private_IonConstants.lnIsNullAtom:
return 0;
case _Private_IonConstants.lnIsVarLen:
return readVarUIntAsInt();
default:
return ln;
}
case _Private_IonConstants.tidNopPad: // 99
switch (ln) {
case _Private_IonConstants.lnIsVarLen:
return readVarUIntAsInt();
default:
return ln;
}
case _Private_IonConstants.tidStruct: // 13
switch (ln) {
case _Private_IonConstants.lnIsEmptyContainer:
case _Private_IonConstants.lnIsNullStruct:
return 0;
case _Private_IonConstants.lnIsOrderedStruct:
case _Private_IonConstants.lnIsVarLen:
return readVarUIntAsInt();
default:
return ln;
}
case _Private_IonConstants.tidUnused: // unused(15)
default:
// TODO use InvalidBinaryDataException
throw new BlockedBuffer.BlockedBufferException("invalid type id encountered: " + td);
}
}
/*
public long readFixedIntLongValue(int len) throws IOException {
long retvalue = 0;
int b;
switch (len) {
case 8:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (7*8);
case 7:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (6*8);
case 6:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (5*8);
case 5:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (4*8);
case 4:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (3*8);
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (2*8);
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (1*8);
case 1:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (0*8);
}
return retvalue;
}
public int readFixedIntIntValue(int len) throws IOException {
int retvalue = 0;
int b;
switch (len) {
case 4:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (3*8);
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (2*8);
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (1*8);
case 1:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue |= b << (0*8);
}
return retvalue;
}
*/
public long readIntAsLong(int len) throws IOException {
long retvalue = 0;
boolean is_negative = false;
int b;
if (len > 0) {
// read the first byte
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (b & 0x7F);
is_negative = ((b & 0x80) != 0);
switch (len - 1) { // we read 1 already
case 8: // even after reading the 1st byte we may have 8 remaining
// bytes of value when the value is Long.MIN_VALUE since it
// has all 8 bytes for data and the ninth for the sign bit
// all by itself (which we read above)
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 7:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 6:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 5:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 4:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 1:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
default:
}
if (is_negative) {
// this is correct even when retvalue == Long.MIN_VALUE
retvalue = -retvalue;
}
}
return retvalue;
}
/** @throws IOException
* @deprecated */
@Deprecated
public int readIntAsInt(int len) throws IOException {
int retvalue = 0;
boolean is_negative = false;
int b;
if (len > 0) {
// read the first byte
b = read();
if (b < 0) throwUnexpectedEOFException();
retvalue = (b & 0x7F);
is_negative = ((b & 0x80) != 0);
switch (len - 1) { // we read 1 already
case 7:
case 6:
case 5:
case 4:
throw new IonException("overflow attempt to read long value into an int");
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
}
if (is_negative) {
// this is correct even when retvalue == Integer.MIN_VALUE
retvalue = -retvalue;
}
}
return retvalue;
}
/**
* Reads the specified magnitude as a {@link BigInteger}.
*
* @param len The length of the UInt octets to read.
* @param signum The sign as per {@link BigInteger#BigInteger(int, byte[])}.
* @return The signed {@link BigInteger}.
*
* @throws IOException Thrown if there are an I/O errors on the underlying stream.
*/
public BigInteger readUIntAsBigInteger(int len, int signum) throws IOException {
byte[] magnitude = new byte[len];
for (int i = 0; i < len; i++) {
int octet = read();
if (octet < 0) {
throwUnexpectedEOFException();
}
magnitude[i] = (byte) octet;
}
// both BigInteger and ion store this magnitude as big-endian
return new BigInteger(signum, magnitude);
}
public long readUIntAsLong(int len) throws IOException {
long retvalue = 0;
int b;
switch (len) {
default:
throw new IonException("overflow attempt to read long value into an int");
case 8:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 7:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 6:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 5:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 4:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 1:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 0:
}
return retvalue;
}
public int readUIntAsInt(int len) throws IOException {
int retvalue = 0;
int b;
switch (len) {
default:
throw new IonException("overflow attempt to read long value into an int");
case 4:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = b;
case 3:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 2:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 1:
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 8) | b;
case 0:
}
return retvalue;
}
public long readVarUIntAsLong() throws IOException {
long retvalue = 0;
int b;
for (;;) {
if ((b = read()) < 0) throwUnexpectedEOFException();
if ((retvalue & 0xFE00000000000000L) != 0) {
// if any of the top 7 bits are set at this point there's
// a problem, because we'll be shifting the into oblivian
// just below, so ...
throw new IonException("overflow attempt to read long value into a long");
}
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
}
return retvalue;
}
public int readVarUIntAsInt() throws IOException {
int retvalue = 0;
int b;
for (;;) { // fake loop to create a "goto done"
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break;
// if we get here we have more bits than we have room for :(
throw new IonException("var int overflow at: "+this.position());
}
return retvalue;
}
public long readVarIntAsLong() throws IOException
{
long retvalue = 0;
boolean is_negative = false;
int b;
// synthetic label "done" (yuck)
done: for (;;) {
// read the first byte - it has the sign bit
if ((b = read()) < 0) throwUnexpectedEOFException();
if ((b & 0x40) != 0) {
is_negative = true;
}
retvalue = (b & 0x3F);
if ((b & 0x80) != 0) break done;
// for the second byte we shift our eariler bits just as much,
// but there are fewer of them there to shift
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
for (;;) {
if ((b = read()) < 0) throwUnexpectedEOFException();
if ((retvalue & 0xFE00000000000000L) != 0) {
// if any of the top 7 bits are set at this point there's
// a problem, because we'll be shifting the into oblivian
// just below, so ...
throw new IonException("overflow attempt to read long value into a long");
}
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
}
}
if (is_negative) {
// this is correct even when retvalue == Long.MIN_VALUE
retvalue = -retvalue;
}
return retvalue;
}
public int readVarIntAsInt() throws IOException
{
int retvalue = 0;
boolean is_negative = false;
int b;
// synthetic label "done" (yuck)
done: for (;;) {
// read the first byte - it has the sign bit
if ((b = read()) < 0) throwUnexpectedEOFException();
if ((b & 0x40) != 0) {
is_negative = true;
}
retvalue = (b & 0x3F);
if ((b & 0x80) != 0) break done;
// for the second byte we shift our eariler bits just as much,
// but there are fewer of them there to shift
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// if we get here we have more bits than we have room for :(
throw new IonException("var int overflow at: "+this.position());
}
if (is_negative) {
// this is correct even when retvalue == Integer.MIN_VALUE
retvalue = -retvalue;
}
return retvalue;
}
/**
* Reads an integer value, returning null to mean -0.
* @throws IOException
*/
public Integer readVarIntWithNegativeZero() throws IOException
{
int retvalue = 0;
boolean is_negative = false;
int b;
// sythetic label "done" (yuck)
done: for (;;) {
// read the first byte - it has the sign bit
if ((b = read()) < 0) throwUnexpectedEOFException();
if ((b & 0x40) != 0) {
is_negative = true;
}
retvalue = (b & 0x3F);
if ((b & 0x80) != 0) break done;
// for the second byte we shift our eariler bits just as much,
// but there are fewer of them there to shift
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// for the rest, they're all the same
if ((b = read()) < 0) throwUnexpectedEOFException();
retvalue = (retvalue << 7) | (b & 0x7F);
if ((b & 0x80) != 0) break done;
// if we get here we have more bits than we have room for :(
throw new IonException("var int overflow at: "+this.position());
}
Integer retInteger = null;
if (is_negative) {
if (retvalue != 0) {
// this is correct even when retvalue == Long.MIN_VALUE
retvalue = -retvalue;
retInteger = Integer.valueOf(retvalue);
}
}
else {
retInteger = Integer.valueOf(retvalue);
}
return retInteger;
}
public double readFloatValue(int len) throws IOException
{
if (len == 0)
{
// special case, return pos zero
return 0.0d;
}
if (len != 8)
{
throw new IonException("Length of float read must be 0 or 8");
}
long dBits = readUIntAsLong(len);
return Double.longBitsToDouble(dBits);
}
/**
* Near clone of {@link SimpleByteBuffer.SimpleByteReader#readDecimal(int)}
* and {@link IonReaderBinaryRawX#readDecimal(int)}
* so keep them in sync!
*/
public Decimal readDecimalValue(int len) throws IOException
{
MathContext mathContext = MathContext.UNLIMITED;
Decimal bd;
// we only write out the '0' value as the nibble 0
if (len == 0) {
bd = Decimal.valueOf(0, mathContext);
}
else {
// otherwise we to it the hard way ....
int startpos = this.position();
int exponent = this.readVarIntAsInt();
int bitlen = len - (this.position() - startpos);
BigInteger value;
int signum;
if (bitlen > 0)
{
byte[] bits = new byte[bitlen];
readAll(this, bits, 0, bitlen);
signum = 1;
if (bits[0] < 0)
{
// value is negative, clear the sign
bits[0] &= 0x7F;
signum = -1;
}
value = new BigInteger(signum, bits);
}
else {
signum = 0;
value = BigInteger.ZERO;
}
// Ion stores exponent, BigDecimal uses the negation "scale"
int scale = -exponent;
if (value.signum() == 0 && signum == -1)
{
assert value.equals(BigInteger.ZERO);
bd = Decimal.negativeZero(scale, mathContext);
}
else
{
bd = Decimal.valueOf(value, scale, mathContext);
}
}
return bd;
}
/**
* @see IonReaderBinaryRawX#readTimestamp
*/
public Timestamp readTimestampValue(int len) throws IOException
{
if (len < 1) {
// nothing to do here - and the timestamp will be NULL
return null;
}
int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0;
Decimal frac = null;
int end = this.position() + len;
// first up is the offset, which requires a special int reader
// to return the -0 as a null Integer
Integer offset = this.readVarIntWithNegativeZero();
// now we'll read the struct values from the input stream
// year is from 0001 to 9999
// or 0x1 to 0x270F or 14 bits - 1 or 2 bytes
year = readVarUIntAsInt();
Precision p = Precision.YEAR; // our lowest significant option
if (position() < end) {
month = readVarUIntAsInt();
p = Precision.MONTH; // our lowest significant option
if (position() < end) {
day = readVarUIntAsInt();
p = Precision.DAY; // our lowest significant option
// now we look for hours and minutes
if (position() < end) {
hour = readVarUIntAsInt();
minute = readVarUIntAsInt();
p = Precision.MINUTE;
if (position() < end) {
second = readVarUIntAsInt();
p = Precision.SECOND;
int remaining = end - position();
if (remaining > 0) {
// now we read in our actual "milliseconds since the epoch"
frac = this.readDecimalValue(remaining);
}
}
}
}
}
// now we let timestamp put it all together
try {
Timestamp val =
Timestamp.createFromUtcFields(p, year, month, day,
hour, minute, second,
frac, offset);
return val;
}
catch (IllegalArgumentException e) {
throw new IonException(e.getMessage() + " at: " + position());
}
}
public String readString(int len) throws IOException
{
char[] cb = new char[len]; // since we know the length in bytes (which must be
// greater than or equal to chars) there's no need
// for a stringbuffer (even surrgated char's are ok)
int ii=0;
int c;
int endPosition = this.position() + len;
while (this.position() < endPosition) {
c = readUnicodeScalar();
if (c < 0) throwUnexpectedEOFException();
//sb.append((char)c);
if (c < 0x10000) {
cb[ii++] = (char)c;
}
else {
cb[ii++] = (char)_Private_IonConstants.makeHighSurrogate(c);
cb[ii++] = (char)_Private_IonConstants.makeLowSurrogate(c);
}
}
if (this.position() < endPosition) throwUnexpectedEOFException();
return new String(cb, 0, ii); // sb.toString();
}
public int readUnicodeScalar() throws IOException {
int c = -1, b;
b = read();
if (b < 0) return -1;
// ascii is all good
if ((b & 0x80) == 0) {
return b;
}
// now we start gluing the multi-byte value together
if ((b & 0xe0) == 0xc0) {
// for values from 0x80 to 0x7FF (all legal)
c = (b & ~0xe0);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
}
else if ((b & 0xf0) == 0xe0) {
// for values from 0x800 to 0xFFFFF (NOT all legal)
c = (b & ~0xf0);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
if (c > 0x00D7FF && c < 0x00E000) {
throw new IonException("illegal surrgate value encountered in input utf-8 stream");
}
}
else if ((b & 0xf8) == 0xf0) {
// for values from 0x010000 to 0x1FFFFF (NOT all legal)
c = (b & ~0xf8);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
b = read();
if ((b & 0xc0) != 0x80) throwUTF8Exception();
c <<= 6;
c |= (b & ~0x80);
if (c > 0x10FFFF) {
throw new IonException("illegal surrgate value encountered in input utf-8 stream");
}
}
else {
throwUTF8Exception();
}
return c;
}
void throwUTF8Exception()
{
throw new IonException("Invalid UTF-8 character encounter in a string at pos " + this.position());
}
void throwUnexpectedEOFException() {
throw new BlockedBuffer.BlockedBufferException("unexpected EOF in value at offset " + this.position());
}
public String readString() throws IOException {
int td = read();
if (_Private_IonConstants.getTypeCode(td) != _Private_IonConstants.tidString) {
throw new IonException("readString helper only works for string(7) not "+((td >> 4 & 0xf)));
}
int len = (td & 0xf);
if (len == _Private_IonConstants.lnIsNullAtom) {
return null;
}
else if (len == _Private_IonConstants.lnIsVarLen) {
len = this.readVarUIntAsInt();
}
return readString(len);
}
/**
* Skips through a NOP pad if there is one. Must be called at the beginning of type description
*
* It's no-op if there is no Nop pad to skip through at the current position
*
* @return true if it skipped through a nop pad
*/
boolean skipThroughNopPad() throws IOException
{
int originalPosition = this._pos;
// check to see if there is a type declaration as it's not valid to annotate nop pads
int c = this.read();
boolean hasTypedecl = _Private_IonConstants.getTypeCode(c) == _Private_IonConstants.tidTypedecl;
this.setPosition(originalPosition);
int typeDesc = this.readActualTypeDesc();
int tid = _Private_IonConstants.getTypeCode(typeDesc);
int len = _Private_IonConstants.getLowNibble(typeDesc);
if(tid == _Private_IonConstants.tidNull && len != _Private_IonConstants.lnIsNull){
if(hasTypedecl) {
throw new IonException("NOP padding is not allowed within annotation wrappers.");
}
int toSkip = this.readLength(_Private_IonConstants.tidNopPad, len);
long skipped = this.skip(toSkip);
if(toSkip > 0 && toSkip != skipped) {
throw new IonException("Nop pad too short declared length: " + toSkip + " pad actual size: " + skipped);
}
return true;
}
// resets the reader if it wasn't a NOP pad
this.setPosition(originalPosition);
return false;
}
}
public static final class Writer
extends BlockedBuffer.BlockedByteOutputStream
{
/**
* creates writable stream (OutputStream) that writes
* to a fresh blocked buffer. The stream is initially
* position at offset 0.
*/
public Writer() { super(); }
/**
* creates writable stream (OutputStream) that writes
* to the supplied byte buffer. The stream is initially
* position at offset 0.
* @param bb blocked buffer to write to
*/
public Writer(BlockedBuffer bb) { super(bb); }
/**
* creates writable stream (OutputStream) that can write
* to the supplied byte buffer. The stream is initially
* position at offset off.
* @param bb blocked buffer to write to
* @param off initial offset to write to
*/
public Writer(BlockedBuffer bb, int off) { super(bb, off); }
Stack<PositionMarker> _pos_stack;
Stack<Integer> _pending_high_surrogate_stack;
int _pending_high_surrogate;
public void pushPosition(Object o)
{
PositionMarker pm = new PositionMarker(this.position(), o);
if (_pos_stack == null) {
_pos_stack = new Stack<PositionMarker>();
_pending_high_surrogate_stack = new Stack<Integer>();
}
_pos_stack.push(pm);
_pending_high_surrogate_stack.push(_pending_high_surrogate);
_pending_high_surrogate = 0;
}
public PositionMarker popPosition()
{
if (_pending_high_surrogate != 0) {
throw new IonException("unmatched high surrogate encountered in input, illegal utf-16 character sequence");
}
PositionMarker pm = _pos_stack.pop();
_pending_high_surrogate = _pending_high_surrogate_stack.pop();
return pm;
}
/*****************************************************************************
*
* These routines work together to write very long values from an input
* reader where we don't know how long the value is going to be in advance.
*
* Basically it tries to write a short value (len <= BB_SHORT_LEN_MAX) and if
* that fails it moves the data around in the buffers and moves buffers worth
* of data at a time.
*
*/
static class lhNode
{
int _hn;
int _lownibble;
boolean _length_follows;
lhNode(int hn
,int lownibble
,boolean length_follows
) {
_hn = hn;
_lownibble = lownibble;
_length_follows = length_follows;
}
}
public void startLongWrite(int hn) throws IOException
{
if (debugValidation) _validate();
pushLongHeader(hn, 0, false);
this.writeCommonHeader(hn, 0);
if (debugValidation) _validate();
}
public void pushLongHeader(int hn
,int lownibble
,boolean length_follows
) {
lhNode n = new lhNode(hn, lownibble, length_follows);
pushPosition(n);
}
/**
* Update the TD/LEN header of a value.
*
* @param hn high nibble value (or type id)
* @param lownibble is the low nibble value.
* If -1, then the low nibble is pulled from the header node pushed
* previously. If the header node had lengthFollows==false then this
* is ignored and the LN is computed from what was actually written.
*/
public void patchLongHeader(int hn, int lownibble) throws IOException
{
int currpos = this.position();
if (debugValidation) _validate();
// get the header description we pushed on the stack a while ago
// pop also checks to make sure we don't have a dangling high
// surrogate pending
PositionMarker pm = this.popPosition();
lhNode n = (lhNode)pm.getUserData();
int totallen = currpos - pm.getPosition();
// fix up the low nibble if we need to
if (lownibble == -1) {
lownibble = n._lownibble;
}
// calculate the length just the value itself
// we don't count the type descriptor in the value len
int writtenValueLen = totallen - _Private_IonConstants.BB_TOKEN_LEN;
// now we can figure out how long the value is going to be
// This is the length of the length (it does NOT
// count the typedesc byte however)
int len_o_len = IonBinary.lenVarUInt(writtenValueLen);
// TODO cleanup this logic. lengthFollows == is struct
if (n._length_follows) {
assert hn == _Private_IonConstants.tidStruct;
if (lownibble == _Private_IonConstants.lnIsOrderedStruct)
{
// leave len_o_len alone
}
else
{
if (writtenValueLen < _Private_IonConstants.lnIsVarLen) {
lownibble = writtenValueLen;
len_o_len = 0;
}
else {
lownibble = _Private_IonConstants.lnIsVarLen;
}
assert lownibble != _Private_IonConstants.lnIsOrderedStruct;
}
}
else {
if (writtenValueLen < _Private_IonConstants.lnIsVarLen) {
lownibble = writtenValueLen;
len_o_len = 0;
}
else {
lownibble = _Private_IonConstants.lnIsVarLen;
}
}
// first we go back to the beginning
this.setPosition(pm.getPosition());
// figure out if we need to move the trailing data to make
// room for the variable length length
int needed = len_o_len;
if (needed > 0) {
// insert does the heavy lifting of making room for us
// at the current position for "needed" additional bytes
insert(needed);
}
// so, we have room (or already had enough) now we can write
// replacement type descriptor and the length and the reset the pos
this.writeByte(_Private_IonConstants.makeTypeDescriptor(hn, lownibble));
if (len_o_len > 0) {
this.writeVarUIntValue(writtenValueLen, true);
}
if (needed < 0) {
// in the unlikely event we wrote more than we needed, now
// is the time to remove it. (which does happen from time to time)
this.remove(-needed);
}
// return the cursor to it's correct location,
// taking into account the added length data
this.setPosition(currpos + needed);
if (debugValidation) _validate();
}
// TODO - this may have problems with unicode utf16/utf8 conversions
// note that this reads characters that have already had the escape
// sequence processed (so we don't want to do that a second time)
// the chars here were filled by the IonTokenReader which already
// de-escaped the escaped sequences from the input, so all chars are "real"
public void appendToLongValue(CharSequence chars, boolean onlyByteSizedCharacters) throws IOException
{
if (debugValidation) _validate();
int len = chars.length(); // TODO is this ever >0 for clob?
for (int ii = 0; ii < len; ii++)
{
int c = chars.charAt(ii);
if (onlyByteSizedCharacters) {
if (c > 255) {
throw new IonException("escaped character value too large in clob (0 to 255 only)");
}
write((byte)(0xff & c));
}
else {
if (_pending_high_surrogate != 0) {
if ((c & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) {
throw new IonException("unmatched high surrogate character encountered, invalid utf-16");
}
c = _Private_IonConstants.makeUnicodeScalar(_pending_high_surrogate, c);
_pending_high_surrogate = 0;
}
else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.high_surrogate_value) {
ii++;
if (ii >= len) {
// a trailing high surrogate, we just remember it for later
// and (hopefully, and usually, the low surrogate will be
// appended shortly
_pending_high_surrogate = c;
break;
}
int c2 = chars.charAt(ii);
if ((c2 & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) {
throw new IonException("unmatched high surrogate character encountered, invalid utf-16");
}
c = _Private_IonConstants.makeUnicodeScalar(c, c2);
}
else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.low_surrogate_value) {
throw new IonException("unmatched low surrogate character encountered, invalid utf-16");
}
writeUnicodeScalarAsUTF8(c);
}
}
if (debugValidation) _validate();
}
// helper for appendToLongValue below - this never cares about surrogates
// as it only consumes terminators which are ascii
final boolean isLongTerminator(int terminator, PushbackReader r) throws IOException {
int c;
// look for terminator 2 - if it's not there put back what we saw
c = r.read();
if (c != terminator) {
r.unread(c);
return false;
}
// look for terminator 3 - otherwise put back what we saw and the
// preceeding terminator we found above
c = r.read();
if (c != terminator) {
r.unread(c);
r.unread(terminator);
return false;
}
// we found 3 in a row - now that's a long terminator
return true;
}
/**
* Reads the remainder of a quoted string/symbol into this buffer.
* The closing quotes are consumed from the reader.
* <p>
* XXX WARNING XXX
* Almost identical logic is found in
* {@link IonTokenReader#finishScanString(boolean)}
*
* @param terminator the closing quote character.
* @param longstring
* @param r
* @throws IOException
*/
public void appendToLongValue(int terminator
,boolean longstring
,boolean onlyByteSizedCharacters
,boolean decodeEscapeSequences
,PushbackReader r
)
throws IOException, UnexpectedEofException
{
int c;
if (debugValidation) {
if (terminator == -1 && longstring) {
throw new IllegalStateException("longstrings have to have a terminator, no eof termination");
}
_validate();
}
assert(terminator != '\\');
for (;;) {
c = r.read(); // we put off the surrogate logic as long as possible (so not here)
if (c == terminator) {
if (!longstring || isLongTerminator(terminator, r)) {
// if it's not a long string one quote is enough otherwise look ahead
break;
}
}
else if (c == -1) {
throw new UnexpectedEofException();
}
else if (c == '\n' || c == '\r') {
// here we'll handle embedded new line detection and escaped characters
if ((terminator != -1) && !longstring) {
throw new IonException("unexpected line terminator encountered in quoted string");
}
}
else if (decodeEscapeSequences && c == '\\') {
// if this is an escape sequence we need to process it now
// since we allow a surrogate to be encoded using \ u (or \ U)
// encoding
c = IonTokenReader.readEscapedCharacter(r, onlyByteSizedCharacters);
if (c == IonTokenReader.EMPTY_ESCAPE_SEQUENCE) {
continue;
}
}
if (onlyByteSizedCharacters) {
assert(_pending_high_surrogate == 0); // if it's byte sized only, then we shouldn't have a dangling surrogate
if ((c & (~0xff)) != 0) {
throw new IonException("escaped character value too large in clob (0 to 255 only)");
}
write((byte)(0xff & c));
}
else {
// for larger characters we have to glue together surrogates, regardless
// of how they were encoded. If we have a high surrogate and go to peek
// for the low surrogate and hit the end of a segment of a long string
// (triple quoted multi-line string) we leave a dangling high surrogate
// that will get picked up on the next call into this routine when the
// next segment of the long string is processed
if (_pending_high_surrogate != 0) {
if ((c & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) {
String message =
"Text contains unmatched UTF-16 high surrogate " +
IonTextUtils.printCodePointAsString(_pending_high_surrogate);
throw new IonException(message);
}
c = _Private_IonConstants.makeUnicodeScalar(_pending_high_surrogate, c);
_pending_high_surrogate = 0;
}
else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.high_surrogate_value) {
int c2 = r.read();
if (c2 == terminator) {
if (longstring && isLongTerminator(terminator, r)) {
// if it's a long string termination we'll hang onto the current c as the pending surrogate
_pending_high_surrogate = c;
c = terminator;
break;
}
// otherwise this is an error
String message =
"Text contains unmatched UTF-16 high surrogate " +
IonTextUtils.printCodePointAsString(c);
throw new IonException(message);
}
else if (c2 == -1) {
// eof is also an error - really two errors
throw new UnexpectedEofException();
}
//here we convert escape sequences into characters and continue until
//we encounter a non-newline escape (typically immediately)
while (decodeEscapeSequences && c2 == '\\') {
c2 = IonTokenReader.readEscapedCharacter(r, onlyByteSizedCharacters);
if (c2 != IonTokenReader.EMPTY_ESCAPE_SEQUENCE) break;
c2 = r.read();
if (c2 == terminator) {
if (longstring && isLongTerminator(terminator, r)) {
// if it's a long string termination we'll hang onto the current c as the pending surrogate
_pending_high_surrogate = c;
c = c2; // we'll be checking this below
break;
}
// otherwise this is an error
String message =
"Text contains unmatched UTF-16 high surrogate " +
IonTextUtils.printCodePointAsString(c);
throw new IonException(message);
}
else if (c2 == -1) {
// eof is also an error - really two errors
throw new UnexpectedEofException();
}
}
// check to see how we broke our of the while loop above, we may be "done"
if (_pending_high_surrogate != 0) {
break;
}
if ((c2 & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) {
String message =
"Text contains unmatched UTF-16 high surrogate " +
IonTextUtils.printCodePointAsString(c);
throw new IonException(message);
}
c = _Private_IonConstants.makeUnicodeScalar(c, c2);
}
else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.low_surrogate_value) {
String message =
"Text contains unmatched UTF-16 low surrogate " +
IonTextUtils.printCodePointAsString(c);
throw new IonException(message);
}
writeUnicodeScalarAsUTF8(c);
}
}
if (c != terminator) {
// TODO determine if this can really happen.
throw new UnexpectedEofException();
}
if (debugValidation) _validate();
return;
}
/*
* this is a set of routines that put data into an Ion buffer
* using the various type encodeing techniques
*
* these are the "high" level write routines (and read as well,
* in the fullness of time)
*
*
* methods with names of the form:
* void write<type>Value( value ... )
* only write the value portion of the data
*
* methods with name like:
* int(len) write<type>WithLength( ... value ... )
* write the trailing length and the value they return the
* length written in case it is less than BB_SHORT_LEN_MAX (13)
* so that the caller can backpatch the type descriptor if they
* need to
*
* methods with names of the form:
* void write<type>( nibble, ..., value )
* write the type descriptor the length if needed and the value
*
* methods with a name like:
* int <type>ValueLength( ... )
* return the number of bytes that will be needed to write
* the value out
*
*/
/**
* Buffer for decoding (un)signed VarInt's and Int's
*/
private byte[] numberBuffer = new byte[10];
public int writeVarUIntValue(long value, boolean force_zero_write) throws IOException
{
assert value >= 0;
int len = 0;
if (value == 0) {
if (force_zero_write) {
write((byte)0x80);
len = 1;
}
} else {
int i = numberBuffer.length;
// write every 7 bits of the value
while (value > 0) {
numberBuffer[--i] = (byte)(value & 0x7f);
value = value >>> 7;
}
// set the end bit
numberBuffer[numberBuffer.length - 1] |= 0x80;
len = numberBuffer.length - i;
write(numberBuffer, i, len);
}
return len;
}
/**
* Writes a uint field of maximum length 8.
* Note that this will write from the lowest to highest
* order bits in the long value given.
*/
public int writeUIntValue(long value) throws IOException
{
int i = numberBuffer.length;
// even if value is Long.MIN_VALUE we will still serialize it correctly :)
while (value != 0) {
numberBuffer[--i] = (byte)(value & 0xff);
value = value >>> 8;
}
int len = numberBuffer.length - i;
write(numberBuffer, i, len);
return len;
}
/**
* Writes a uint field at given length
* Note that this will write from the lowest to highest
* order bits in the long value given.
*/
public int writeUIntValue(long value, int len) throws IOException
{
int i = numberBuffer.length;
for (int j = 0; j < len; ++j) {
numberBuffer[--i] = (byte)(value & 0xff);
value = value >>> 8;
}
write(numberBuffer, i, len);
return len;
}
/**
* Writes a uint field of arbitary length. It does
* check the value to see if a simpler routine can
* handle the work;
* Note that this will write from the lowest to highest
* order bits in the long value given.
*/
public int writeUIntValue(BigInteger value, int len) throws IOException
{
int returnlen = 0;
int signum = value.signum();
if (signum == 0) {
// Zero has no bytes of data at all! Nothing to write.
}
else if (signum < 0) {
throw new IllegalArgumentException("value must be greater than or equal to 0");
}
else if (value.compareTo(MAX_LONG_VALUE) == -1) {
long lvalue = value.longValue();
returnlen = writeUIntValue(lvalue, len);
}
else {
assert(signum > 0);
byte[] bits = value.toByteArray();
// BigInteger will pad this with a null byte sometimes
// for negative numbers...let's skip past any leading null bytes
int offset = 0;
while (offset < bits.length && bits[offset] == 0) {
offset++;
}
int bitlen = bits.length - offset;
this.write(bits, offset, bitlen);
returnlen += bitlen;
}
assert(returnlen == len);
return len;
}
public int writeVarIntValue(long value, boolean force_zero_write) throws IOException
{
int len = 0;
if (value == 0) {
if (force_zero_write) {
write((byte)0x80);
len = 1;
}
} else {
int i = numberBuffer.length;
boolean negative = value < 0;
if (negative) {
value = -value;
}
// write every 7 bits of the value
while (value > 0) {
numberBuffer[--i] = (byte)(value & 0x7f);
value = value >>> 7;
}
// set the end bit
numberBuffer[numberBuffer.length - 1] |= 0x80;
// increase the length of VarInt if the sign bit is 'occupied'
// by the value to properly flag it
if ((numberBuffer[i] & 0x40) == 0x40) {
numberBuffer[--i] = 0x00;
}
// set the sign bit
if (negative) {
// add the sign bit to MSB
numberBuffer[i] |= 0x40;
}
len = numberBuffer.length - i;
write(numberBuffer, i, len);
}
return len;
}
public int writeIntValue(long value) throws IOException {
int i = numberBuffer.length;
boolean negative = value < 0;
if (negative) {
value = -value;
}
while (value > 0) {
numberBuffer[--i] = (byte)(value & 0xff);
value = value >>> 8;
}
// increase the length of Int if the sign bit is 'occupied'
// by the value to properly flag it
if ((numberBuffer[i] & 0x80) == 0x40) {
numberBuffer[--i] = 0x00;
}
// set the sign bit
if (negative) {
// add the sign bit to MSB
numberBuffer[i] |= 0x80;
}
int len = numberBuffer.length - i;
write(numberBuffer, i, len);
return len;
}
public int writeFloatValue(double d) throws IOException
{
if (Double.valueOf(d).equals(DOUBLE_POS_ZERO))
{
// pos zero special case
return 0;
}
// TODO write "custom" serialization or verify that
// the java routine is doing the right thing
long dBits = Double.doubleToRawLongBits(d);
return writeUIntValue(dBits, _ib_FLOAT64_LEN);
}
byte[] singleCodepointUtf8Buffer = new byte[4];
public int writeUnicodeScalarAsUTF8(int c) throws IOException
{
int len;
if (c < 0x80) {
len = 1;
this.start_write();
_write((byte)(c & 0xff));
this.end_write();
} else {
len = _writeUnicodeScalarToByteBuffer(c, singleCodepointUtf8Buffer, 0);
this.write(singleCodepointUtf8Buffer, 0, len);
}
return len;
}
// this will write at least 2 byte unicode scalar to buffer
private final int _writeUnicodeScalarToByteBuffer(int c, byte[] buffer, int offset) throws IOException
{
int len = -1;
assert offset + 4 <= buffer.length;
// first the quick, easy and common case - ascii
if (c < 0x800) {
// 2 bytes characters from 0x000080 to 0x0007FF
buffer[offset] = (byte)( 0xff & (0xC0 | (c >> 6)) );
buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) );
len = 2;
}
else if (c < 0x10000) {
// 3 byte characters from 0x800 to 0xFFFF
// but only 0x800...0xD7FF and 0xE000...0xFFFF are valid
if (c > 0xD7FF && c < 0xE000) {
this.throwUTF8Exception();
}
buffer[offset] = (byte)( 0xff & (0xE0 | (c >> 12)) );
buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 6) & 0x3F)) );
buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) );
len = 3;
}
else if (c <= 0x10FFFF) {
// 4 byte characters 0x010000 to 0x10FFFF
// these are are valid
buffer[offset] = (byte)( 0xff & (0xF0 | (c >> 18)) );
buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 12) & 0x3F)) );
buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 6) & 0x3F)) );
buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) );
len = 4;
}
else {
this.throwUTF8Exception();
}
return len;
}
/******************************
*
* These are the "write typed value with header" routines
* they all are of the form:
*
* void write<type>(nibble, ...)
*/
public int writeByte(HighNibble hn, int len) throws IOException
{
if (len < 0) {
throw new IonException("negative token length encountered");
}
if (len > 13) len = 14; // TODO remove magic numbers
int t = _Private_IonConstants.makeTypeDescriptor( hn.value(), len );
write(t);
return 1;
}
/**
* Write one byte.
*
* @param b the byte to write.
* @return the number of bytes written (always 1).
* @throws IOException
*/
public int writeByte(byte b) throws IOException
{
write(b);
return 1;
}
/**
* Write one byte.
*
* @param b integer containing the byte to write; only the 8 low-order
* bits are written.
*
* @return the number of bytes written (always 1).
* @throws IOException
*/
public int writeByte(int b) throws IOException
{
write(b);
return 1;
}
public int writeAnnotations(SymbolToken[] annotations,
SymbolTable symbolTable) throws IOException
{
int startPosition = this.position();
int annotationLen = 0;
for (int ii=0; ii<annotations.length; ii++) {
int sid = annotations[ii].getSid();
assert sid != SymbolTable.UNKNOWN_SYMBOL_ID;
annotationLen += IonBinary.lenVarUInt(sid);
}
// write the len of the list
this.writeVarUIntValue(annotationLen, true);
// write the symbol id's
for (int ii=0; ii<annotations.length; ii++) {
int sid = annotations[ii].getSid();
this.writeVarUIntValue(sid, true);
}
return this.position() - startPosition;
}
public int writeAnnotations(ArrayList<Integer> annotations)
throws IOException
{
int startPosition = this.position();
int annotationLen = 0;
for (Integer ii : annotations) {
annotationLen += IonBinary.lenVarUInt(ii.intValue());
}
// write the len of the list
this.writeVarUIntValue(annotationLen, true);
// write the symbol id's
for (Integer ii : annotations) {
this.writeVarUIntValue(ii.intValue(), true);
}
return this.position() - startPosition;
}
public void writeStubStructHeader(int hn, int ln)
throws IOException
{
// write the hn and ln as the typedesc, we'll patch it later.
writeByte(_Private_IonConstants.makeTypeDescriptor(hn, ln));
}
/**
* Write typedesc and length for the common case.
*/
public int writeCommonHeader(int hn, int len)
throws IOException
{
int returnlen = 0;
// write then len in low nibble
if (len < _Private_IonConstants.lnIsVarLen) {
returnlen += writeByte(_Private_IonConstants.makeTypeDescriptor(hn, len));
}
else {
returnlen += writeByte(_Private_IonConstants.makeTypeDescriptor(hn, _Private_IonConstants.lnIsVarLen));
returnlen += writeVarUIntValue(len, false);
}
return returnlen;
}
/***************************************
*
* Here are the write type value with type descriptor and everything
* family of methods, these depend on the others to do much of the work
*
*/
/**
* @param sid must be valid id (>=1)
*/
public int writeSymbolWithTD(int sid) // was: (String s, SymbolTable symtab)
throws IOException
{
// was: int sid = symtab.addSymbol(s);
assert sid > 0;
int vlen = lenUInt(sid);
int len = this.writeCommonHeader(
_Private_IonConstants.tidSymbol
,vlen
);
len += this.writeUIntValue(sid, vlen);
return len;
}
/**
* Writes the full string header + data.
*/
public int writeStringWithTD(String s) throws IOException
{
// first we have to see how long this will be in the output
/// buffer - part of the cost of length prefixed values
int len = IonBinary.lenIonString(s);
if (len < 0) this.throwUTF8Exception();
// first we write the type desc and length
len += this.writeCommonHeader(_Private_IonConstants.tidString, len);
// now we write just the value out
writeStringData(s);
//for (int ii=0; ii<s.length(); ii++) {
// char c = s.charAt(ii);
// this.writeCharValue(c);
//}
return len;
}
static final int stringBufferLen = 128;
byte[] stringBuffer = new byte[stringBufferLen];
public int writeStringData(String s) throws IOException
{
int len = 0;
int bufPos = 0;
for (int ii=0; ii<s.length(); ii++) {
int c = s.charAt(ii);
if (bufPos > stringBufferLen - 4) { // 4 is the max UTF-8 encoding size
this.write(stringBuffer, 0, bufPos);
bufPos = 0;
}
// at this point stringBuffer contains enough space for UTF-8 encoded code point
if (c < 128) {
// don't even both to call the "utf8" converter for ascii
stringBuffer[bufPos++] = (byte)c;
len++;
continue;
}
// multi-byte utf8
if (c >= 0xD800 && c <= 0xDFFF) {
if (_Private_IonConstants.isHighSurrogate(c)) {
// houston we have a high surrogate (let's hope it has a partner
if (++ii >= s.length()) {
throw new IllegalArgumentException("invalid string, unpaired high surrogate character");
}
int c2 = s.charAt(ii);
if (!_Private_IonConstants.isLowSurrogate(c2)) {
throw new IllegalArgumentException("invalid string, unpaired high surrogate character");
}
c = _Private_IonConstants.makeUnicodeScalar(c, c2);
}
else if (_Private_IonConstants.isLowSurrogate(c)) {
// it's a loner low surrogate - that's an error
throw new IllegalArgumentException("invalid string, unpaired low surrogate character");
}
// from 0xE000 up the _writeUnicodeScalar will check for us
}
int utf8len = this._writeUnicodeScalarToByteBuffer(c, stringBuffer, bufPos);
bufPos += utf8len;
len += utf8len;
}
if (bufPos > 0) {
this.write(stringBuffer, 0, bufPos);
}
return len;
}
public int writeNullWithTD(HighNibble hn) throws IOException
{
writeByte(hn, _Private_IonConstants.lnIsNullAtom);
return 1;
}
public int writeTimestampWithTD(Timestamp di)
throws IOException
{
int returnlen;
if (di == null) {
returnlen = this.writeCommonHeader(
_Private_IonConstants.tidTimestamp
,_Private_IonConstants.lnIsNullAtom);
}
else {
int vlen = IonBinary.lenIonTimestamp(di);
returnlen = this.writeCommonHeader(
_Private_IonConstants.tidTimestamp
,vlen);
int wroteLen = writeTimestamp(di);
assert wroteLen == vlen;
returnlen += wroteLen;
}
return returnlen;
}
public int writeTimestamp(Timestamp di)
throws IOException
{
if (di == null) return 0;
int returnlen = 0;
Precision precision = di.getPrecision();
Integer offset = di.getLocalOffset();
if (offset == null) {
// TODO don't use magic numbers!
this.write((byte)(0xff & (0x80 | 0x40))); // negative 0 (no timezone)
returnlen ++;
}
else {
returnlen += this.writeVarIntValue(offset.intValue(), true);
}
// now the date - year, month, day as VarUInts
// if we have a non-null value we have at least the date
if (precision.includes(Precision.YEAR)) {
returnlen += this.writeVarUIntValue(di.getZYear(), true);
}
if (precision.includes(Precision.MONTH)) {
returnlen += this.writeVarUIntValue(di.getZMonth(), true);
}
if (precision.includes(Precision.DAY)) {
returnlen += this.writeVarUIntValue(di.getZDay(), true);
}
// now the time portion
if (precision.includes(Precision.MINUTE)) {
returnlen += this.writeVarUIntValue(di.getZHour(), true);
returnlen += this.writeVarUIntValue(di.getZMinute(), true);
}
if (precision.includes(Precision.SECOND)) {
returnlen += this.writeVarUIntValue(di.getZSecond(), true);
// and, finally, any fractional component that is known
BigDecimal fraction = di.getZFractionalSecond();
if (fraction != null) {
assert !fraction.equals(BigDecimal.ZERO);
returnlen += this.writeDecimalContent(fraction);
}
}
return returnlen;
}
public int writeDecimalWithTD(BigDecimal bd) throws IOException
{
int returnlen;
// we only write out the '0' value as the nibble 0
if (bd == null) {
returnlen =
this.writeByte(NULL_DECIMAL_TYPEDESC);
}
else if (isNibbleZero(bd)) {
returnlen =
this.writeByte(ZERO_DECIMAL_TYPEDESC);
}
else {
// otherwise we to it the hard way ....
int len = IonBinary.lenIonDecimal(bd);
if (len < _Private_IonConstants.lnIsVarLen) {
returnlen = this.writeByte(
_Private_IonConstants.makeTypeDescriptor(
_Private_IonConstants.tidDecimal
, len
)
);
}
else {
returnlen = this.writeByte(
_Private_IonConstants.makeTypeDescriptor(
_Private_IonConstants.tidDecimal
, _Private_IonConstants.lnIsVarLen
)
);
this.writeVarIntValue(len, false);
}
int wroteDecimalLen = writeDecimalContent(bd);
assert wroteDecimalLen == len;
returnlen += wroteDecimalLen;
}
return returnlen;
}
private static final byte[] negativeZeroBitArray = new byte[] { (byte)0x80 };
/** Zero-length byte array. */
private static final byte[] positiveZeroBitArray = EMPTY_BYTE_ARRAY;
/**
* @see com.amazon.ion.impl.lite.ReverseBinaryEncoder#writeIonDecimalContent
*/
public int writeDecimalContent(BigDecimal bd)
throws IOException
{
// check for null and 0. which are encoded in the nibble itself.
if (bd == null) return 0;
if (isNibbleZero(bd)) return 0;
// Ion stores exponent, BigDecimal uses the negation "scale"
int exponent = -bd.scale();
// The exponent isn't optional (except for the 0d0 case above).
int returnlen = writeVarIntValue(exponent,
/* force_zero_write*/ true);
BigInteger mantissa = bd.unscaledValue();
byte[] mantissaBits;
switch (mantissa.signum()) {
case 0:
if (Decimal.isNegativeZero(bd)) {
mantissaBits = negativeZeroBitArray;
}
else {
mantissaBits = positiveZeroBitArray;
}
break;
case -1:
// Obtain the unsigned value of the BigInteger
// We cannot use the twos complement representation of a
// negative BigInteger as this is different from the encoding
// of basic field Int.
mantissaBits = mantissa.negate().toByteArray();
// Set the sign on the highest order bit of the first octet
mantissaBits[0] |= 0x80;
break;
case 1:
mantissaBits = mantissa.toByteArray();
break;
default:
throw new IllegalStateException("mantissa signum out of range");
}
this.write(mantissaBits, 0, mantissaBits.length);
returnlen += mantissaBits.length;
return returnlen;
}
void throwUTF8Exception()
{
throwException("Invalid UTF-8 character encounter in a string at pos " + this.position());
}
void throwException(String s)
{
throw new BlockedBuffer.BlockedBufferException(s);
}
}
public static class PositionMarker
{
int _pos;
Object _userData;
public PositionMarker() {}
public PositionMarker(int pos, Object o) {
_pos = pos;
_userData = o;
}
public int getPosition() { return _pos; }
public Object getUserData() { return _userData; }
public void setPosition(int pos) {
_pos = pos;
}
public void setUserData(Object o) {
_userData = o;
}
}
}