private static void testWriteSequence(int[] sentence, int beginId, int endId, int maxGap)
throws IOException {
int partitionId = 1;
SimpleGapEncoder encoder = new SimpleGapEncoder(maxGap, 3, new BytesWritable());
encoder.setPartitionId(partitionId);
// print sentence
System.out.println();
System.out.println(
"Sequence: "
+ Arrays.toString(sentence)
+ " / id range: "
+ beginId
+ "-"
+ endId
+ " / max gap: "
+ maxGap);
encoder.encode(sentence, 0, sentence.length - 1, beginId, endId, false, true);
ByteArrayInputStream bin =
new ByteArrayInputStream(encoder.target().getBytes(), 0, encoder.target().getLength());
DataInputStream in = new DataInputStream(bin);
System.out.print("Encoded sequence has " + in.available() + " byte(s): ");
while (in.available() > 0) {
long v = WritableUtils.readVInt(in);
System.out.print(v + " ");
}
System.out.println();
}
@Override
public void readFields(DataInput input) throws IOException {
int encodedByteLengthAndBool = WritableUtils.readVInt(input);
int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
this.byteValue = new byte[byteLength];
input.readFully(byteValue, 0, byteLength);
int sortOrderAndDeterminism = WritableUtils.readVInt(input);
if (sortOrderAndDeterminism <= 2) {
// client is on an older version
this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
;
} else {
int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
this.determinism = Determinism.values()[determinismOrdinal];
int sortOrderValue =
sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
}
int typeOrdinal = WritableUtils.readVInt(input);
if (typeOrdinal < 0) {
this.type = null;
} else {
this.type = PDataType.values()[typeOrdinal];
}
if (this.byteValue.length == 0) {
this.value = null;
} else {
this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
}
}
private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {
ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
int prevDocID = 0;
try {
while (true) {
int left = WritableUtils.readVInt(dataIn);
int right = WritableUtils.readVInt(dataIn);
if (right != 0) {
posting.add(new PairOfInts(left + prevDocID, right));
prevDocID += left;
}
}
} catch (EOFException e) {
} catch (IOException e) {
}
try {
dataIn.close();
} catch (IOException e) {
}
return posting;
}
public static TupleProjector deserializeProjectorFromScan(Scan scan) {
byte[] proj = scan.getAttribute(SCAN_PROJECTOR);
if (proj == null) {
return null;
}
ByteArrayInputStream stream = new ByteArrayInputStream(proj);
try {
DataInputStream input = new DataInputStream(stream);
KeyValueSchema schema = new KeyValueSchema();
schema.readFields(input);
int count = WritableUtils.readVInt(input);
Expression[] expressions = new Expression[count];
for (int i = 0; i < count; i++) {
int ordinal = WritableUtils.readVInt(input);
expressions[i] = ExpressionType.values()[ordinal].newInstance();
expressions[i].readFields(input);
}
return new TupleProjector(schema, expressions);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@Override
public void readFields(DataInput input) throws IOException {
byte[] tableNameBytes = Bytes.readByteArray(input);
PName tableName = new PNameImpl(tableNameBytes);
PTableType tableType = PTableType.values()[WritableUtils.readVInt(input)];
long sequenceNumber = WritableUtils.readVLong(input);
long timeStamp = input.readLong();
byte[] pkNameBytes = Bytes.readByteArray(input);
String pkName = pkNameBytes.length == 0 ? null : Bytes.toString(pkNameBytes);
int nColumns = WritableUtils.readVInt(input);
List<PColumn> columns = Lists.newArrayListWithExpectedSize(nColumns);
for (int i = 0; i < nColumns; i++) {
PColumn column = new PColumnImpl();
column.readFields(input);
columns.add(column);
}
Map<String, byte[][]> guidePosts = new HashMap<String, byte[][]>();
int size = WritableUtils.readVInt(input);
for (int i = 0; i < size; i++) {
String key = WritableUtils.readString(input);
int valueSize = WritableUtils.readVInt(input);
byte[][] value = new byte[valueSize][];
for (int j = 0; j < valueSize; j++) {
value[j] = Bytes.readByteArray(input);
}
guidePosts.put(key, value);
}
PTableStats stats = new PTableStatsImpl(guidePosts);
init(tableName, tableType, timeStamp, sequenceNumber, pkName, columns, stats);
}
@Override
public void readFields(DataInput input) throws IOException {
super.readFields(input);
toType = PDataType.values()[WritableUtils.readVInt(input)];
toMod = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
int byteSize = WritableUtils.readVInt(input);
this.byteSize = byteSize == -1 ? null : byteSize;
}
/** Decode Writable postings and compare by id */
public int compare(byte[] arg0, int arg1, int arg2, byte[] arg3, int arg4, int arg5) {
try {
DataInputStream di1 = new DataInputStream(new ByteArrayInputStream(arg0, arg1, arg2));
DataInputStream di2 = new DataInputStream(new ByteArrayInputStream(arg3, arg4, arg5));
return WritableUtils.readVInt(di1) - WritableUtils.readVInt(di2);
} catch (Exception e) {
return 0;
}
}
@Override
public void readFields(DataInput in) throws IOException {
int minNullable = WritableUtils.readVInt(in);
int nFields = WritableUtils.readVInt(in);
List<Field> fields = Lists.newArrayListWithExpectedSize(nFields);
for (int i = 0; i < nFields; i++) {
Field field = new Field();
field.readFields(in);
fields.add(field);
}
init(minNullable, fields);
}
@Override
public void readFields(DataInput in) throws IOException {
id = WritableUtils.readVInt(in);
expirationDate = WritableUtils.readVLong(in);
int keyLength = WritableUtils.readVInt(in);
if (keyLength < 0) {
secret = null;
} else {
byte[] keyBytes = new byte[keyLength];
in.readFully(keyBytes);
secret = AuthenticationTokenSecretManager.createSecretKey(keyBytes);
}
}
@Override
public void readFields(DataInput input) throws IOException {
// read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
// save space
int typeAndFlag = WritableUtils.readVInt(input);
isNullable = (typeAndFlag & 0x01) != 0;
if ((typeAndFlag & 0x02) != 0) {
scale = WritableUtils.readVInt(input);
}
if ((typeAndFlag & 0x04) != 0) {
maxLength = WritableUtils.readVInt(input);
}
type = PDataType.values()[typeAndFlag >>> 3];
sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
}
/** Serializes this object. */
public void readFields(DataInput in) throws IOException {
int length = WritableUtils.readVInt(in);
byte[] bytes = new byte[length];
in.readFully(bytes, 0, length);
WikipediaPage.readPage(this, new String(bytes, "UTF-8"));
language = in.readUTF();
}
@Override
public void readFields(DataInput in) throws IOException {
clear();
final int termCount = WritableUtils.readVInt(in);
for (int i = 0; i < termCount; i++) {
final String term = Text.readString(in);
final int freq = WritableUtils.readVInt(in);
final int bf = WritableUtils.readVInt(in);
insert(freq, term);
if (bf == 0) continue;
final int[] blocks = new int[bf];
blocks[0] = WritableUtils.readVInt(in) - 1;
for (int j = 1; j < bf; j++) blocks[j] = WritableUtils.readVInt(in) - blocks[j - 1];
term_blocks.put(term, new TIntHashSet(blocks));
}
}
private static void rawValueToTextBytes(
DataOutputBuffer dataBuffer, DataInputBuffer inputBuffer, TextBytes textOut)
throws IOException {
inputBuffer.reset(dataBuffer.getData(), dataBuffer.getLength());
int newLength = WritableUtils.readVInt(inputBuffer);
textOut.set(inputBuffer.getData(), inputBuffer.getPosition(), newLength);
}
private void loadDfs(Path dfStatsPath) throws IOException {
if (dfs != null) return;
FSDataInputStream dfStatsInput = fileSys.open(dfStatsPath);
int l = dfStatsInput.readInt();
if (l != prefixSet.size()) {
throw new RuntimeException("df length mismatch: " + l + "\t" + prefixSet.size());
}
dfs = new int[l];
for (int i = 0; i < l; i++) dfs[i] = WritableUtils.readVInt(dfStatsInput);
dfStatsInput.close();
}
@Override
public void readFields(DataInput input) throws IOException {
// Encode isNullable in sign bit of type ordinal (offset by 1, since ordinal could be 0)
int typeOrdinal = WritableUtils.readVInt(input);
if (typeOrdinal < 0) {
typeOrdinal *= -1;
this.isNullable = true;
}
this.type = PDataType.values()[typeOrdinal - 1];
this.count = WritableUtils.readVInt(input);
if (this.count < 0) {
this.count *= -1;
this.sortOrder = SortOrder.DESC;
} else {
this.sortOrder = SortOrder.ASC;
}
if (this.type.isFixedWidth() && this.type.getByteSize() == null) {
this.byteSize = WritableUtils.readVInt(input);
}
}
@Override
public void readFields(DataInput input) throws IOException {
byte[] columnNameBytes = Bytes.readByteArray(input);
PName columnName = PNameFactory.newName(columnNameBytes);
byte[] familyNameBytes = Bytes.readByteArray(input);
PName familyName = familyNameBytes.length == 0 ? null : PNameFactory.newName(familyNameBytes);
// TODO: optimize the reading/writing of this b/c it could likely all fit in a single byte or
// two
PDataType dataType = PDataType.values()[WritableUtils.readVInt(input)];
int maxLength = WritableUtils.readVInt(input);
int scale = WritableUtils.readVInt(input);
boolean nullable = input.readBoolean();
int position = WritableUtils.readVInt(input);
SortOrder sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
int arrSize = WritableUtils.readVInt(input);
init(
columnName,
familyName,
dataType,
maxLength == NO_MAXLENGTH ? null : maxLength,
scale == NO_SCALE ? null : scale,
nullable,
position,
sortOrder,
arrSize == -1 ? null : arrSize);
}
private static List<Expression> deserializeExpressions(byte[] b) {
ByteArrayInputStream stream = new ByteArrayInputStream(b);
try {
DataInputStream input = new DataInputStream(stream);
int size = WritableUtils.readVInt(input);
List<Expression> selectExpressions = Lists.newArrayListWithExpectedSize(size);
for (int i = 0; i < size; i++) {
ExpressionType type = ExpressionType.values()[WritableUtils.readVInt(input)];
Expression selectExpression = type.newInstance();
selectExpression.readFields(input);
selectExpressions.add(selectExpression);
}
return selectExpressions;
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
try {
final int ret;
di.reset(b1, s1, l1);
final int x1 = WritableUtils.readVInt(di);
di.reset(b2, s2, l2);
final int x2 = WritableUtils.readVInt(di);
final int t1 = b1[s1 + x1];
final int t2 = b2[s2 + x2];
if (t1 == GridmixKey.REDUCE_SPEC || t2 == GridmixKey.REDUCE_SPEC) {
ret = t1 - t2;
} else {
assert t1 == GridmixKey.DATA;
assert t2 == GridmixKey.DATA;
ret = WritableComparator.compareBytes(b1, s1, x1, b2, s2, x2);
}
di.reset(reset, 0, 0);
return ret;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static Map<String, List<FaunusEdge>> readFields(
final DataInput in, final Direction idToRead, final long otherId) throws IOException {
final Map<String, List<FaunusEdge>> edges = new HashMap<String, List<FaunusEdge>>();
int edgeTypes = in.readShort();
for (int i = 0; i < edgeTypes; i++) {
final String label = in.readUTF();
final int size = WritableUtils.readVInt(in);
final List<FaunusEdge> temp = new ArrayList<FaunusEdge>(size);
for (int j = 0; j < size; j++) {
final FaunusEdge edge = new FaunusEdge();
edge.readFieldsCompressed(in, idToRead);
edge.setLabel(label);
if (idToRead.equals(Direction.OUT)) edge.inVertex = otherId;
else edge.outVertex = otherId;
temp.add(edge);
}
edges.put(label, temp);
}
return edges;
}
@Override
public void readFields(DataInput in) throws IOException {
int bufferSize = in.readInt();
byte[] bytes = new byte[bufferSize]; // TODO usa buffer allocator?
in.readFully(bytes, 0, bufferSize);
Buffer buffer = Buffer.wrap(bytes);
int stringCount = WritableUtils.readVInt(in);
List<String> strings = new ArrayList<String>(stringCount);
for (int i = 0; i < stringCount; i++) {
strings.add(in.readUTF());
}
Page newPage = Page.wrap(buffer);
newPage.setStringReferences(strings);
if (page != null) {
page.release();
}
page = newPage;
}
public void readFields(DataInput in) throws IOException {
this.taskid.readFields(in);
this.progress = in.readFloat();
this.state = Text.readString(in);
this.startTime = in.readLong();
this.finishTime = in.readLong();
diagnostics = WritableUtils.readStringArray(in);
counters = new Counters();
counters.readFields(in);
currentStatus = WritableUtils.readEnum(in, TIPStatus.class);
if (currentStatus == TIPStatus.RUNNING) {
int num = WritableUtils.readVInt(in);
for (int i = 0; i < num; i++) {
TaskAttemptID t = new TaskAttemptID();
t.readFields(in);
runningAttempts.add(t);
}
} else if (currentStatus == TIPStatus.COMPLETE) {
successfulAttempt.readFields(in);
}
}
@Override
public void readFields(DataInput in) throws IOException {
this.freeEntries = WritableUtils.readVInt(in);
this.distinct = WritableUtils.readVInt(in);
this.lowWaterMark = WritableUtils.readVInt(in);
this.highWaterMark = WritableUtils.readVInt(in);
this.minLoadFactor = in.readDouble();
this.maxLoadFactor = in.readDouble();
this.table = new int[WritableUtils.readVInt(in)];
for (int i = 0; i < table.length; i++) {
table[i] = WritableUtils.readVInt(in);
}
this.values = new double[WritableUtils.readVInt(in)];
for (int i = 0; i < values.length; i++) {
values[i] = in.readDouble();
}
this.state = new byte[WritableUtils.readVInt(in)];
for (int i = 0; i < state.length; i++) {
state[i] = in.readByte();
}
}
@Override
public void readFields(DataInput input) throws IOException {
this.returnCode = MutationCode.values()[WritableUtils.readVInt(input)];
this.mutationTime = input.readLong();
boolean hasTable = input.readBoolean();
if (hasTable) {
this.table = new PTableImpl();
this.table.readFields(input);
}
columnName = Bytes.readByteArray(input);
if (columnName.length > 0) {
familyName = Bytes.readByteArray(input);
}
boolean hasTablesToDelete = input.readBoolean();
if (hasTablesToDelete) {
int count = input.readInt();
tableNamesToDelete = Lists.newArrayListWithExpectedSize(count);
for (int i = 0; i < count; i++) {
byte[] tableName = Bytes.readByteArray(input);
tableNamesToDelete.add(tableName);
}
}
}
/**
* Creates a <code>DfTableArray</code> object.
*
* @param file collection frequency data file path
* @param fs FileSystem to read from
* @throws IOException
*/
public DfTableArray(Path file, FileSystem fs) throws IOException {
FSDataInputStream in = fs.open(file);
this.numTerms = in.readInt();
dfs = new int[numTerms];
for (int i = 0; i < numTerms; i++) {
int df = WritableUtils.readVInt(in);
dfs[i] = df;
if (df > maxDf) {
maxDf = df;
maxDfTerm = i + 1;
}
if (df == 1) {
numSingletonTerms++;
}
}
in.close();
}
public void readFields(DataInput in) throws IOException {
bytes = WritableUtils.readVLong(in);
records = WritableUtils.readVLong(in);
checksum = WritableUtils.readVInt(in);
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
empName = WritableUtils.readString(in);
empID = WritableUtils.readVInt(in);
}
public void readFields(DataInput in) throws IOException {
docLength = WritableUtils.readVInt(in);
weightedTerms = new HMapIFW();
weightedTerms.readFields(in);
}
@Override
public void readFields(DataInput in) throws IOException {
split = WritableUtils.readVInt(in);
sequence = WritableUtils.readVLong(in);
}
@Override
public void readFields(DataInput in) throws IOException {
id = WritableUtils.readVInt(in);
distance = WritableUtils.readVInt(in);
}
@Override
public void readFields(DataInput arg0) throws IOException {
// TODO Auto-generated method stub
id = WritableUtils.readVInt(arg0);
}