/*
* this.values = copy.values; this.table = copy.table; this.state = copy.state; this.freeEntries =
* copy.freeEntries; this.distinct = copy.distinct; this.lowWaterMark = copy.lowWaterMark;
* this.highWaterMark = copy.highWaterMark; this.minLoadFactor = copy.minLoadFactor;
* this.maxLoadFactor = copy.maxLoadFactor;
*/
@Override
public void write(DataOutput out) throws IOException {
this.trimToSize();
WritableUtils.writeVInt(out, freeEntries);
WritableUtils.writeVInt(out, distinct);
WritableUtils.writeVInt(out, lowWaterMark);
WritableUtils.writeVInt(out, highWaterMark);
out.writeDouble(minLoadFactor);
out.writeDouble(maxLoadFactor);
WritableUtils.writeVInt(out, table.length);
for (int i = 0; i < table.length; i++) {
WritableUtils.writeVInt(out, table[i]);
}
WritableUtils.writeVInt(out, values.length);
for (int i = 0; i < values.length; i++) {
out.writeDouble(values[i]);
}
WritableUtils.writeVInt(out, state.length);
for (int i = 0; i < state.length; i++) {
out.writeByte(state[i]);
}
}
@Override
public void readFields(DataInput input) throws IOException {
super.readFields(input);
formatString = WritableUtils.readString(input);
type = WritableUtils.readEnum(input, FunctionArgumentType.class);
formatter = type.getFormatter(formatString);
}
@Override
public void readFields(DataInput input) throws IOException {
int encodedByteLengthAndBool = WritableUtils.readVInt(input);
int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
this.byteValue = new byte[byteLength];
input.readFully(byteValue, 0, byteLength);
int sortOrderAndDeterminism = WritableUtils.readVInt(input);
if (sortOrderAndDeterminism <= 2) {
// client is on an older version
this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
;
} else {
int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
this.determinism = Determinism.values()[determinismOrdinal];
int sortOrderValue =
sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
}
int typeOrdinal = WritableUtils.readVInt(input);
if (typeOrdinal < 0) {
this.type = null;
} else {
this.type = PDataType.values()[typeOrdinal];
}
if (this.byteValue.length == 0) {
this.value = null;
} else {
this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
}
}
@Override
public void write(DataOutput output) throws IOException {
super.write(output);
WritableUtils.writeVInt(output, toType.ordinal());
WritableUtils.writeVInt(output, ColumnModifier.toSystemValue(toMod));
WritableUtils.writeVInt(output, byteSize == null ? -1 : byteSize);
}
private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {
ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
int prevDocID = 0;
try {
while (true) {
int left = WritableUtils.readVInt(dataIn);
int right = WritableUtils.readVInt(dataIn);
if (right != 0) {
posting.add(new PairOfInts(left + prevDocID, right));
prevDocID += left;
}
}
} catch (EOFException e) {
} catch (IOException e) {
}
try {
dataIn.close();
} catch (IOException e) {
}
return posting;
}
public static TupleProjector deserializeProjectorFromScan(Scan scan) {
byte[] proj = scan.getAttribute(SCAN_PROJECTOR);
if (proj == null) {
return null;
}
ByteArrayInputStream stream = new ByteArrayInputStream(proj);
try {
DataInputStream input = new DataInputStream(stream);
KeyValueSchema schema = new KeyValueSchema();
schema.readFields(input);
int count = WritableUtils.readVInt(input);
Expression[] expressions = new Expression[count];
for (int i = 0; i < count; i++) {
int ordinal = WritableUtils.readVInt(input);
expressions[i] = ExpressionType.values()[ordinal].newInstance();
expressions[i].readFields(input);
}
return new TupleProjector(schema, expressions);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@Override
public void readFields(DataInput in) throws IOException {
first_key = WritableUtils.readString(in);
second_key = WritableUtils.readString(in);
third_key = WritableUtils.readString(in);
}
/**
* @param cell
* @param out
* @param encodingCtx
* @return unencoded size added
* @throws IOException
*/
protected final int afterEncodingKeyValue(
Cell cell, DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx)
throws IOException {
int size = 0;
if (encodingCtx.getHFileContext().isIncludesTags()) {
int tagsLength = cell.getTagsLength();
ByteBufferUtils.putCompressedInt(out, tagsLength);
// There are some tags to be written
if (tagsLength > 0) {
TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
// When tag compression is enabled, tagCompressionContext will have a not null value. Write
// the tags using Dictionary compression in such a case
if (tagCompressionContext != null) {
tagCompressionContext.compressTags(
out, cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
} else {
out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
}
}
size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
}
if (encodingCtx.getHFileContext().isIncludesMvcc()) {
// Copy memstore timestamp from the byte buffer to the output stream.
long memstoreTS = cell.getSequenceId();
WritableUtils.writeVLong(out, memstoreTS);
// TODO use a writeVLong which returns the #bytes written so that 2 time parsing can be
// avoided.
size += WritableUtils.getVIntSize(memstoreTS);
}
return size;
}
public void readFields(DataInput in) throws IOException {
in.readFully(internalBytes, 0, 4);
if (TimestampWritable.hasDecimalOrSecondVInt(internalBytes[0])) {
in.readFully(internalBytes, 4, 1);
int len = (byte) WritableUtils.decodeVIntSize(internalBytes[4]);
if (len > 1) {
in.readFully(internalBytes, 5, len - 1);
}
long vlong = LazyBinaryUtils.readVLongFromByteArray(internalBytes, 4);
if (vlong < -1000000000 || vlong > 999999999) {
throw new IOException(
"Invalid first vint value (encoded nanoseconds) of a TimestampWritable: "
+ vlong
+ ", expected to be between -1000000000 and 999999999.");
// Note that -1000000000 is a valid value corresponding to a nanosecond timestamp
// of 999999999, because if the second VInt is present, we use the value
// (-reversedNanoseconds - 1) as the second VInt.
}
if (vlong < 0) {
// This indicates there is a second VInt containing the additional bits of the seconds
// field.
in.readFully(internalBytes, 4 + len, 1);
int secondVIntLen = (byte) WritableUtils.decodeVIntSize(internalBytes[4 + len]);
if (secondVIntLen > 1) {
in.readFully(internalBytes, 5 + len, secondVIntLen - 1);
}
}
}
currentBytes = internalBytes;
this.offset = 0;
}
public int getEstimatedByteSize() {
int size = 0;
size += WritableUtils.getVIntSize(minNullable);
size += WritableUtils.getVIntSize(fields.size());
size += fields.size() * 3;
return size;
}
/**
* Write a Map<String, String>
*
* @param out DataOutput
* @param map Map to write
* @throws IOException I/O errors
*/
public static void writeStrStrMap(DataOutput out, Map<String, String> map) throws IOException {
out.writeInt(map.size());
for (Map.Entry<String, String> entry : map.entrySet()) {
WritableUtils.writeString(out, entry.getKey());
WritableUtils.writeString(out, entry.getValue());
}
}
public void serialize(DataOutput output) throws IOException {
WritableUtils.writeVInt(output, minNullable);
WritableUtils.writeVInt(output, fields.size());
for (int i = 0; i < fields.size(); i++) {
fields.get(i).write(output);
}
}
@Override
public void write(DataOutput out) throws IOException {
WritableUtils.writeString(out, first_key);
WritableUtils.writeString(out, second_key);
WritableUtils.writeString(out, third_key);
}
@Override
public void write(DataOutput output) throws IOException {
WritableUtils.writeVInt(output, (type.ordinal() + 1) * (this.isNullable ? -1 : 1));
WritableUtils.writeVInt(output, count * (sortOrder == SortOrder.ASC ? 1 : -1));
if (type.isFixedWidth() && type.getByteSize() == null) {
WritableUtils.writeVInt(output, byteSize);
}
}
@Override
public void readFields(DataInput input) throws IOException {
super.readFields(input);
toType = PDataType.values()[WritableUtils.readVInt(input)];
toMod = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
int byteSize = WritableUtils.readVInt(input);
this.byteSize = byteSize == -1 ? null : byteSize;
}
/* Receive a response.
* Because only one receiver, so no synchronization on in.
*/
protected void receiveResponse() {
if (shouldCloseConnection.get()) {
return;
}
touch();
try {
// See HBaseServer.Call.setResponse for where we write out the response.
// It writes the call.id (int), a flag byte, then optionally the length
// of the response (int) followed by data.
// Read the call id.
int id = in.readInt();
if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + id);
Call call = calls.get(id);
// Read the flag byte
byte flag = in.readByte();
boolean isError = ResponseFlag.isError(flag);
if (ResponseFlag.isLength(flag)) {
// Currently length if present is unused.
in.readInt();
}
int state = in.readInt(); // Read the state. Currently unused.
if (isError) {
if (call != null) {
//noinspection ThrowableInstanceNeverThrown
call.setException(
new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)));
}
} else {
Writable value = ReflectionUtils.newInstance(valueClass, conf);
value.readFields(in); // read value
// it's possible that this call may have been cleaned up due to a RPC
// timeout, so check if it still exists before setting the value.
if (call != null) {
call.setValue(value);
}
}
calls.remove(id);
} catch (IOException e) {
if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
// Clean up open calls but don't treat this as a fatal condition,
// since we expect certain responses to not make it by the specified
// {@link ConnectionId#rpcTimeout}.
closeException = e;
} else {
// Since the server did not respond within the default ping interval
// time, treat this as a fatal condition and close this connection
markClosed(e);
}
} finally {
if (remoteId.rpcTimeout > 0) {
cleanupCalls(remoteId.rpcTimeout);
}
}
}
/**
* Write key-value pair at the head of this stream to the objects provided; get next key-value
* pair from proxied RR.
*/
public boolean next(K key, U value) throws IOException {
if (hasNext()) {
WritableUtils.cloneInto(key, khead);
WritableUtils.cloneInto(value, vhead);
next();
return true;
}
return false;
}
/**
* Read in a Map<String, String>
*
* @param in DataInput
* @param map Map to read into
* @throws IOException I/O errors
*/
public static void readStrStrMap(DataInput in, Map<String, String> map) throws IOException {
int size = in.readInt();
map.clear();
for (int i = 0; i < size; ++i) {
String key = WritableUtils.readString(in);
String value = WritableUtils.readString(in);
map.put(key, value);
}
}
/** Decode Writable postings and compare by id */
public int compare(byte[] arg0, int arg1, int arg2, byte[] arg3, int arg4, int arg5) {
try {
DataInputStream di1 = new DataInputStream(new ByteArrayInputStream(arg0, arg1, arg2));
DataInputStream di2 = new DataInputStream(new ByteArrayInputStream(arg3, arg4, arg5));
return WritableUtils.readVInt(di1) - WritableUtils.readVInt(di2);
} catch (Exception e) {
return 0;
}
}
public static int getTotalLength(byte[] bytes, int offset) {
int len = 4;
if (hasDecimalOrSecondVInt(bytes[offset])) {
int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]);
len += firstVIntLen;
if (hasSecondVInt(bytes[offset + 4])) {
len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]);
}
}
return len;
}
@Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
try {
int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1);
int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2);
return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2);
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
@Override
public void readFields(DataInput in) throws IOException {
int minNullable = WritableUtils.readVInt(in);
int nFields = WritableUtils.readVInt(in);
List<Field> fields = Lists.newArrayListWithExpectedSize(nFields);
for (int i = 0; i < nFields; i++) {
Field field = new Field();
field.readFields(in);
fields.add(field);
}
init(minNullable, fields);
}
/** Optimization hook. */
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
long thisLeftValue = readLong(b1, s1);
long thatLeftValue = readLong(b2, s2);
if (thisLeftValue == thatLeftValue) {
int n1 = WritableUtils.decodeVIntSize(b1[s1 + 8]);
int n2 = WritableUtils.decodeVIntSize(b2[s2 + 8]);
return compareBytes(b1, s1 + 8 + n1, l1 - n1 - 8, b2, s2 + n2 + 8, l2 - n2 - 8);
}
return thisLeftValue < thatLeftValue ? -1 : 1;
}
@Override
public void write(DataOutput out) throws IOException {
WritableUtils.writeVInt(out, id);
WritableUtils.writeVLong(out, expirationDate);
if (secret == null) {
WritableUtils.writeVInt(out, -1);
} else {
byte[] keyBytes = secret.getEncoded();
WritableUtils.writeVInt(out, keyBytes.length);
out.write(keyBytes);
}
}
@Override
public void write(DataOutput output) throws IOException {
WritableUtils.writeVInt(
output, (byteValue.length + 1) * (this.determinism == Determinism.ALWAYS ? 1 : -1));
output.write(byteValue);
// since we need to support clients of a lower version, serialize the determinism enum ordinal
// in the int used to
// serialize sort order system value (which is either 1 or 2)
int sortOrderAndDeterminism =
((this.determinism.ordinal() + 1) << 2) + sortOrder.getSystemValue();
WritableUtils.writeVInt(output, sortOrderAndDeterminism);
WritableUtils.writeVInt(output, this.type == null ? -1 : this.type.ordinal());
}
@Override
public void readFields(DataInput in) throws IOException {
id = WritableUtils.readVInt(in);
expirationDate = WritableUtils.readVLong(in);
int keyLength = WritableUtils.readVInt(in);
if (keyLength < 0) {
secret = null;
} else {
byte[] keyBytes = new byte[keyLength];
in.readFully(keyBytes);
secret = AuthenticationTokenSecretManager.createSecretKey(keyBytes);
}
}
/**
* Read a zero-compressed encoded long from a byte array.
*
* @param bytes the byte array
* @param offset the offset in the byte array where the VLong is stored
* @return the long
*/
public static long readVLongFromByteArray(final byte[] bytes, int offset) {
byte firstByte = bytes[offset++];
int len = WritableUtils.decodeVIntSize(firstByte);
if (len == 1) {
return firstByte;
}
long i = 0;
for (int idx = 0; idx < len - 1; idx++) {
byte b = bytes[offset++];
i = i << 8;
i = i | (b & 0xFF);
}
return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
/** Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuff}. */
public static long readVLong(ByteBuff in) {
byte firstByte = in.get();
int len = WritableUtils.decodeVIntSize(firstByte);
if (len == 1) {
return firstByte;
}
long i = 0;
for (int idx = 0; idx < len - 1; idx++) {
byte b = in.get();
i = i << 8;
i = i | (b & 0xFF);
}
return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
}
@Override
public void write(DataOutput output) throws IOException {
Bytes.writeByteArray(output, name.getBytes());
WritableUtils.writeVInt(output, type.ordinal());
WritableUtils.writeVLong(output, sequenceNumber);
output.writeLong(timeStamp);
Bytes.writeByteArray(
output, pkName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(pkName));
WritableUtils.writeVInt(output, allColumns.size());
for (int i = 0; i < allColumns.size(); i++) {
PColumn column = allColumns.get(i);
column.write(output);
}
stats.write(output);
}
@Override
public void readFields(DataInput input) throws IOException {
// read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
// save space
int typeAndFlag = WritableUtils.readVInt(input);
isNullable = (typeAndFlag & 0x01) != 0;
if ((typeAndFlag & 0x02) != 0) {
scale = WritableUtils.readVInt(input);
}
if ((typeAndFlag & 0x04) != 0) {
maxLength = WritableUtils.readVInt(input);
}
type = PDataType.values()[typeAndFlag >>> 3];
sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
}
