protected final void afterDecodingKeyValue(
DataInputStream source, ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx)
throws IOException {
if (decodingCtx.getHFileContext().isIncludesTags()) {
int tagsLength = ByteBufferUtils.readCompressedInt(source);
// Put as unsigned short
dest.put((byte) ((tagsLength >> 8) & 0xff));
dest.put((byte) (tagsLength & 0xff));
if (tagsLength > 0) {
TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
// When tag compression is been used in this file, tagCompressionContext will have a not
// null value passed.
if (tagCompressionContext != null) {
tagCompressionContext.uncompressTags(source, dest, tagsLength);
} else {
ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
}
}
}
if (decodingCtx.getHFileContext().isIncludesMvcc()) {
long memstoreTS = -1;
try {
// Copy memstore timestamp from the data input stream to the byte
// buffer.
memstoreTS = WritableUtils.readVLong(source);
ByteBufferUtils.writeVLong(dest, memstoreTS);
} catch (IOException ex) {
throw new RuntimeException(
"Unable to copy memstore timestamp " + memstoreTS + " after decoding a key/value");
}
}
}
@Test
public void testGetKeyMethods() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getKeyLength());
ByteBufferUtils.copyFromArrayToBuffer(
buf, kvCell.getBuffer(), kvCell.getKeyOffset(), kvCell.getKeyLength());
ByteBufferedCell offheapKeyOnlyKV = new ByteBufferedKeyOnlyKeyValue(buf, 0, buf.capacity());
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getRowByteBuffer(),
offheapKeyOnlyKV.getRowPositionInByteBuffer(),
offheapKeyOnlyKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getFamilyByteBuffer(),
offheapKeyOnlyKV.getFamilyPositionInByteBuffer(),
offheapKeyOnlyKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getQualifierByteBuffer(),
offheapKeyOnlyKV.getQualifierPositionInByteBuffer(),
offheapKeyOnlyKV.getQualifierLength()));
assertEquals(0L, offheapKeyOnlyKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKeyOnlyKV.getTypeByte());
}
@Override
public SingleByteBuff put(int offset, ByteBuff src, int srcOffset, int length) {
if (src instanceof SingleByteBuff) {
ByteBufferUtils.copyFromBufferToBuffer(
((SingleByteBuff) src).buf, this.buf, srcOffset, offset, length);
} else {
// TODO we can do some optimization here? Call to asSubByteBuffer might
// create a copy.
Pair<ByteBuffer, Integer> pair = new Pair<ByteBuffer, Integer>();
src.asSubByteBuffer(srcOffset, length, pair);
ByteBufferUtils.copyFromBufferToBuffer(
pair.getFirst(), this.buf, pair.getSecond(), offset, length);
}
return this;
}
/**
* @param cell
* @param out
* @param encodingCtx
* @return unencoded size added
* @throws IOException
*/
protected final int afterEncodingKeyValue(
Cell cell, DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx)
throws IOException {
int size = 0;
if (encodingCtx.getHFileContext().isIncludesTags()) {
int tagsLength = cell.getTagsLength();
ByteBufferUtils.putCompressedInt(out, tagsLength);
// There are some tags to be written
if (tagsLength > 0) {
TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
// When tag compression is enabled, tagCompressionContext will have a not null value. Write
// the tags using Dictionary compression in such a case
if (tagCompressionContext != null) {
tagCompressionContext.compressTags(
out, cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
} else {
out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
}
}
size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
}
if (encodingCtx.getHFileContext().isIncludesMvcc()) {
// Copy memstore timestamp from the byte buffer to the output stream.
long memstoreTS = cell.getSequenceId();
WritableUtils.writeVLong(out, memstoreTS);
// TODO use a writeVLong which returns the #bytes written so that 2 time parsing can be
// avoided.
size += WritableUtils.getVIntSize(memstoreTS);
}
return size;
}
/**
* Special compareTo method for subclasses, to avoid copying bytes unnecessarily.
*
* @param value bytes to compare within a ByteBuffer
* @param offset offset into value
* @param length number of bytes to compare
* @return a negative integer, zero, or a positive integer as this object is less than, equal to,
* or greater than the specified object.
*/
public int compareTo(ByteBuffer value, int offset, int length) {
// For BC, providing a default implementation here which is doing a bytes copy to a temp byte[]
// and calling compareTo(byte[]). Make sure to override this method in subclasses to avoid
// copying bytes unnecessarily.
byte[] temp = new byte[length];
ByteBufferUtils.copyFromBufferToArray(temp, value, offset, 0, length);
return compareTo(temp);
}
@Test
public void testByteBufferBackedKeyValueWithTags() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), true, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
// change tags to handle both onheap and offheap stuff
List<Tag> resTags =
Tag.asList(offheapKV.getTagsArray(), offheapKV.getTagsOffset(), offheapKV.getTagsLength());
Tag tag1 = resTags.get(0);
assertEquals(t1.getType(), tag1.getType());
assertEquals(Bytes.toString(t1.getValue()), Bytes.toString(getTagValue(tag1)));
Tag tag2 = resTags.get(1);
assertEquals(tag2.getType(), tag2.getType());
assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
Tag res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 2);
assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 3);
assertNull(res);
}
/**
* Do the encoding, but do not cache the encoded data.
*
* @return encoded data block with header and checksum
*/
public byte[] encodeData() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
DataOutputStream out = new DataOutputStream(baos);
this.dataBlockEncoder.startBlockEncoding(encodingCtx, out);
ByteBuffer in = getUncompressedBuffer();
in.rewind();
int klength, vlength;
int tagsLength = 0;
long memstoreTS = 0L;
KeyValue kv = null;
while (in.hasRemaining()) {
int kvOffset = in.position();
klength = in.getInt();
vlength = in.getInt();
ByteBufferUtils.skip(in, klength + vlength);
if (this.meta.isIncludesTags()) {
tagsLength = ((in.get() & 0xff) << 8) ^ (in.get() & 0xff);
ByteBufferUtils.skip(in, tagsLength);
}
if (this.meta.isIncludesMvcc()) {
memstoreTS = ByteBufferUtils.readVLong(in);
}
kv =
new KeyValue(
in.array(),
kvOffset,
(int) KeyValue.getKeyValueDataStructureSize(klength, vlength, tagsLength));
kv.setSequenceId(memstoreTS);
this.dataBlockEncoder.encode(kv, encodingCtx, out);
}
BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
baos.writeTo(stream);
this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, stream.buf);
} catch (IOException e) {
throw new RuntimeException(
String.format(
"Bug in encoding part of algorithm %s. "
+ "Probably it requested more bytes than are available.",
toString()),
e);
}
return baos.toByteArray();
}
@Override
public int write(OutputStream out, boolean withTags) throws IOException {
int lenToWrite =
KeyValueUtil.length(
rowLength, familyLength, qualifierLength, valueLength, tagsLength, withTags);
ByteBufferUtils.putInt(out, lenToWrite);
ByteBufferUtils.putInt(out, keyBuffer.capacity());
ByteBufferUtils.putInt(out, valueLength);
// Write key
out.write(keyBuffer.array());
// Write value
ByteBufferUtils.writeByteBuffer(out, this.valueBuffer, this.valueOffset, this.valueLength);
if (withTags) {
// 2 bytes tags length followed by tags bytes
// tags length is serialized with 2 bytes only(short way) even if the type is int.
// As this is non -ve numbers, we save the sign bit. See HBASE-11437
out.write((byte) (0xff & (this.tagsLength >> 8)));
out.write((byte) (0xff & this.tagsLength));
ByteBufferUtils.writeByteBuffer(out, this.tagsBuffer, this.tagsOffset, this.tagsLength);
}
return lenToWrite + Bytes.SIZEOF_INT;
}
/**
* Convert list of KeyValues to byte buffer.
*
* @param keyValues list of KeyValues to be converted.
* @return buffer with content from key values
*/
public static ByteBuffer convertKvToByteBuffer(
List<KeyValue> keyValues, boolean includesMemstoreTS) {
int totalSize = 0;
for (KeyValue kv : keyValues) {
totalSize += kv.getLength();
if (includesMemstoreTS) {
totalSize += WritableUtils.getVIntSize(kv.getMvccVersion());
}
}
ByteBuffer result = ByteBuffer.allocate(totalSize);
for (KeyValue kv : keyValues) {
result.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
if (includesMemstoreTS) {
ByteBufferUtils.writeVLong(result, kv.getMvccVersion());
}
}
return result;
}
@Override
public byte[] toBytes(int offset, int length) {
byte[] output = new byte[length];
ByteBufferUtils.copyFromBufferToArray(output, buf, offset, 0, length);
return output;
}
@Override
public long getLongAfterPosition(int offset) {
return ByteBufferUtils.toLong(this.buf, this.buf.position() + offset);
}
@Override
public long getLong(int index) {
return ByteBufferUtils.toLong(this.buf, index);
}
@Override
public SingleByteBuff putLong(long value) {
ByteBufferUtils.putLong(this.buf, value);
return this;
}
@Override
public int getIntAfterPosition(int offset) {
return ByteBufferUtils.toInt(this.buf, this.buf.position() + offset);
}
@Test
public void testByteBufferBackedKeyValue() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
// Use the array() APIs
assertEquals(
ROW1,
Bytes.toStringBinary(
offheapKV.getRowArray(), offheapKV.getRowOffset(), offheapKV.getRowLength()));
assertEquals(
FAM1,
Bytes.toStringBinary(
offheapKV.getFamilyArray(), offheapKV.getFamilyOffset(), offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
Bytes.toStringBinary(
offheapKV.getQualifierArray(),
offheapKV.getQualifierOffset(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
Bytes.toStringBinary(
offheapKV.getValueArray(), offheapKV.getValueOffset(), offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
kvCell = new KeyValue(row1, fam2, qual2, 0l, Type.Put, row1);
buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
FAM2,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL2,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
byte[] nullQualifier = new byte[0];
kvCell = new KeyValue(row1, fam1, nullQualifier, 0L, Type.Put, row1);
buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
"",
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
}
@Override
public long getTimestamp() {
return ByteBufferUtils.toLong(this.buf, getTimestampOffset());
}
@Override
public short getShortAfterPosition(int offset) {
return ByteBufferUtils.toShort(this.buf, this.buf.position() + offset);
}
@Override
public short getShort(int index) {
return ByteBufferUtils.toShort(this.buf, index);
}
private byte getFamilyLength(int famLenPos) {
return ByteBufferUtils.toByte(this.buf, famLenPos);
}
/**
* A setter that helps to avoid object creation every time and whenever there is a need to create
* new OffheapKeyOnlyKeyValue.
*
* @param key
* @param offset
* @param length
*/
public void setKey(ByteBuffer key, int offset, int length) {
this.buf = key;
this.offset = offset;
this.length = length;
this.rowLen = ByteBufferUtils.toShort(this.buf, this.offset);
}
@Override
public void get(byte[] dst, int offset, int length) {
ByteBufferUtils.copyFromBufferToArray(dst, buf, buf.position(), offset, length);
buf.position(buf.position() + length);
}
@Override
public byte getByteAfterPosition(int offset) {
return ByteBufferUtils.toByte(this.buf, this.buf.position() + offset);
}
@Override
public byte get(int index) {
return ByteBufferUtils.toByte(this.buf, index);
}
@Override
public void get(ByteBuffer out, int sourceOffset, int length) {
ByteBufferUtils.copyFromBufferToBuffer(buf, out, sourceOffset, length);
}
@Override
public byte getTypeByte() {
return ByteBufferUtils.toByte(this.buf, this.offset + this.length - 1);
}
@Override
public SingleByteBuff putInt(int value) {
ByteBufferUtils.putInt(this.buf, value);
return this;
}
@Override
public SingleByteBuff put(byte[] src, int offset, int length) {
ByteBufferUtils.copyFromArrayToBuffer(this.buf, src, offset, length);
return this;
}
@Override
public int getInt(int index) {
return ByteBufferUtils.toInt(this.buf, index);
}