protected void decodeTags() {
current.tagsLength = ByteBuff.readCompressedInt(currentBuffer);
if (tagCompressionContext != null) {
if (current.uncompressTags) {
// Tag compression is been used. uncompress it into tagsBuffer
current.ensureSpaceForTags();
try {
current.tagsCompressedLength =
tagCompressionContext.uncompressTags(
currentBuffer, current.tagsBuffer, 0, current.tagsLength);
} catch (IOException e) {
throw new RuntimeException("Exception while uncompressing tags", e);
}
} else {
currentBuffer.skip(current.tagsCompressedLength);
current.uncompressTags = true; // Reset this.
}
current.tagsOffset = -1;
} else {
// When tag compress is not used, let us not do copying of tags bytes into tagsBuffer.
// Just mark the tags Offset so as to create the KV buffer later in getKeyValueBuffer()
current.tagsOffset = currentBuffer.position();
currentBuffer.skip(current.tagsLength);
}
}
/** Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuff}. */
public static long readVLong(ByteBuff in) {
byte firstByte = in.get();
int len = WritableUtils.decodeVIntSize(firstByte);
if (len == 1) {
return firstByte;
}
long i = 0;
for (int idx = 0; idx < len - 1; idx++) {
byte b = in.get();
i = i << 8;
i = i | (b & 0xFF);
}
return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
}
private void moveToPrevious() {
if (!previous.isValid()) {
throw new IllegalStateException(
"Can move back only once and not in first key in the block.");
}
STATE tmp = previous;
previous = current;
current = tmp;
// move after last key value
currentBuffer.position(current.nextKvOffset);
// Already decoded the tag bytes. We cache this tags into current state and also the total
// compressed length of the tags bytes. For the next time decodeNext() we don't need to decode
// the tags again. This might pollute the Data Dictionary what we use for the compression.
// When current.uncompressTags is false, we will just reuse the current.tagsBuffer and skip
// 'tagsCompressedLength' bytes of source stream.
// See in decodeTags()
current.tagsBuffer = previous.tagsBuffer;
current.tagsCompressedLength = previous.tagsCompressedLength;
current.uncompressTags = false;
// The current key has to be reset with the previous Cell
current.setKey(current.keyBuffer, current.memstoreTS);
previous.invalidate();
}
/**
* Read long which was written to fitInBytes bytes and increment position.
*
* @param fitInBytes In how many bytes given long is stored.
* @return The value of parsed long.
*/
public static long readLong(ByteBuff in, final int fitInBytes) {
long tmpLength = 0;
for (int i = 0; i < fitInBytes; ++i) {
tmpLength |= (in.get() & 0xffl) << (8l * i);
}
return tmpLength;
}
@Override
public ByteBuffer getValueShallowCopy() {
currentBuffer.asSubByteBuffer(current.valueOffset, current.valueLength, tmpPair);
ByteBuffer dup = tmpPair.getFirst().duplicate();
dup.position(tmpPair.getSecond());
dup.limit(tmpPair.getSecond() + current.valueLength);
return dup.slice();
}
/**
* Read integer from ByteBuff coded in 7 bits and increment position.
*
* @return Read integer.
*/
public static int readCompressedInt(ByteBuff buf) {
byte b = buf.get();
if ((b & ByteBufferUtils.NEXT_BIT_MASK) != 0) {
return (b & ByteBufferUtils.VALUE_MASK)
+ (readCompressedInt(buf) << ByteBufferUtils.NEXT_BIT_SHIFT);
}
return b & ByteBufferUtils.VALUE_MASK;
}
public static String toStringBinary(final ByteBuff b, int off, int len) {
StringBuilder result = new StringBuilder();
// Just in case we are passed a 'len' that is > buffer length...
if (off >= b.capacity()) return result.toString();
if (off + len > b.capacity()) len = b.capacity() - off;
for (int i = off; i < off + len; ++i) {
int ch = b.get(i) & 0xFF;
if ((ch >= '0' && ch <= '9')
|| (ch >= 'A' && ch <= 'Z')
|| (ch >= 'a' && ch <= 'z')
|| " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
result.append((char) ch);
} else {
result.append(String.format("\\x%02X", ch));
}
}
return result.toString();
}
@Override
public boolean next() {
if (!currentBuffer.hasRemaining()) {
return false;
}
decodeNext();
current.setKey(current.keyBuffer, current.memstoreTS);
previous.invalidate();
return true;
}
@Override
public void rewind() {
currentBuffer.rewind();
if (tagCompressionContext != null) {
tagCompressionContext.clear();
}
decodeFirst();
current.setKey(current.keyBuffer, current.memstoreTS);
previous.invalidate();
}
/**
* Compares two ByteBuffs
*
* @param buf1 the first ByteBuff
* @param o1 the offset in the first ByteBuff from where the compare has to happen
* @param len1 the length in the first ByteBuff upto which the compare has to happen
* @param buf2 the second ByteBuff
* @param o2 the offset in the second ByteBuff from where the compare has to happen
* @param len2 the length in the second ByteBuff upto which the compare has to happen
* @return Positive if buf1 is bigger than buf2, 0 if they are equal, and negative if buf1 is
* smaller than buf2.
*/
public static int compareTo(ByteBuff buf1, int o1, int len1, ByteBuff buf2, int o2, int len2) {
if (buf1.hasArray() && buf2.hasArray()) {
return Bytes.compareTo(
buf1.array(), buf1.arrayOffset() + o1, len1, buf2.array(), buf2.arrayOffset() + o2, len2);
}
int end1 = o1 + len1;
int end2 = o2 + len2;
for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) {
int a = buf1.get(i) & 0xFF;
int b = buf2.get(j) & 0xFF;
if (a != b) {
return a - b;
}
}
return len1 - len2;
}
@Override
public ByteBuffer getKeyValueBuffer() {
ByteBuffer kvBuffer = createKVBuffer();
kvBuffer.putInt(current.keyLength);
kvBuffer.putInt(current.valueLength);
kvBuffer.put(current.keyBuffer, 0, current.keyLength);
currentBuffer.get(kvBuffer, current.valueOffset, current.valueLength);
if (current.tagsLength > 0) {
// Put short as unsigned
kvBuffer.put((byte) (current.tagsLength >> 8 & 0xff));
kvBuffer.put((byte) (current.tagsLength & 0xff));
if (current.tagsOffset != -1) {
// the offset of the tags bytes in the underlying buffer is marked. So the temp
// buffer,tagsBuffer was not been used.
currentBuffer.get(kvBuffer, current.tagsOffset, current.tagsLength);
} else {
// When tagsOffset is marked as -1, tag compression was present and so the tags were
// uncompressed into temp buffer, tagsBuffer. Let us copy it from there
kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
}
}
kvBuffer.rewind();
return kvBuffer;
}
@Override
public SingleByteBuff put(int offset, ByteBuff src, int srcOffset, int length) {
if (src instanceof SingleByteBuff) {
ByteBufferUtils.copyFromBufferToBuffer(
((SingleByteBuff) src).buf, this.buf, srcOffset, offset, length);
} else {
// TODO we can do some optimization here? Call to asSubByteBuffer might
// create a copy.
Pair<ByteBuffer, Integer> pair = new Pair<ByteBuffer, Integer>();
src.asSubByteBuffer(srcOffset, length, pair);
ByteBufferUtils.copyFromBufferToBuffer(
pair.getFirst(), this.buf, pair.getSecond(), offset, length);
}
return this;
}
/**
* Search sorted array "a" for byte "key".
*
* @param a Array to search. Entries must be sorted and unique.
* @param fromIndex First index inclusive of "a" to include in the search.
* @param toIndex Last index exclusive of "a" to include in the search.
* @param key The byte to search for.
* @return The index of key if found. If not found, return -(index + 1), where negative indicates
* "not found" and the "index + 1" handles the "-0" case.
*/
public static int unsignedBinarySearch(ByteBuff a, int fromIndex, int toIndex, byte key) {
int unsignedKey = key & 0xff;
int low = fromIndex;
int high = toIndex - 1;
while (low <= high) {
int mid = (low + high) >>> 1;
int midVal = a.get(mid) & 0xff;
if (midVal < unsignedKey) {
low = mid + 1;
} else if (midVal > unsignedKey) {
high = mid - 1;
} else {
return mid; // key found
}
}
return -(low + 1); // key not found.
}
@Override
public int seekToKeyInBlock(Cell seekCell, boolean seekBefore) {
int rowCommonPrefix = 0;
int familyCommonPrefix = 0;
int qualCommonPrefix = 0;
previous.invalidate();
do {
int comp;
keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength);
if (current.lastCommonPrefix != 0) {
// The KV format has row key length also in the byte array. The
// common prefix
// includes it. So we need to subtract to find out the common prefix
// in the
// row part alone
rowCommonPrefix = Math.min(rowCommonPrefix, current.lastCommonPrefix - 2);
}
if (current.lastCommonPrefix <= 2) {
rowCommonPrefix = 0;
}
rowCommonPrefix += findCommonPrefixInRowPart(seekCell, keyOnlyKV, rowCommonPrefix);
comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix);
if (comp == 0) {
comp = compareTypeBytes(seekCell, keyOnlyKV);
if (comp == 0) {
// Subtract the fixed row key length and the family key fixed length
familyCommonPrefix =
Math.max(
0,
Math.min(
familyCommonPrefix,
current.lastCommonPrefix - (3 + keyOnlyKV.getRowLength())));
familyCommonPrefix +=
findCommonPrefixInFamilyPart(seekCell, keyOnlyKV, familyCommonPrefix);
comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix);
if (comp == 0) {
// subtract the rowkey fixed length and the family key fixed
// length
qualCommonPrefix =
Math.max(
0,
Math.min(
qualCommonPrefix,
current.lastCommonPrefix
- (3 + keyOnlyKV.getRowLength() + keyOnlyKV.getFamilyLength())));
qualCommonPrefix +=
findCommonPrefixInQualifierPart(seekCell, keyOnlyKV, qualCommonPrefix);
comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix);
if (comp == 0) {
comp = CellComparator.compareTimestamps(seekCell, keyOnlyKV);
if (comp == 0) {
// Compare types. Let the delete types sort ahead of puts;
// i.e. types
// of higher numbers sort before those of lesser numbers.
// Maximum
// (255)
// appears ahead of everything, and minimum (0) appears
// after
// everything.
comp = (0xff & keyOnlyKV.getTypeByte()) - (0xff & seekCell.getTypeByte());
}
}
}
}
}
if (comp == 0) { // exact match
if (seekBefore) {
if (!previous.isValid()) {
// The caller (seekBefore) has to ensure that we are not at the
// first key in the block.
throw new IllegalStateException(
"Cannot seekBefore if "
+ "positioned at the first key in the block: key="
+ Bytes.toStringBinary(seekCell.getRowArray()));
}
moveToPrevious();
return 1;
}
return 0;
}
if (comp < 0) { // already too large, check previous
if (previous.isValid()) {
moveToPrevious();
} else {
return HConstants.INDEX_KEY_MAGIC; // using optimized index key
}
return 1;
}
// move to next, if more data is available
if (currentBuffer.hasRemaining()) {
previous.copyFromNext(current);
decodeNext();
current.setKey(current.keyBuffer, current.memstoreTS);
} else {
break;
}
} while (true);
// we hit the end of the block, not an exact match
return 1;
}
@Test
public void testAllocateByteBuffToReadInto() throws Exception {
int maxBuffersInPool = 10;
ByteBufferPool pool = new ByteBufferPool(6 * 1024, maxBuffersInPool);
initPoolWithAllBuffers(pool, maxBuffersInPool);
ByteBuff buff = null;
Pair<ByteBuff, CallCleanup> pair;
// When the request size is less than 1/6th of the pool buffer size. We should use on demand
// created on heap Buffer
pair =
RpcServer.allocateByteBuffToReadInto(pool, RpcServer.getMinSizeForReservoirUse(pool), 200);
buff = pair.getFirst();
assertTrue(buff.hasArray());
assertEquals(maxBuffersInPool, pool.getQueueSize());
assertNull(pair.getSecond());
// When the request size is > 1/6th of the pool buffer size.
pair =
RpcServer.allocateByteBuffToReadInto(pool, RpcServer.getMinSizeForReservoirUse(pool), 1024);
buff = pair.getFirst();
assertFalse(buff.hasArray());
assertEquals(maxBuffersInPool - 1, pool.getQueueSize());
assertNotNull(pair.getSecond());
pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
assertEquals(maxBuffersInPool, pool.getQueueSize());
// Request size> pool buffer size
pair =
RpcServer.allocateByteBuffToReadInto(
pool, RpcServer.getMinSizeForReservoirUse(pool), 7 * 1024);
buff = pair.getFirst();
assertFalse(buff.hasArray());
assertTrue(buff instanceof MultiByteBuff);
ByteBuffer[] bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
assertEquals(2, bbs.length);
assertTrue(bbs[0].isDirect());
assertTrue(bbs[1].isDirect());
assertEquals(6 * 1024, bbs[0].limit());
assertEquals(1024, bbs[1].limit());
assertEquals(maxBuffersInPool - 2, pool.getQueueSize());
assertNotNull(pair.getSecond());
pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
assertEquals(maxBuffersInPool, pool.getQueueSize());
pair =
RpcServer.allocateByteBuffToReadInto(
pool, RpcServer.getMinSizeForReservoirUse(pool), 6 * 1024 + 200);
buff = pair.getFirst();
assertFalse(buff.hasArray());
assertTrue(buff instanceof MultiByteBuff);
bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
assertEquals(2, bbs.length);
assertTrue(bbs[0].isDirect());
assertFalse(bbs[1].isDirect());
assertEquals(6 * 1024, bbs[0].limit());
assertEquals(200, bbs[1].limit());
assertEquals(maxBuffersInPool - 1, pool.getQueueSize());
assertNotNull(pair.getSecond());
pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
assertEquals(maxBuffersInPool, pool.getQueueSize());
ByteBuffer[] buffers = new ByteBuffer[maxBuffersInPool - 1];
for (int i = 0; i < maxBuffersInPool - 1; i++) {
buffers[i] = pool.getBuffer();
}
pair =
RpcServer.allocateByteBuffToReadInto(
pool, RpcServer.getMinSizeForReservoirUse(pool), 20 * 1024);
buff = pair.getFirst();
assertFalse(buff.hasArray());
assertTrue(buff instanceof MultiByteBuff);
bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
assertEquals(2, bbs.length);
assertTrue(bbs[0].isDirect());
assertFalse(bbs[1].isDirect());
assertEquals(6 * 1024, bbs[0].limit());
assertEquals(14 * 1024, bbs[1].limit());
assertEquals(0, pool.getQueueSize());
assertNotNull(pair.getSecond());
pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
assertEquals(1, pool.getQueueSize());
pool.getBuffer();
pair =
RpcServer.allocateByteBuffToReadInto(
pool, RpcServer.getMinSizeForReservoirUse(pool), 7 * 1024);
buff = pair.getFirst();
assertTrue(buff.hasArray());
assertTrue(buff instanceof SingleByteBuff);
assertEquals(7 * 1024, ((SingleByteBuff) buff).getEnclosingByteBuffer().limit());
assertNull(pair.getSecond());
}