Ejemplo n.º 1
0
 protected void decodeTags() {
   current.tagsLength = ByteBuff.readCompressedInt(currentBuffer);
   if (tagCompressionContext != null) {
     if (current.uncompressTags) {
       // Tag compression is been used. uncompress it into tagsBuffer
       current.ensureSpaceForTags();
       try {
         current.tagsCompressedLength =
             tagCompressionContext.uncompressTags(
                 currentBuffer, current.tagsBuffer, 0, current.tagsLength);
       } catch (IOException e) {
         throw new RuntimeException("Exception while uncompressing tags", e);
       }
     } else {
       currentBuffer.skip(current.tagsCompressedLength);
       current.uncompressTags = true; // Reset this.
     }
     current.tagsOffset = -1;
   } else {
     // When tag compress is not used, let us not do copying of tags bytes into tagsBuffer.
     // Just mark the tags Offset so as to create the KV buffer later in getKeyValueBuffer()
     current.tagsOffset = currentBuffer.position();
     currentBuffer.skip(current.tagsLength);
   }
 }
Ejemplo n.º 2
0
 /** Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuff}. */
 public static long readVLong(ByteBuff in) {
   byte firstByte = in.get();
   int len = WritableUtils.decodeVIntSize(firstByte);
   if (len == 1) {
     return firstByte;
   }
   long i = 0;
   for (int idx = 0; idx < len - 1; idx++) {
     byte b = in.get();
     i = i << 8;
     i = i | (b & 0xFF);
   }
   return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
 }
Ejemplo n.º 3
0
    private void moveToPrevious() {
      if (!previous.isValid()) {
        throw new IllegalStateException(
            "Can move back only once and not in first key in the block.");
      }

      STATE tmp = previous;
      previous = current;
      current = tmp;

      // move after last key value
      currentBuffer.position(current.nextKvOffset);
      // Already decoded the tag bytes. We cache this tags into current state and also the total
      // compressed length of the tags bytes. For the next time decodeNext() we don't need to decode
      // the tags again. This might pollute the Data Dictionary what we use for the compression.
      // When current.uncompressTags is false, we will just reuse the current.tagsBuffer and skip
      // 'tagsCompressedLength' bytes of source stream.
      // See in decodeTags()
      current.tagsBuffer = previous.tagsBuffer;
      current.tagsCompressedLength = previous.tagsCompressedLength;
      current.uncompressTags = false;
      // The current key has to be reset with the previous Cell
      current.setKey(current.keyBuffer, current.memstoreTS);
      previous.invalidate();
    }
Ejemplo n.º 4
0
 /**
  * Read long which was written to fitInBytes bytes and increment position.
  *
  * @param fitInBytes In how many bytes given long is stored.
  * @return The value of parsed long.
  */
 public static long readLong(ByteBuff in, final int fitInBytes) {
   long tmpLength = 0;
   for (int i = 0; i < fitInBytes; ++i) {
     tmpLength |= (in.get() & 0xffl) << (8l * i);
   }
   return tmpLength;
 }
Ejemplo n.º 5
0
 @Override
 public ByteBuffer getValueShallowCopy() {
   currentBuffer.asSubByteBuffer(current.valueOffset, current.valueLength, tmpPair);
   ByteBuffer dup = tmpPair.getFirst().duplicate();
   dup.position(tmpPair.getSecond());
   dup.limit(tmpPair.getSecond() + current.valueLength);
   return dup.slice();
 }
Ejemplo n.º 6
0
 /**
  * Read integer from ByteBuff coded in 7 bits and increment position.
  *
  * @return Read integer.
  */
 public static int readCompressedInt(ByteBuff buf) {
   byte b = buf.get();
   if ((b & ByteBufferUtils.NEXT_BIT_MASK) != 0) {
     return (b & ByteBufferUtils.VALUE_MASK)
         + (readCompressedInt(buf) << ByteBufferUtils.NEXT_BIT_SHIFT);
   }
   return b & ByteBufferUtils.VALUE_MASK;
 }
Ejemplo n.º 7
0
 public static String toStringBinary(final ByteBuff b, int off, int len) {
   StringBuilder result = new StringBuilder();
   // Just in case we are passed a 'len' that is > buffer length...
   if (off >= b.capacity()) return result.toString();
   if (off + len > b.capacity()) len = b.capacity() - off;
   for (int i = off; i < off + len; ++i) {
     int ch = b.get(i) & 0xFF;
     if ((ch >= '0' && ch <= '9')
         || (ch >= 'A' && ch <= 'Z')
         || (ch >= 'a' && ch <= 'z')
         || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
       result.append((char) ch);
     } else {
       result.append(String.format("\\x%02X", ch));
     }
   }
   return result.toString();
 }
Ejemplo n.º 8
0
 @Override
 public boolean next() {
   if (!currentBuffer.hasRemaining()) {
     return false;
   }
   decodeNext();
   current.setKey(current.keyBuffer, current.memstoreTS);
   previous.invalidate();
   return true;
 }
Ejemplo n.º 9
0
 @Override
 public void rewind() {
   currentBuffer.rewind();
   if (tagCompressionContext != null) {
     tagCompressionContext.clear();
   }
   decodeFirst();
   current.setKey(current.keyBuffer, current.memstoreTS);
   previous.invalidate();
 }
Ejemplo n.º 10
0
 /**
  * Compares two ByteBuffs
  *
  * @param buf1 the first ByteBuff
  * @param o1 the offset in the first ByteBuff from where the compare has to happen
  * @param len1 the length in the first ByteBuff upto which the compare has to happen
  * @param buf2 the second ByteBuff
  * @param o2 the offset in the second ByteBuff from where the compare has to happen
  * @param len2 the length in the second ByteBuff upto which the compare has to happen
  * @return Positive if buf1 is bigger than buf2, 0 if they are equal, and negative if buf1 is
  *     smaller than buf2.
  */
 public static int compareTo(ByteBuff buf1, int o1, int len1, ByteBuff buf2, int o2, int len2) {
   if (buf1.hasArray() && buf2.hasArray()) {
     return Bytes.compareTo(
         buf1.array(), buf1.arrayOffset() + o1, len1, buf2.array(), buf2.arrayOffset() + o2, len2);
   }
   int end1 = o1 + len1;
   int end2 = o2 + len2;
   for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) {
     int a = buf1.get(i) & 0xFF;
     int b = buf2.get(j) & 0xFF;
     if (a != b) {
       return a - b;
     }
   }
   return len1 - len2;
 }
Ejemplo n.º 11
0
 @Override
 public ByteBuffer getKeyValueBuffer() {
   ByteBuffer kvBuffer = createKVBuffer();
   kvBuffer.putInt(current.keyLength);
   kvBuffer.putInt(current.valueLength);
   kvBuffer.put(current.keyBuffer, 0, current.keyLength);
   currentBuffer.get(kvBuffer, current.valueOffset, current.valueLength);
   if (current.tagsLength > 0) {
     // Put short as unsigned
     kvBuffer.put((byte) (current.tagsLength >> 8 & 0xff));
     kvBuffer.put((byte) (current.tagsLength & 0xff));
     if (current.tagsOffset != -1) {
       // the offset of the tags bytes in the underlying buffer is marked. So the temp
       // buffer,tagsBuffer was not been used.
       currentBuffer.get(kvBuffer, current.tagsOffset, current.tagsLength);
     } else {
       // When tagsOffset is marked as -1, tag compression was present and so the tags were
       // uncompressed into temp buffer, tagsBuffer. Let us copy it from there
       kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
     }
   }
   kvBuffer.rewind();
   return kvBuffer;
 }
Ejemplo n.º 12
0
 @Override
 public SingleByteBuff put(int offset, ByteBuff src, int srcOffset, int length) {
   if (src instanceof SingleByteBuff) {
     ByteBufferUtils.copyFromBufferToBuffer(
         ((SingleByteBuff) src).buf, this.buf, srcOffset, offset, length);
   } else {
     // TODO we can do some optimization here? Call to asSubByteBuffer might
     // create a copy.
     Pair<ByteBuffer, Integer> pair = new Pair<ByteBuffer, Integer>();
     src.asSubByteBuffer(srcOffset, length, pair);
     ByteBufferUtils.copyFromBufferToBuffer(
         pair.getFirst(), this.buf, pair.getSecond(), offset, length);
   }
   return this;
 }
Ejemplo n.º 13
0
  /**
   * Search sorted array "a" for byte "key".
   *
   * @param a Array to search. Entries must be sorted and unique.
   * @param fromIndex First index inclusive of "a" to include in the search.
   * @param toIndex Last index exclusive of "a" to include in the search.
   * @param key The byte to search for.
   * @return The index of key if found. If not found, return -(index + 1), where negative indicates
   *     "not found" and the "index + 1" handles the "-0" case.
   */
  public static int unsignedBinarySearch(ByteBuff a, int fromIndex, int toIndex, byte key) {
    int unsignedKey = key & 0xff;
    int low = fromIndex;
    int high = toIndex - 1;

    while (low <= high) {
      int mid = (low + high) >>> 1;
      int midVal = a.get(mid) & 0xff;

      if (midVal < unsignedKey) {
        low = mid + 1;
      } else if (midVal > unsignedKey) {
        high = mid - 1;
      } else {
        return mid; // key found
      }
    }
    return -(low + 1); // key not found.
  }
Ejemplo n.º 14
0
    @Override
    public int seekToKeyInBlock(Cell seekCell, boolean seekBefore) {
      int rowCommonPrefix = 0;
      int familyCommonPrefix = 0;
      int qualCommonPrefix = 0;
      previous.invalidate();
      do {
        int comp;
        keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength);
        if (current.lastCommonPrefix != 0) {
          // The KV format has row key length also in the byte array. The
          // common prefix
          // includes it. So we need to subtract to find out the common prefix
          // in the
          // row part alone
          rowCommonPrefix = Math.min(rowCommonPrefix, current.lastCommonPrefix - 2);
        }
        if (current.lastCommonPrefix <= 2) {
          rowCommonPrefix = 0;
        }
        rowCommonPrefix += findCommonPrefixInRowPart(seekCell, keyOnlyKV, rowCommonPrefix);
        comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix);
        if (comp == 0) {
          comp = compareTypeBytes(seekCell, keyOnlyKV);
          if (comp == 0) {
            // Subtract the fixed row key length and the family key fixed length
            familyCommonPrefix =
                Math.max(
                    0,
                    Math.min(
                        familyCommonPrefix,
                        current.lastCommonPrefix - (3 + keyOnlyKV.getRowLength())));
            familyCommonPrefix +=
                findCommonPrefixInFamilyPart(seekCell, keyOnlyKV, familyCommonPrefix);
            comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix);
            if (comp == 0) {
              // subtract the rowkey fixed length and the family key fixed
              // length
              qualCommonPrefix =
                  Math.max(
                      0,
                      Math.min(
                          qualCommonPrefix,
                          current.lastCommonPrefix
                              - (3 + keyOnlyKV.getRowLength() + keyOnlyKV.getFamilyLength())));
              qualCommonPrefix +=
                  findCommonPrefixInQualifierPart(seekCell, keyOnlyKV, qualCommonPrefix);
              comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix);
              if (comp == 0) {
                comp = CellComparator.compareTimestamps(seekCell, keyOnlyKV);
                if (comp == 0) {
                  // Compare types. Let the delete types sort ahead of puts;
                  // i.e. types
                  // of higher numbers sort before those of lesser numbers.
                  // Maximum
                  // (255)
                  // appears ahead of everything, and minimum (0) appears
                  // after
                  // everything.
                  comp = (0xff & keyOnlyKV.getTypeByte()) - (0xff & seekCell.getTypeByte());
                }
              }
            }
          }
        }
        if (comp == 0) { // exact match
          if (seekBefore) {
            if (!previous.isValid()) {
              // The caller (seekBefore) has to ensure that we are not at the
              // first key in the block.
              throw new IllegalStateException(
                  "Cannot seekBefore if "
                      + "positioned at the first key in the block: key="
                      + Bytes.toStringBinary(seekCell.getRowArray()));
            }
            moveToPrevious();
            return 1;
          }
          return 0;
        }

        if (comp < 0) { // already too large, check previous
          if (previous.isValid()) {
            moveToPrevious();
          } else {
            return HConstants.INDEX_KEY_MAGIC; // using optimized index key
          }
          return 1;
        }

        // move to next, if more data is available
        if (currentBuffer.hasRemaining()) {
          previous.copyFromNext(current);
          decodeNext();
          current.setKey(current.keyBuffer, current.memstoreTS);
        } else {
          break;
        }
      } while (true);

      // we hit the end of the block, not an exact match
      return 1;
    }
Ejemplo n.º 15
0
  @Test
  public void testAllocateByteBuffToReadInto() throws Exception {
    int maxBuffersInPool = 10;
    ByteBufferPool pool = new ByteBufferPool(6 * 1024, maxBuffersInPool);
    initPoolWithAllBuffers(pool, maxBuffersInPool);
    ByteBuff buff = null;
    Pair<ByteBuff, CallCleanup> pair;
    // When the request size is less than 1/6th of the pool buffer size. We should use on demand
    // created on heap Buffer
    pair =
        RpcServer.allocateByteBuffToReadInto(pool, RpcServer.getMinSizeForReservoirUse(pool), 200);
    buff = pair.getFirst();
    assertTrue(buff.hasArray());
    assertEquals(maxBuffersInPool, pool.getQueueSize());
    assertNull(pair.getSecond());
    // When the request size is > 1/6th of the pool buffer size.
    pair =
        RpcServer.allocateByteBuffToReadInto(pool, RpcServer.getMinSizeForReservoirUse(pool), 1024);
    buff = pair.getFirst();
    assertFalse(buff.hasArray());
    assertEquals(maxBuffersInPool - 1, pool.getQueueSize());
    assertNotNull(pair.getSecond());
    pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
    assertEquals(maxBuffersInPool, pool.getQueueSize());
    // Request size> pool buffer size
    pair =
        RpcServer.allocateByteBuffToReadInto(
            pool, RpcServer.getMinSizeForReservoirUse(pool), 7 * 1024);
    buff = pair.getFirst();
    assertFalse(buff.hasArray());
    assertTrue(buff instanceof MultiByteBuff);
    ByteBuffer[] bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
    assertEquals(2, bbs.length);
    assertTrue(bbs[0].isDirect());
    assertTrue(bbs[1].isDirect());
    assertEquals(6 * 1024, bbs[0].limit());
    assertEquals(1024, bbs[1].limit());
    assertEquals(maxBuffersInPool - 2, pool.getQueueSize());
    assertNotNull(pair.getSecond());
    pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
    assertEquals(maxBuffersInPool, pool.getQueueSize());

    pair =
        RpcServer.allocateByteBuffToReadInto(
            pool, RpcServer.getMinSizeForReservoirUse(pool), 6 * 1024 + 200);
    buff = pair.getFirst();
    assertFalse(buff.hasArray());
    assertTrue(buff instanceof MultiByteBuff);
    bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
    assertEquals(2, bbs.length);
    assertTrue(bbs[0].isDirect());
    assertFalse(bbs[1].isDirect());
    assertEquals(6 * 1024, bbs[0].limit());
    assertEquals(200, bbs[1].limit());
    assertEquals(maxBuffersInPool - 1, pool.getQueueSize());
    assertNotNull(pair.getSecond());
    pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
    assertEquals(maxBuffersInPool, pool.getQueueSize());

    ByteBuffer[] buffers = new ByteBuffer[maxBuffersInPool - 1];
    for (int i = 0; i < maxBuffersInPool - 1; i++) {
      buffers[i] = pool.getBuffer();
    }
    pair =
        RpcServer.allocateByteBuffToReadInto(
            pool, RpcServer.getMinSizeForReservoirUse(pool), 20 * 1024);
    buff = pair.getFirst();
    assertFalse(buff.hasArray());
    assertTrue(buff instanceof MultiByteBuff);
    bbs = ((MultiByteBuff) buff).getEnclosingByteBuffers();
    assertEquals(2, bbs.length);
    assertTrue(bbs[0].isDirect());
    assertFalse(bbs[1].isDirect());
    assertEquals(6 * 1024, bbs[0].limit());
    assertEquals(14 * 1024, bbs[1].limit());
    assertEquals(0, pool.getQueueSize());
    assertNotNull(pair.getSecond());
    pair.getSecond().run(); // CallCleanup#run should put back the BB to pool.
    assertEquals(1, pool.getQueueSize());
    pool.getBuffer();
    pair =
        RpcServer.allocateByteBuffToReadInto(
            pool, RpcServer.getMinSizeForReservoirUse(pool), 7 * 1024);
    buff = pair.getFirst();
    assertTrue(buff.hasArray());
    assertTrue(buff instanceof SingleByteBuff);
    assertEquals(7 * 1024, ((SingleByteBuff) buff).getEnclosingByteBuffer().limit());
    assertNull(pair.getSecond());
  }