protected final void afterDecodingKeyValue(
     DataInputStream source, ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx)
     throws IOException {
   if (decodingCtx.getHFileContext().isIncludesTags()) {
     int tagsLength = ByteBufferUtils.readCompressedInt(source);
     // Put as unsigned short
     dest.put((byte) ((tagsLength >> 8) & 0xff));
     dest.put((byte) (tagsLength & 0xff));
     if (tagsLength > 0) {
       TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
       // When tag compression is been used in this file, tagCompressionContext will have a not
       // null value passed.
       if (tagCompressionContext != null) {
         tagCompressionContext.uncompressTags(source, dest, tagsLength);
       } else {
         ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
       }
     }
   }
   if (decodingCtx.getHFileContext().isIncludesMvcc()) {
     long memstoreTS = -1;
     try {
       // Copy memstore timestamp from the data input stream to the byte
       // buffer.
       memstoreTS = WritableUtils.readVLong(source);
       ByteBufferUtils.writeVLong(dest, memstoreTS);
     } catch (IOException ex) {
       throw new RuntimeException(
           "Unable to copy memstore timestamp " + memstoreTS + " after decoding a key/value");
     }
   }
 }
예제 #2
0
 @Test
 public void testGetKeyMethods() throws Exception {
   KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
   ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getKeyLength());
   ByteBufferUtils.copyFromArrayToBuffer(
       buf, kvCell.getBuffer(), kvCell.getKeyOffset(), kvCell.getKeyLength());
   ByteBufferedCell offheapKeyOnlyKV = new ByteBufferedKeyOnlyKeyValue(buf, 0, buf.capacity());
   assertEquals(
       ROW1,
       ByteBufferUtils.toStringBinary(
           offheapKeyOnlyKV.getRowByteBuffer(),
           offheapKeyOnlyKV.getRowPositionInByteBuffer(),
           offheapKeyOnlyKV.getRowLength()));
   assertEquals(
       FAM1,
       ByteBufferUtils.toStringBinary(
           offheapKeyOnlyKV.getFamilyByteBuffer(),
           offheapKeyOnlyKV.getFamilyPositionInByteBuffer(),
           offheapKeyOnlyKV.getFamilyLength()));
   assertEquals(
       QUAL1,
       ByteBufferUtils.toStringBinary(
           offheapKeyOnlyKV.getQualifierByteBuffer(),
           offheapKeyOnlyKV.getQualifierPositionInByteBuffer(),
           offheapKeyOnlyKV.getQualifierLength()));
   assertEquals(0L, offheapKeyOnlyKV.getTimestamp());
   assertEquals(Type.Put.getCode(), offheapKeyOnlyKV.getTypeByte());
 }
예제 #3
0
 @Override
 public SingleByteBuff put(int offset, ByteBuff src, int srcOffset, int length) {
   if (src instanceof SingleByteBuff) {
     ByteBufferUtils.copyFromBufferToBuffer(
         ((SingleByteBuff) src).buf, this.buf, srcOffset, offset, length);
   } else {
     // TODO we can do some optimization here? Call to asSubByteBuffer might
     // create a copy.
     Pair<ByteBuffer, Integer> pair = new Pair<ByteBuffer, Integer>();
     src.asSubByteBuffer(srcOffset, length, pair);
     ByteBufferUtils.copyFromBufferToBuffer(
         pair.getFirst(), this.buf, pair.getSecond(), offset, length);
   }
   return this;
 }
 /**
  * @param cell
  * @param out
  * @param encodingCtx
  * @return unencoded size added
  * @throws IOException
  */
 protected final int afterEncodingKeyValue(
     Cell cell, DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx)
     throws IOException {
   int size = 0;
   if (encodingCtx.getHFileContext().isIncludesTags()) {
     int tagsLength = cell.getTagsLength();
     ByteBufferUtils.putCompressedInt(out, tagsLength);
     // There are some tags to be written
     if (tagsLength > 0) {
       TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
       // When tag compression is enabled, tagCompressionContext will have a not null value. Write
       // the tags using Dictionary compression in such a case
       if (tagCompressionContext != null) {
         tagCompressionContext.compressTags(
             out, cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
       } else {
         out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
       }
     }
     size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
   }
   if (encodingCtx.getHFileContext().isIncludesMvcc()) {
     // Copy memstore timestamp from the byte buffer to the output stream.
     long memstoreTS = cell.getSequenceId();
     WritableUtils.writeVLong(out, memstoreTS);
     // TODO use a writeVLong which returns the #bytes written so that 2 time parsing can be
     // avoided.
     size += WritableUtils.getVIntSize(memstoreTS);
   }
   return size;
 }
예제 #5
0
 /**
  * Special compareTo method for subclasses, to avoid copying bytes unnecessarily.
  *
  * @param value bytes to compare within a ByteBuffer
  * @param offset offset into value
  * @param length number of bytes to compare
  * @return a negative integer, zero, or a positive integer as this object is less than, equal to,
  *     or greater than the specified object.
  */
 public int compareTo(ByteBuffer value, int offset, int length) {
   // For BC, providing a default implementation here which is doing a bytes copy to a temp byte[]
   // and calling compareTo(byte[]). Make sure to override this method in subclasses to avoid
   // copying bytes unnecessarily.
   byte[] temp = new byte[length];
   ByteBufferUtils.copyFromBufferToArray(temp, value, offset, 0, length);
   return compareTo(temp);
 }
예제 #6
0
 @Test
 public void testByteBufferBackedKeyValueWithTags() throws Exception {
   KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
   ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
   ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
   ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), true, 0l);
   assertEquals(
       ROW1,
       ByteBufferUtils.toStringBinary(
           offheapKV.getRowByteBuffer(),
           offheapKV.getRowPositionInByteBuffer(),
           offheapKV.getRowLength()));
   assertEquals(
       FAM1,
       ByteBufferUtils.toStringBinary(
           offheapKV.getFamilyByteBuffer(),
           offheapKV.getFamilyPositionInByteBuffer(),
           offheapKV.getFamilyLength()));
   assertEquals(
       QUAL1,
       ByteBufferUtils.toStringBinary(
           offheapKV.getQualifierByteBuffer(),
           offheapKV.getQualifierPositionInByteBuffer(),
           offheapKV.getQualifierLength()));
   assertEquals(
       ROW1,
       ByteBufferUtils.toStringBinary(
           offheapKV.getValueByteBuffer(),
           offheapKV.getValuePositionInByteBuffer(),
           offheapKV.getValueLength()));
   assertEquals(0L, offheapKV.getTimestamp());
   assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
   // change tags to handle both onheap and offheap stuff
   List<Tag> resTags =
       Tag.asList(offheapKV.getTagsArray(), offheapKV.getTagsOffset(), offheapKV.getTagsLength());
   Tag tag1 = resTags.get(0);
   assertEquals(t1.getType(), tag1.getType());
   assertEquals(Bytes.toString(t1.getValue()), Bytes.toString(getTagValue(tag1)));
   Tag tag2 = resTags.get(1);
   assertEquals(tag2.getType(), tag2.getType());
   assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
   Tag res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 2);
   assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
   res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 3);
   assertNull(res);
 }
예제 #7
0
 /**
  * Do the encoding, but do not cache the encoded data.
  *
  * @return encoded data block with header and checksum
  */
 public byte[] encodeData() {
   ByteArrayOutputStream baos = new ByteArrayOutputStream();
   try {
     baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
     DataOutputStream out = new DataOutputStream(baos);
     this.dataBlockEncoder.startBlockEncoding(encodingCtx, out);
     ByteBuffer in = getUncompressedBuffer();
     in.rewind();
     int klength, vlength;
     int tagsLength = 0;
     long memstoreTS = 0L;
     KeyValue kv = null;
     while (in.hasRemaining()) {
       int kvOffset = in.position();
       klength = in.getInt();
       vlength = in.getInt();
       ByteBufferUtils.skip(in, klength + vlength);
       if (this.meta.isIncludesTags()) {
         tagsLength = ((in.get() & 0xff) << 8) ^ (in.get() & 0xff);
         ByteBufferUtils.skip(in, tagsLength);
       }
       if (this.meta.isIncludesMvcc()) {
         memstoreTS = ByteBufferUtils.readVLong(in);
       }
       kv =
           new KeyValue(
               in.array(),
               kvOffset,
               (int) KeyValue.getKeyValueDataStructureSize(klength, vlength, tagsLength));
       kv.setSequenceId(memstoreTS);
       this.dataBlockEncoder.encode(kv, encodingCtx, out);
     }
     BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
     baos.writeTo(stream);
     this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, stream.buf);
   } catch (IOException e) {
     throw new RuntimeException(
         String.format(
             "Bug in encoding part of algorithm %s. "
                 + "Probably it requested more bytes than are available.",
             toString()),
         e);
   }
   return baos.toByteArray();
 }
 @Override
 public int write(OutputStream out, boolean withTags) throws IOException {
   int lenToWrite =
       KeyValueUtil.length(
           rowLength, familyLength, qualifierLength, valueLength, tagsLength, withTags);
   ByteBufferUtils.putInt(out, lenToWrite);
   ByteBufferUtils.putInt(out, keyBuffer.capacity());
   ByteBufferUtils.putInt(out, valueLength);
   // Write key
   out.write(keyBuffer.array());
   // Write value
   ByteBufferUtils.writeByteBuffer(out, this.valueBuffer, this.valueOffset, this.valueLength);
   if (withTags) {
     // 2 bytes tags length followed by tags bytes
     // tags length is serialized with 2 bytes only(short way) even if the type is int.
     // As this is non -ve numbers, we save the sign bit. See HBASE-11437
     out.write((byte) (0xff & (this.tagsLength >> 8)));
     out.write((byte) (0xff & this.tagsLength));
     ByteBufferUtils.writeByteBuffer(out, this.tagsBuffer, this.tagsOffset, this.tagsLength);
   }
   return lenToWrite + Bytes.SIZEOF_INT;
 }
예제 #9
0
  /**
   * Convert list of KeyValues to byte buffer.
   *
   * @param keyValues list of KeyValues to be converted.
   * @return buffer with content from key values
   */
  public static ByteBuffer convertKvToByteBuffer(
      List<KeyValue> keyValues, boolean includesMemstoreTS) {
    int totalSize = 0;
    for (KeyValue kv : keyValues) {
      totalSize += kv.getLength();
      if (includesMemstoreTS) {
        totalSize += WritableUtils.getVIntSize(kv.getMvccVersion());
      }
    }

    ByteBuffer result = ByteBuffer.allocate(totalSize);
    for (KeyValue kv : keyValues) {
      result.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
      if (includesMemstoreTS) {
        ByteBufferUtils.writeVLong(result, kv.getMvccVersion());
      }
    }
    return result;
  }
예제 #10
0
 @Override
 public byte[] toBytes(int offset, int length) {
   byte[] output = new byte[length];
   ByteBufferUtils.copyFromBufferToArray(output, buf, offset, 0, length);
   return output;
 }
예제 #11
0
 @Override
 public long getLongAfterPosition(int offset) {
   return ByteBufferUtils.toLong(this.buf, this.buf.position() + offset);
 }
예제 #12
0
 @Override
 public long getLong(int index) {
   return ByteBufferUtils.toLong(this.buf, index);
 }
예제 #13
0
 @Override
 public SingleByteBuff putLong(long value) {
   ByteBufferUtils.putLong(this.buf, value);
   return this;
 }
예제 #14
0
 @Override
 public int getIntAfterPosition(int offset) {
   return ByteBufferUtils.toInt(this.buf, this.buf.position() + offset);
 }
예제 #15
0
  @Test
  public void testByteBufferBackedKeyValue() throws Exception {
    KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1);
    ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
    ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
    ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
    assertEquals(
        ROW1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getRowByteBuffer(),
            offheapKV.getRowPositionInByteBuffer(),
            offheapKV.getRowLength()));
    assertEquals(
        FAM1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getFamilyByteBuffer(),
            offheapKV.getFamilyPositionInByteBuffer(),
            offheapKV.getFamilyLength()));
    assertEquals(
        QUAL1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getQualifierByteBuffer(),
            offheapKV.getQualifierPositionInByteBuffer(),
            offheapKV.getQualifierLength()));
    assertEquals(
        ROW1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getValueByteBuffer(),
            offheapKV.getValuePositionInByteBuffer(),
            offheapKV.getValueLength()));
    assertEquals(0L, offheapKV.getTimestamp());
    assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());

    // Use the array() APIs
    assertEquals(
        ROW1,
        Bytes.toStringBinary(
            offheapKV.getRowArray(), offheapKV.getRowOffset(), offheapKV.getRowLength()));
    assertEquals(
        FAM1,
        Bytes.toStringBinary(
            offheapKV.getFamilyArray(), offheapKV.getFamilyOffset(), offheapKV.getFamilyLength()));
    assertEquals(
        QUAL1,
        Bytes.toStringBinary(
            offheapKV.getQualifierArray(),
            offheapKV.getQualifierOffset(),
            offheapKV.getQualifierLength()));
    assertEquals(
        ROW1,
        Bytes.toStringBinary(
            offheapKV.getValueArray(), offheapKV.getValueOffset(), offheapKV.getValueLength()));
    assertEquals(0L, offheapKV.getTimestamp());
    assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());

    kvCell = new KeyValue(row1, fam2, qual2, 0l, Type.Put, row1);
    buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
    ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
    offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
    assertEquals(
        FAM2,
        ByteBufferUtils.toStringBinary(
            offheapKV.getFamilyByteBuffer(),
            offheapKV.getFamilyPositionInByteBuffer(),
            offheapKV.getFamilyLength()));
    assertEquals(
        QUAL2,
        ByteBufferUtils.toStringBinary(
            offheapKV.getQualifierByteBuffer(),
            offheapKV.getQualifierPositionInByteBuffer(),
            offheapKV.getQualifierLength()));
    byte[] nullQualifier = new byte[0];
    kvCell = new KeyValue(row1, fam1, nullQualifier, 0L, Type.Put, row1);
    buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
    ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
    offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
    assertEquals(
        ROW1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getRowByteBuffer(),
            offheapKV.getRowPositionInByteBuffer(),
            offheapKV.getRowLength()));
    assertEquals(
        FAM1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getFamilyByteBuffer(),
            offheapKV.getFamilyPositionInByteBuffer(),
            offheapKV.getFamilyLength()));
    assertEquals(
        "",
        ByteBufferUtils.toStringBinary(
            offheapKV.getQualifierByteBuffer(),
            offheapKV.getQualifierPositionInByteBuffer(),
            offheapKV.getQualifierLength()));
    assertEquals(
        ROW1,
        ByteBufferUtils.toStringBinary(
            offheapKV.getValueByteBuffer(),
            offheapKV.getValuePositionInByteBuffer(),
            offheapKV.getValueLength()));
    assertEquals(0L, offheapKV.getTimestamp());
    assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
  }
 @Override
 public long getTimestamp() {
   return ByteBufferUtils.toLong(this.buf, getTimestampOffset());
 }
예제 #17
0
 @Override
 public short getShortAfterPosition(int offset) {
   return ByteBufferUtils.toShort(this.buf, this.buf.position() + offset);
 }
예제 #18
0
 @Override
 public short getShort(int index) {
   return ByteBufferUtils.toShort(this.buf, index);
 }
 private byte getFamilyLength(int famLenPos) {
   return ByteBufferUtils.toByte(this.buf, famLenPos);
 }
 /**
  * A setter that helps to avoid object creation every time and whenever there is a need to create
  * new OffheapKeyOnlyKeyValue.
  *
  * @param key
  * @param offset
  * @param length
  */
 public void setKey(ByteBuffer key, int offset, int length) {
   this.buf = key;
   this.offset = offset;
   this.length = length;
   this.rowLen = ByteBufferUtils.toShort(this.buf, this.offset);
 }
예제 #21
0
 @Override
 public void get(byte[] dst, int offset, int length) {
   ByteBufferUtils.copyFromBufferToArray(dst, buf, buf.position(), offset, length);
   buf.position(buf.position() + length);
 }
예제 #22
0
 @Override
 public byte getByteAfterPosition(int offset) {
   return ByteBufferUtils.toByte(this.buf, this.buf.position() + offset);
 }
예제 #23
0
 @Override
 public byte get(int index) {
   return ByteBufferUtils.toByte(this.buf, index);
 }
예제 #24
0
 @Override
 public void get(ByteBuffer out, int sourceOffset, int length) {
   ByteBufferUtils.copyFromBufferToBuffer(buf, out, sourceOffset, length);
 }
 @Override
 public byte getTypeByte() {
   return ByteBufferUtils.toByte(this.buf, this.offset + this.length - 1);
 }
예제 #26
0
 @Override
 public SingleByteBuff putInt(int value) {
   ByteBufferUtils.putInt(this.buf, value);
   return this;
 }
예제 #27
0
 @Override
 public SingleByteBuff put(byte[] src, int offset, int length) {
   ByteBufferUtils.copyFromArrayToBuffer(this.buf, src, offset, length);
   return this;
 }
예제 #28
0
 @Override
 public int getInt(int index) {
   return ByteBufferUtils.toInt(this.buf, index);
 }