Exemplo n.º 1
0
  /**
   * Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on
   * construction.
   *
   * @param kv KeyValue to add. Cannot be empty nor null.
   * @throws IOException
   */
  @Override
  public void append(final KeyValue kv) throws IOException {
    byte[] key = kv.getBuffer();
    int koffset = kv.getKeyOffset();
    int klength = kv.getKeyLength();
    byte[] value = kv.getValueArray();
    int voffset = kv.getValueOffset();
    int vlength = kv.getValueLength();
    boolean dupKey = checkKey(key, koffset, klength);
    checkValue(value, voffset, vlength);
    if (!dupKey) {
      checkBlockBoundary();
    }

    if (!fsBlockWriter.isWriting()) newBlock();

    fsBlockWriter.write(kv);

    totalKeyLength += klength;
    totalValueLength += vlength;

    // Are we the first key in this block?
    if (firstKeyInBlock == null) {
      // Copy the key.
      firstKeyInBlock = new byte[klength];
      System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
    }

    lastKeyBuffer = key;
    lastKeyOffset = koffset;
    lastKeyLength = klength;
    entryCount++;
    this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion());
  }
Exemplo n.º 2
0
  /**
   * At a block boundary, write all the inline blocks and opens new block.
   *
   * @throws IOException
   */
  private void checkBlockBoundary() throws IOException {
    if (fsBlockWriter.blockSizeWritten() < blockSize) return;

    finishBlock();
    writeInlineBlocks(false);
    newBlock();
  }
Exemplo n.º 3
0
  /**
   * At a block boundary, write all the inline blocks and opens new block.
   *
   * @throws IOException
   */
  protected void checkBlockBoundary() throws IOException {
    if (fsBlockWriter.blockSizeWritten() < hFileContext.getBlocksize()) return;

    finishBlock();
    writeInlineBlocks(false);
    newBlock();
  }
Exemplo n.º 4
0
 /**
  * Ready a new block for writing.
  *
  * @throws IOException
  */
 private void newBlock() throws IOException {
   // This is where the next block begins.
   fsBlockWriter.startWriting(BlockType.DATA);
   firstKeyInBlock = null;
   if (lastKeyLength > 0) {
     lastKeyOfPreviousBlock = new byte[lastKeyLength];
     System.arraycopy(lastKeyBuffer, lastKeyOffset, lastKeyOfPreviousBlock, 0, lastKeyLength);
   }
 }
Exemplo n.º 5
0
  /**
   * Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on
   * construction.
   *
   * @param key
   * @param koffset
   * @param klength
   * @param value
   * @param voffset
   * @param vlength
   * @throws IOException
   */
  private void append(
      final long memstoreTS,
      final byte[] key,
      final int koffset,
      final int klength,
      final byte[] value,
      final int voffset,
      final int vlength)
      throws IOException {
    boolean dupKey = checkKey(key, koffset, klength);
    checkValue(value, voffset, vlength);
    if (!dupKey) {
      checkBlockBoundary();
    }

    if (!fsBlockWriter.isWriting()) newBlock();

    // Write length of key and value and then actual key and value bytes.
    // Additionally, we may also write down the memstoreTS.
    {
      DataOutputStream out = fsBlockWriter.getUserDataStream();
      out.writeInt(klength);
      totalKeyLength += klength;
      out.writeInt(vlength);
      totalValueLength += vlength;
      out.write(key, koffset, klength);
      out.write(value, voffset, vlength);
      if (this.includeMemstoreTS) {
        WritableUtils.writeVLong(out, memstoreTS);
      }
    }

    // Are we the first key in this block?
    if (firstKeyInBlock == null) {
      // Copy the key.
      firstKeyInBlock = new byte[klength];
      System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
    }

    lastKeyBuffer = key;
    lastKeyOffset = koffset;
    lastKeyLength = klength;
    entryCount++;
  }
Exemplo n.º 6
0
  /** Clean up the current block */
  private void finishBlock() throws IOException {
    if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return;

    long startTimeNs = System.nanoTime();
    // Update the first data block offset for scanning.
    if (firstDataBlockOffset == -1) {
      firstDataBlockOffset = outputStream.getPos();
    }
    // Update the last data block offset
    lastDataBlockOffset = outputStream.getPos();
    fsBlockWriter.writeHeaderAndData(outputStream);
    int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
    // Generate a shorter faked key into index block. For example, consider a block boundary
    // between the keys "the quick brown fox" and "the who test text".  We can use "the r" as the
    // key for the index block entry since it is > all entries in the previous block and <= all
    // entries in subsequent blocks.
    if (comparator instanceof KeyComparator) {
      byte[] fakeKey =
          ((KeyComparator) comparator).getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock);
      if (comparator.compare(fakeKey, firstKeyInBlock) > 0) {
        throw new IOException(
            "Unexpected getShortMidpointKey result, fakeKey:"
                + Bytes.toStringBinary(fakeKey)
                + ", firstKeyInBlock:"
                + Bytes.toStringBinary(firstKeyInBlock));
      }
      if (lastKeyOfPreviousBlock != null
          && comparator.compare(lastKeyOfPreviousBlock, fakeKey) >= 0) {
        throw new IOException(
            "Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:"
                + Bytes.toStringBinary(lastKeyOfPreviousBlock)
                + ", fakeKey:"
                + Bytes.toStringBinary(fakeKey));
      }
      dataBlockIndexWriter.addEntry(fakeKey, lastDataBlockOffset, onDiskSize);
    } else {
      dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset, onDiskSize);
    }
    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
    HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
    if (cacheConf.shouldCacheDataOnWrite()) {
      doCacheOnWrite(lastDataBlockOffset);
    }
  }
Exemplo n.º 7
0
  /** Gives inline block writers an opportunity to contribute blocks. */
  private void writeInlineBlocks(boolean closing) throws IOException {
    for (InlineBlockWriter ibw : inlineBlockWriters) {
      while (ibw.shouldWriteBlock(closing)) {
        long offset = outputStream.getPos();
        boolean cacheThisBlock = ibw.getCacheOnWrite();
        ibw.writeInlineBlock(fsBlockWriter.startWriting(ibw.getInlineBlockType()));
        fsBlockWriter.writeHeaderAndData(outputStream);
        ibw.blockWritten(
            offset,
            fsBlockWriter.getOnDiskSizeWithHeader(),
            fsBlockWriter.getUncompressedSizeWithoutHeader());
        totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();

        if (cacheThisBlock) {
          doCacheOnWrite(offset);
        }
      }
    }
  }
Exemplo n.º 8
0
  /** Clean up the current block */
  private void finishBlock() throws IOException {
    if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return;

    // Update the first data block offset for scanning.
    if (firstDataBlockOffset == -1) {
      firstDataBlockOffset = outputStream.getPos();
    }
    // Update the last data block offset
    lastDataBlockOffset = outputStream.getPos();
    fsBlockWriter.writeHeaderAndData(outputStream);
    int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();

    byte[] indexKey = comparator.calcIndexKey(lastKeyOfPreviousBlock, firstKeyInBlock);
    dataBlockIndexWriter.addEntry(indexKey, lastDataBlockOffset, onDiskSize);
    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
    if (cacheConf.shouldCacheDataOnWrite()) {
      doCacheOnWrite(lastDataBlockOffset);
    }
  }
Exemplo n.º 9
0
 /**
  * Caches the last written HFile block.
  *
  * @param offset the offset of the block we want to cache. Used to determine the cache key.
  */
 private void doCacheOnWrite(long offset) {
   // We don't cache-on-write data blocks on compaction, so assume this is not
   // a compaction.
   final boolean isCompaction = false;
   HFileBlock cacheFormatBlock =
       blockEncoder.diskToCacheFormat(fsBlockWriter.getBlockForCaching(), isCompaction);
   cacheConf
       .getBlockCache()
       .cacheBlock(
           new BlockCacheKey(
               name, offset, blockEncoder.getEncodingInCache(), cacheFormatBlock.getBlockType()),
           cacheFormatBlock);
 }
Exemplo n.º 10
0
 private int writeBlock(FSDataOutputStream os, HFileContext fileContext, int size)
     throws IOException {
   HFileBlock.Writer hbw = new HFileBlock.Writer(null, fileContext);
   DataOutputStream dos = hbw.startWriting(BlockType.DATA);
   for (int j = 0; j < size; j++) {
     dos.writeInt(j);
   }
   hbw.writeHeaderAndData(os);
   LOG.info(
       "Wrote a block at "
           + os.getPos()
           + " with"
           + " onDiskSizeWithHeader="
           + hbw.getOnDiskSizeWithHeader()
           + " uncompressedSizeWithoutHeader="
           + hbw.getOnDiskSizeWithoutHeader()
           + " uncompressedSizeWithoutHeader="
           + hbw.getUncompressedSizeWithoutHeader());
   return hbw.getOnDiskSizeWithHeader();
 }
Exemplo n.º 11
0
  @Override
  public void close() throws IOException {
    if (outputStream == null) {
      return;
    }
    // Save data block encoder metadata in the file info.
    blockEncoder.saveMetadata(this);
    // Write out the end of the data blocks, then write meta data blocks.
    // followed by fileinfo, data block index and meta block index.

    finishBlock();
    writeInlineBlocks(true);

    FixedFileTrailer trailer = new FixedFileTrailer(2, HFileReaderV2.MAX_MINOR_VERSION);

    // Write out the metadata blocks if any.
    if (!metaNames.isEmpty()) {
      for (int i = 0; i < metaNames.size(); ++i) {
        // store the beginning offset
        long offset = outputStream.getPos();
        // write the metadata content
        DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META);
        metaData.get(i).write(dos);

        fsBlockWriter.writeHeaderAndData(outputStream);
        totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();

        // Add the new meta block to the meta index.
        metaBlockIndexWriter.addEntry(
            metaNames.get(i), offset, fsBlockWriter.getOnDiskSizeWithHeader());
      }
    }

    // Load-on-open section.

    // Data block index.
    //
    // In version 2, this section of the file starts with the root level data
    // block index. We call a function that writes intermediate-level blocks
    // first, then root level, and returns the offset of the root level block
    // index.

    long rootIndexOffset = dataBlockIndexWriter.writeIndexBlocks(outputStream);
    trailer.setLoadOnOpenOffset(rootIndexOffset);

    // Meta block index.
    metaBlockIndexWriter.writeSingleLevelIndex(
        fsBlockWriter.startWriting(BlockType.ROOT_INDEX), "meta");
    fsBlockWriter.writeHeaderAndData(outputStream);
    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();

    if (this.includeMemstoreTS) {
      appendFileInfo(MAX_MEMSTORE_TS_KEY, Bytes.toBytes(maxMemstoreTS));
      appendFileInfo(KEY_VALUE_VERSION, Bytes.toBytes(KEY_VALUE_VER_WITH_MEMSTORE));
    }

    // File info
    writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO));
    fsBlockWriter.writeHeaderAndData(outputStream);
    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();

    // Load-on-open data supplied by higher levels, e.g. Bloom filters.
    for (BlockWritable w : additionalLoadOnOpenData) {
      fsBlockWriter.writeBlock(w, outputStream);
      totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
    }

    // Now finish off the trailer.
    trailer.setNumDataIndexLevels(dataBlockIndexWriter.getNumLevels());
    trailer.setUncompressedDataIndexSize(dataBlockIndexWriter.getTotalUncompressedSize());
    trailer.setFirstDataBlockOffset(firstDataBlockOffset);
    trailer.setLastDataBlockOffset(lastDataBlockOffset);
    trailer.setComparatorClass(comparator.getClass());
    trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());

    finishClose(trailer);

    fsBlockWriter.release();
  }
Exemplo n.º 12
0
 /**
  * Caches the last written HFile block.
  *
  * @param offset the offset of the block we want to cache. Used to determine the cache key.
  */
 private void doCacheOnWrite(long offset) {
   HFileBlock cacheFormatBlock = fsBlockWriter.getBlockForCaching();
   cacheConf.getBlockCache().cacheBlock(new BlockCacheKey(name, offset), cacheFormatBlock);
 }