/** * Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on * construction. * * @param kv KeyValue to add. Cannot be empty nor null. * @throws IOException */ @Override public void append(final KeyValue kv) throws IOException { byte[] key = kv.getBuffer(); int koffset = kv.getKeyOffset(); int klength = kv.getKeyLength(); byte[] value = kv.getValueArray(); int voffset = kv.getValueOffset(); int vlength = kv.getValueLength(); boolean dupKey = checkKey(key, koffset, klength); checkValue(value, voffset, vlength); if (!dupKey) { checkBlockBoundary(); } if (!fsBlockWriter.isWriting()) newBlock(); fsBlockWriter.write(kv); totalKeyLength += klength; totalValueLength += vlength; // Are we the first key in this block? if (firstKeyInBlock == null) { // Copy the key. firstKeyInBlock = new byte[klength]; System.arraycopy(key, koffset, firstKeyInBlock, 0, klength); } lastKeyBuffer = key; lastKeyOffset = koffset; lastKeyLength = klength; entryCount++; this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion()); }
/** * At a block boundary, write all the inline blocks and opens new block. * * @throws IOException */ private void checkBlockBoundary() throws IOException { if (fsBlockWriter.blockSizeWritten() < blockSize) return; finishBlock(); writeInlineBlocks(false); newBlock(); }
/** * At a block boundary, write all the inline blocks and opens new block. * * @throws IOException */ protected void checkBlockBoundary() throws IOException { if (fsBlockWriter.blockSizeWritten() < hFileContext.getBlocksize()) return; finishBlock(); writeInlineBlocks(false); newBlock(); }
/** * Ready a new block for writing. * * @throws IOException */ private void newBlock() throws IOException { // This is where the next block begins. fsBlockWriter.startWriting(BlockType.DATA); firstKeyInBlock = null; if (lastKeyLength > 0) { lastKeyOfPreviousBlock = new byte[lastKeyLength]; System.arraycopy(lastKeyBuffer, lastKeyOffset, lastKeyOfPreviousBlock, 0, lastKeyLength); } }
/** * Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on * construction. * * @param key * @param koffset * @param klength * @param value * @param voffset * @param vlength * @throws IOException */ private void append( final long memstoreTS, final byte[] key, final int koffset, final int klength, final byte[] value, final int voffset, final int vlength) throws IOException { boolean dupKey = checkKey(key, koffset, klength); checkValue(value, voffset, vlength); if (!dupKey) { checkBlockBoundary(); } if (!fsBlockWriter.isWriting()) newBlock(); // Write length of key and value and then actual key and value bytes. // Additionally, we may also write down the memstoreTS. { DataOutputStream out = fsBlockWriter.getUserDataStream(); out.writeInt(klength); totalKeyLength += klength; out.writeInt(vlength); totalValueLength += vlength; out.write(key, koffset, klength); out.write(value, voffset, vlength); if (this.includeMemstoreTS) { WritableUtils.writeVLong(out, memstoreTS); } } // Are we the first key in this block? if (firstKeyInBlock == null) { // Copy the key. firstKeyInBlock = new byte[klength]; System.arraycopy(key, koffset, firstKeyInBlock, 0, klength); } lastKeyBuffer = key; lastKeyOffset = koffset; lastKeyLength = klength; entryCount++; }
/** Clean up the current block */ private void finishBlock() throws IOException { if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return; long startTimeNs = System.nanoTime(); // Update the first data block offset for scanning. if (firstDataBlockOffset == -1) { firstDataBlockOffset = outputStream.getPos(); } // Update the last data block offset lastDataBlockOffset = outputStream.getPos(); fsBlockWriter.writeHeaderAndData(outputStream); int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader(); // Generate a shorter faked key into index block. For example, consider a block boundary // between the keys "the quick brown fox" and "the who test text". We can use "the r" as the // key for the index block entry since it is > all entries in the previous block and <= all // entries in subsequent blocks. if (comparator instanceof KeyComparator) { byte[] fakeKey = ((KeyComparator) comparator).getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock); if (comparator.compare(fakeKey, firstKeyInBlock) > 0) { throw new IOException( "Unexpected getShortMidpointKey result, fakeKey:" + Bytes.toStringBinary(fakeKey) + ", firstKeyInBlock:" + Bytes.toStringBinary(firstKeyInBlock)); } if (lastKeyOfPreviousBlock != null && comparator.compare(lastKeyOfPreviousBlock, fakeKey) >= 0) { throw new IOException( "Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:" + Bytes.toStringBinary(lastKeyOfPreviousBlock) + ", fakeKey:" + Bytes.toStringBinary(fakeKey)); } dataBlockIndexWriter.addEntry(fakeKey, lastDataBlockOffset, onDiskSize); } else { dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset, onDiskSize); } totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); HFile.offerWriteLatency(System.nanoTime() - startTimeNs); if (cacheConf.shouldCacheDataOnWrite()) { doCacheOnWrite(lastDataBlockOffset); } }
/** Gives inline block writers an opportunity to contribute blocks. */ private void writeInlineBlocks(boolean closing) throws IOException { for (InlineBlockWriter ibw : inlineBlockWriters) { while (ibw.shouldWriteBlock(closing)) { long offset = outputStream.getPos(); boolean cacheThisBlock = ibw.getCacheOnWrite(); ibw.writeInlineBlock(fsBlockWriter.startWriting(ibw.getInlineBlockType())); fsBlockWriter.writeHeaderAndData(outputStream); ibw.blockWritten( offset, fsBlockWriter.getOnDiskSizeWithHeader(), fsBlockWriter.getUncompressedSizeWithoutHeader()); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (cacheThisBlock) { doCacheOnWrite(offset); } } } }
/** Clean up the current block */ private void finishBlock() throws IOException { if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return; // Update the first data block offset for scanning. if (firstDataBlockOffset == -1) { firstDataBlockOffset = outputStream.getPos(); } // Update the last data block offset lastDataBlockOffset = outputStream.getPos(); fsBlockWriter.writeHeaderAndData(outputStream); int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader(); byte[] indexKey = comparator.calcIndexKey(lastKeyOfPreviousBlock, firstKeyInBlock); dataBlockIndexWriter.addEntry(indexKey, lastDataBlockOffset, onDiskSize); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (cacheConf.shouldCacheDataOnWrite()) { doCacheOnWrite(lastDataBlockOffset); } }
/** * Caches the last written HFile block. * * @param offset the offset of the block we want to cache. Used to determine the cache key. */ private void doCacheOnWrite(long offset) { // We don't cache-on-write data blocks on compaction, so assume this is not // a compaction. final boolean isCompaction = false; HFileBlock cacheFormatBlock = blockEncoder.diskToCacheFormat(fsBlockWriter.getBlockForCaching(), isCompaction); cacheConf .getBlockCache() .cacheBlock( new BlockCacheKey( name, offset, blockEncoder.getEncodingInCache(), cacheFormatBlock.getBlockType()), cacheFormatBlock); }
private int writeBlock(FSDataOutputStream os, HFileContext fileContext, int size) throws IOException { HFileBlock.Writer hbw = new HFileBlock.Writer(null, fileContext); DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int j = 0; j < size; j++) { dos.writeInt(j); } hbw.writeHeaderAndData(os); LOG.info( "Wrote a block at " + os.getPos() + " with" + " onDiskSizeWithHeader=" + hbw.getOnDiskSizeWithHeader() + " uncompressedSizeWithoutHeader=" + hbw.getOnDiskSizeWithoutHeader() + " uncompressedSizeWithoutHeader=" + hbw.getUncompressedSizeWithoutHeader()); return hbw.getOnDiskSizeWithHeader(); }
@Override public void close() throws IOException { if (outputStream == null) { return; } // Save data block encoder metadata in the file info. blockEncoder.saveMetadata(this); // Write out the end of the data blocks, then write meta data blocks. // followed by fileinfo, data block index and meta block index. finishBlock(); writeInlineBlocks(true); FixedFileTrailer trailer = new FixedFileTrailer(2, HFileReaderV2.MAX_MINOR_VERSION); // Write out the metadata blocks if any. if (!metaNames.isEmpty()) { for (int i = 0; i < metaNames.size(); ++i) { // store the beginning offset long offset = outputStream.getPos(); // write the metadata content DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META); metaData.get(i).write(dos); fsBlockWriter.writeHeaderAndData(outputStream); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); // Add the new meta block to the meta index. metaBlockIndexWriter.addEntry( metaNames.get(i), offset, fsBlockWriter.getOnDiskSizeWithHeader()); } } // Load-on-open section. // Data block index. // // In version 2, this section of the file starts with the root level data // block index. We call a function that writes intermediate-level blocks // first, then root level, and returns the offset of the root level block // index. long rootIndexOffset = dataBlockIndexWriter.writeIndexBlocks(outputStream); trailer.setLoadOnOpenOffset(rootIndexOffset); // Meta block index. metaBlockIndexWriter.writeSingleLevelIndex( fsBlockWriter.startWriting(BlockType.ROOT_INDEX), "meta"); fsBlockWriter.writeHeaderAndData(outputStream); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (this.includeMemstoreTS) { appendFileInfo(MAX_MEMSTORE_TS_KEY, Bytes.toBytes(maxMemstoreTS)); appendFileInfo(KEY_VALUE_VERSION, Bytes.toBytes(KEY_VALUE_VER_WITH_MEMSTORE)); } // File info writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO)); fsBlockWriter.writeHeaderAndData(outputStream); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); // Load-on-open data supplied by higher levels, e.g. Bloom filters. for (BlockWritable w : additionalLoadOnOpenData) { fsBlockWriter.writeBlock(w, outputStream); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); } // Now finish off the trailer. trailer.setNumDataIndexLevels(dataBlockIndexWriter.getNumLevels()); trailer.setUncompressedDataIndexSize(dataBlockIndexWriter.getTotalUncompressedSize()); trailer.setFirstDataBlockOffset(firstDataBlockOffset); trailer.setLastDataBlockOffset(lastDataBlockOffset); trailer.setComparatorClass(comparator.getClass()); trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries()); finishClose(trailer); fsBlockWriter.release(); }
/** * Caches the last written HFile block. * * @param offset the offset of the block we want to cache. Used to determine the cache key. */ private void doCacheOnWrite(long offset) { HFileBlock cacheFormatBlock = fsBlockWriter.getBlockForCaching(); cacheConf.getBlockCache().cacheBlock(new BlockCacheKey(name, offset), cacheFormatBlock); }