/**
* Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on
* construction.
*
* @param kv KeyValue to add. Cannot be empty nor null.
* @throws IOException
*/
@Override
public void append(final KeyValue kv) throws IOException {
byte[] key = kv.getBuffer();
int koffset = kv.getKeyOffset();
int klength = kv.getKeyLength();
byte[] value = kv.getValueArray();
int voffset = kv.getValueOffset();
int vlength = kv.getValueLength();
boolean dupKey = checkKey(key, koffset, klength);
checkValue(value, voffset, vlength);
if (!dupKey) {
checkBlockBoundary();
}
if (!fsBlockWriter.isWriting()) newBlock();
fsBlockWriter.write(kv);
totalKeyLength += klength;
totalValueLength += vlength;
// Are we the first key in this block?
if (firstKeyInBlock == null) {
// Copy the key.
firstKeyInBlock = new byte[klength];
System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
}
lastKeyBuffer = key;
lastKeyOffset = koffset;
lastKeyLength = klength;
entryCount++;
this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion());
}
/**
* Add key/value to file. Keys must be added in an order that agrees with the Comparator passed on
* construction.
*
* @param key
* @param koffset
* @param klength
* @param value
* @param voffset
* @param vlength
* @throws IOException
*/
private void append(
final long memstoreTS,
final byte[] key,
final int koffset,
final int klength,
final byte[] value,
final int voffset,
final int vlength)
throws IOException {
boolean dupKey = checkKey(key, koffset, klength);
checkValue(value, voffset, vlength);
if (!dupKey) {
checkBlockBoundary();
}
if (!fsBlockWriter.isWriting()) newBlock();
// Write length of key and value and then actual key and value bytes.
// Additionally, we may also write down the memstoreTS.
{
DataOutputStream out = fsBlockWriter.getUserDataStream();
out.writeInt(klength);
totalKeyLength += klength;
out.writeInt(vlength);
totalValueLength += vlength;
out.write(key, koffset, klength);
out.write(value, voffset, vlength);
if (this.includeMemstoreTS) {
WritableUtils.writeVLong(out, memstoreTS);
}
}
// Are we the first key in this block?
if (firstKeyInBlock == null) {
// Copy the key.
firstKeyInBlock = new byte[klength];
System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
}
lastKeyBuffer = key;
lastKeyOffset = koffset;
lastKeyLength = klength;
entryCount++;
}
/** Clean up the current block */
private void finishBlock() throws IOException {
if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return;
long startTimeNs = System.nanoTime();
// Update the first data block offset for scanning.
if (firstDataBlockOffset == -1) {
firstDataBlockOffset = outputStream.getPos();
}
// Update the last data block offset
lastDataBlockOffset = outputStream.getPos();
fsBlockWriter.writeHeaderAndData(outputStream);
int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
// Generate a shorter faked key into index block. For example, consider a block boundary
// between the keys "the quick brown fox" and "the who test text". We can use "the r" as the
// key for the index block entry since it is > all entries in the previous block and <= all
// entries in subsequent blocks.
if (comparator instanceof KeyComparator) {
byte[] fakeKey =
((KeyComparator) comparator).getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock);
if (comparator.compare(fakeKey, firstKeyInBlock) > 0) {
throw new IOException(
"Unexpected getShortMidpointKey result, fakeKey:"
+ Bytes.toStringBinary(fakeKey)
+ ", firstKeyInBlock:"
+ Bytes.toStringBinary(firstKeyInBlock));
}
if (lastKeyOfPreviousBlock != null
&& comparator.compare(lastKeyOfPreviousBlock, fakeKey) >= 0) {
throw new IOException(
"Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:"
+ Bytes.toStringBinary(lastKeyOfPreviousBlock)
+ ", fakeKey:"
+ Bytes.toStringBinary(fakeKey));
}
dataBlockIndexWriter.addEntry(fakeKey, lastDataBlockOffset, onDiskSize);
} else {
dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset, onDiskSize);
}
totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
if (cacheConf.shouldCacheDataOnWrite()) {
doCacheOnWrite(lastDataBlockOffset);
}
}
/** Clean up the current block */
private void finishBlock() throws IOException {
if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return;
// Update the first data block offset for scanning.
if (firstDataBlockOffset == -1) {
firstDataBlockOffset = outputStream.getPos();
}
// Update the last data block offset
lastDataBlockOffset = outputStream.getPos();
fsBlockWriter.writeHeaderAndData(outputStream);
int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
byte[] indexKey = comparator.calcIndexKey(lastKeyOfPreviousBlock, firstKeyInBlock);
dataBlockIndexWriter.addEntry(indexKey, lastDataBlockOffset, onDiskSize);
totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
if (cacheConf.shouldCacheDataOnWrite()) {
doCacheOnWrite(lastDataBlockOffset);
}
}
