/**
* Write blocks to the file at the given path until the maxFileSize is reached.
*
* @param path
* @param iterator
* @throws IOException
*/
protected void writeIndex(
String path, BlockWriter blockIndex, Iterator<Entry<Object, Object>> iterator)
throws IOException {
FileOutputStream out = new FileOutputStream(path);
BlockWriter block;
if (compressed) block = new CompressedBlockWriter(true, true);
else block = new DefaultBlockWriter(true, true);
int entryCount = 0;
int blockOffset = 0;
boolean newBlockFile = false;
// write each block to disk
// note that blocks can become slightly larger than the maxFileSize
// depending on the size of the last block
while (iterator.hasNext() && !newBlockFile) {
// add the next key-value pair to the current block
Entry<Object, Object> next = iterator.next();
block.add(next.getKey(), next.getValue());
entryCount++;
// if the block size limit has been reached, or there are no more
// key-value pairs, serialize the block and write it to disk
if (entryCount % maxBlockEntries == 0 || !iterator.hasNext()) {
// serialize the offset of the block into a new buffer
ReusableBuffer buf = ReusableBuffer.wrap(new byte[(Integer.SIZE / 8) + (Short.SIZE / 8)]);
buf.putInt(blockOffset);
buf.putShort(blockFileId);
// add the key-offset mapping to the block index
blockIndex.add(InternalBufferUtil.toBuffer(block.getBlockKey()), buf.array());
// serialize the block and calculate the next block offset
SerializedBlock serializedBlock = block.serialize();
blockOffset += serializedBlock.size();
// write the block
int writtenBytes = 0;
Iterator<Object> it = serializedBlock.iterator();
while (it.hasNext()) {
// for robustness: check if the file descriptor is still valid,
// re-open the file if necessary
if (!out.getFD().valid()) {
out.close();
out = new FileOutputStream(path, true);
}
Object nextBuffer = it.next();
// check if the entry is the last from the buffer; if so, free it
if (nextBuffer instanceof ByteRange) {
ByteRange rng = (ByteRange) nextBuffer;
if (rng.getReusableBuf() != null) BufferPool.free(rng.getReusableBuf());
}
writtenBytes += writeBuffer(out, nextBuffer);
}
assert (writtenBytes == serializedBlock.size());
if (blockOffset >= maxFileSize) {
newBlockFile = true;
} else {
if (iterator.hasNext())
if (compressed) block = new CompressedBlockWriter(true, true);
else block = new DefaultBlockWriter(true, true);
}
}
}
out.close();
}
