/** * Write blocks to the file at the given path until the maxFileSize is reached. * * @param path * @param iterator * @throws IOException */ protected void writeIndex( String path, BlockWriter blockIndex, Iterator<Entry<Object, Object>> iterator) throws IOException { FileOutputStream out = new FileOutputStream(path); BlockWriter block; if (compressed) block = new CompressedBlockWriter(true, true); else block = new DefaultBlockWriter(true, true); int entryCount = 0; int blockOffset = 0; boolean newBlockFile = false; // write each block to disk // note that blocks can become slightly larger than the maxFileSize // depending on the size of the last block while (iterator.hasNext() && !newBlockFile) { // add the next key-value pair to the current block Entry<Object, Object> next = iterator.next(); block.add(next.getKey(), next.getValue()); entryCount++; // if the block size limit has been reached, or there are no more // key-value pairs, serialize the block and write it to disk if (entryCount % maxBlockEntries == 0 || !iterator.hasNext()) { // serialize the offset of the block into a new buffer ReusableBuffer buf = ReusableBuffer.wrap(new byte[(Integer.SIZE / 8) + (Short.SIZE / 8)]); buf.putInt(blockOffset); buf.putShort(blockFileId); // add the key-offset mapping to the block index blockIndex.add(InternalBufferUtil.toBuffer(block.getBlockKey()), buf.array()); // serialize the block and calculate the next block offset SerializedBlock serializedBlock = block.serialize(); blockOffset += serializedBlock.size(); // write the block int writtenBytes = 0; Iterator<Object> it = serializedBlock.iterator(); while (it.hasNext()) { // for robustness: check if the file descriptor is still valid, // re-open the file if necessary if (!out.getFD().valid()) { out.close(); out = new FileOutputStream(path, true); } Object nextBuffer = it.next(); // check if the entry is the last from the buffer; if so, free it if (nextBuffer instanceof ByteRange) { ByteRange rng = (ByteRange) nextBuffer; if (rng.getReusableBuf() != null) BufferPool.free(rng.getReusableBuf()); } writtenBytes += writeBuffer(out, nextBuffer); } assert (writtenBytes == serializedBlock.size()); if (blockOffset >= maxFileSize) { newBlockFile = true; } else { if (iterator.hasNext()) if (compressed) block = new CompressedBlockWriter(true, true); else block = new DefaultBlockWriter(true, true); } } } out.close(); }