/** * Creates an on-disk representation of an index from an iterator of key-value pairs. The iterator * has to return keys in ascending order! * * @param iterator an iterator w/ key-value pairs, keys must be in ascending order * @throws IOException if an I/O error occurs */ public void writeIndex(ResultSet<Object, Object> iterator) throws IOException { BlockWriter blockIndex = new DefaultBlockWriter(true, false); // write all index files while (iterator.hasNext()) { String indexPath = path + "blockfile_" + new Short(blockFileId).toString() + ".idx"; writeIndex(indexPath, blockIndex, iterator); blockFileId++; } iterator.free(); // write the block index new File(path + "blockindex.idx").createNewFile(); FileOutputStream out = new FileOutputStream(path + "blockindex.idx", false); SerializedBlock serializedBuf = blockIndex.serialize(); int bytesWritten = 0; Iterator<Object> it = serializedBuf.iterator(); while (it.hasNext()) bytesWritten += writeBuffer(out, it.next()); assert (bytesWritten == serializedBuf.size()); out.close(); }
@Override void writeInstruction(Instruction _instruction) throws CodeGenException { if ((_instruction instanceof I_IUSHR) || (_instruction instanceof I_LUSHR)) { BinaryOperator binaryInstruction = (BinaryOperator) _instruction; Instruction parent = binaryInstruction.getParentExpr(); boolean needsParenthesis = true; if (parent instanceof AssignToLocalVariable) { needsParenthesis = false; } else if (parent instanceof AssignToField) { needsParenthesis = false; } else if (parent instanceof AssignToArrayElement) { needsParenthesis = false; } if (needsParenthesis) { write("("); } if (binaryInstruction instanceof I_IUSHR) { write("((unsigned int)"); } else { write("((unsigned long)"); } writeInstruction(binaryInstruction.getLhs()); write(")"); write(" >> "); writeInstruction(binaryInstruction.getRhs()); if (needsParenthesis) { write(")"); } } else { super.writeInstruction(_instruction); } }
/** * Write blocks to the file at the given path until the maxFileSize is reached. * * @param path * @param iterator * @throws IOException */ protected void writeIndex( String path, BlockWriter blockIndex, Iterator<Entry<Object, Object>> iterator) throws IOException { FileOutputStream out = new FileOutputStream(path); BlockWriter block; if (compressed) block = new CompressedBlockWriter(true, true); else block = new DefaultBlockWriter(true, true); int entryCount = 0; int blockOffset = 0; boolean newBlockFile = false; // write each block to disk // note that blocks can become slightly larger than the maxFileSize // depending on the size of the last block while (iterator.hasNext() && !newBlockFile) { // add the next key-value pair to the current block Entry<Object, Object> next = iterator.next(); block.add(next.getKey(), next.getValue()); entryCount++; // if the block size limit has been reached, or there are no more // key-value pairs, serialize the block and write it to disk if (entryCount % maxBlockEntries == 0 || !iterator.hasNext()) { // serialize the offset of the block into a new buffer ReusableBuffer buf = ReusableBuffer.wrap(new byte[(Integer.SIZE / 8) + (Short.SIZE / 8)]); buf.putInt(blockOffset); buf.putShort(blockFileId); // add the key-offset mapping to the block index blockIndex.add(InternalBufferUtil.toBuffer(block.getBlockKey()), buf.array()); // serialize the block and calculate the next block offset SerializedBlock serializedBlock = block.serialize(); blockOffset += serializedBlock.size(); // write the block int writtenBytes = 0; Iterator<Object> it = serializedBlock.iterator(); while (it.hasNext()) { // for robustness: check if the file descriptor is still valid, // re-open the file if necessary if (!out.getFD().valid()) { out.close(); out = new FileOutputStream(path, true); } Object nextBuffer = it.next(); // check if the entry is the last from the buffer; if so, free it if (nextBuffer instanceof ByteRange) { ByteRange rng = (ByteRange) nextBuffer; if (rng.getReusableBuf() != null) BufferPool.free(rng.getReusableBuf()); } writtenBytes += writeBuffer(out, nextBuffer); } assert (writtenBytes == serializedBlock.size()); if (blockOffset >= maxFileSize) { newBlockFile = true; } else { if (iterator.hasNext()) if (compressed) block = new CompressedBlockWriter(true, true); else block = new DefaultBlockWriter(true, true); } } } out.close(); }