private final void testChannelWithSegments(int numSegments) throws Exception { final List<MemorySegment> memory = this.memManager.allocatePages(memoryOwner, numSegments); final Channel.ID channel = this.ioManager.createChannel(); BlockChannelWriter writer = null; BlockChannelReader reader = null; try { writer = this.ioManager.createBlockChannelWriter(channel); final ChannelWriterOutputView out = new ChannelWriterOutputView(writer, memory, this.memManager.getPageSize()); long writeStart = System.currentTimeMillis(); int valsLeft = NUM_INTS_WRITTEN; while (valsLeft-- > 0) { out.writeInt(valsLeft); } out.close(); final int numBlocks = out.getBlockCount(); writer.close(); writer = null; long writeElapsed = System.currentTimeMillis() - writeStart; // ---------------------------------------------------------------- reader = ioManager.createBlockChannelReader(channel); final ChannelReaderInputView in = new ChannelReaderInputView(reader, memory, numBlocks, false); long readStart = System.currentTimeMillis(); valsLeft = NUM_INTS_WRITTEN; while (valsLeft-- > 0) { in.readInt(); // Assert.assertTrue(rec.getValue() == valsLeft); } in.close(); reader.close(); long readElapsed = System.currentTimeMillis() - readStart; reader.deleteChannel(); reader = null; LOG.info( "IOManager with " + numSegments + " mem segments: write " + writeElapsed + " msecs, read " + readElapsed + " msecs."); memManager.release(memory); } finally { if (reader != null) { reader.closeAndDelete(); } if (writer != null) { writer.closeAndDelete(); } } }
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException { if (recordsOutFile == null || keysOutFile == null) { throw new IllegalStateException("The LargeRecordHandler has not spilled any records"); } // close the writers and final int lastBlockBytesKeys; final int lastBlockBytesRecords; recordsOutFile.close(); keysOutFile.close(); lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment(); lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment(); recordsOutFile = null; keysOutFile = null; final int pagesForReaders = Math.max( 3 * MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2 * MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50)); final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING); final int pagesForRecordReader = pagesForReaders - pagesForKeyReader; // grab memory for the record reader ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>(); ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>(); for (int i = 0; i < pagesForRecordReader; i++) { memForRecordReader.add(memory.remove(memory.size() - 1)); } for (int i = 0; i < pagesForKeyReader; i++) { memForKeysReader.add(memory.remove(memory.size() - 1)); } keysReader = new FileChannelInputView( ioManager.createBlockChannelReader(keysChannel), memManager, memForKeysReader, lastBlockBytesKeys); InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer); keySorter = new UnilateralSortMerger<Tuple>( memManager, memory, ioManager, keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false); // wait for the sorter to sort the keys MutableObjectIterator<Tuple> result; try { result = keySorter.getIterator(); } catch (InterruptedException e) { throw new IOException(e); } recordsReader = new SeekableFileChannelInputView( ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords); return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields); }