private final void testChannelWithSegments(int numSegments) throws Exception {
final List<MemorySegment> memory = this.memManager.allocatePages(memoryOwner, numSegments);
final Channel.ID channel = this.ioManager.createChannel();
BlockChannelWriter writer = null;
BlockChannelReader reader = null;
try {
writer = this.ioManager.createBlockChannelWriter(channel);
final ChannelWriterOutputView out =
new ChannelWriterOutputView(writer, memory, this.memManager.getPageSize());
long writeStart = System.currentTimeMillis();
int valsLeft = NUM_INTS_WRITTEN;
while (valsLeft-- > 0) {
out.writeInt(valsLeft);
}
out.close();
final int numBlocks = out.getBlockCount();
writer.close();
writer = null;
long writeElapsed = System.currentTimeMillis() - writeStart;
// ----------------------------------------------------------------
reader = ioManager.createBlockChannelReader(channel);
final ChannelReaderInputView in =
new ChannelReaderInputView(reader, memory, numBlocks, false);
long readStart = System.currentTimeMillis();
valsLeft = NUM_INTS_WRITTEN;
while (valsLeft-- > 0) {
in.readInt();
// Assert.assertTrue(rec.getValue() == valsLeft);
}
in.close();
reader.close();
long readElapsed = System.currentTimeMillis() - readStart;
reader.deleteChannel();
reader = null;
LOG.info(
"IOManager with "
+ numSegments
+ " mem segments: write "
+ writeElapsed
+ " msecs, read "
+ readElapsed
+ " msecs.");
memManager.release(memory);
} finally {
if (reader != null) {
reader.closeAndDelete();
}
if (writer != null) {
writer.closeAndDelete();
}
}
}
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory)
throws IOException {
if (recordsOutFile == null || keysOutFile == null) {
throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
}
// close the writers and
final int lastBlockBytesKeys;
final int lastBlockBytesRecords;
recordsOutFile.close();
keysOutFile.close();
lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
recordsOutFile = null;
keysOutFile = null;
final int pagesForReaders =
Math.max(
3 * MIN_SEGMENTS_FOR_KEY_SPILLING,
Math.min(2 * MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
final int pagesForKeyReader =
Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
// grab memory for the record reader
ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
for (int i = 0; i < pagesForRecordReader; i++) {
memForRecordReader.add(memory.remove(memory.size() - 1));
}
for (int i = 0; i < pagesForKeyReader; i++) {
memForKeysReader.add(memory.remove(memory.size() - 1));
}
keysReader =
new FileChannelInputView(
ioManager.createBlockChannelReader(keysChannel),
memManager,
memForKeysReader,
lastBlockBytesKeys);
InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
keySorter =
new UnilateralSortMerger<Tuple>(
memManager,
memory,
ioManager,
keyIterator,
memoryOwner,
keySerializerFactory,
keyComparator,
1,
maxFilehandles,
1.0f,
false);
// wait for the sorter to sort the keys
MutableObjectIterator<Tuple> result;
try {
result = keySorter.getIterator();
} catch (InterruptedException e) {
throw new IOException(e);
}
recordsReader =
new SeekableFileChannelInputView(
ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
