@Override
public Tuple next() throws IOException {
try {
Tuple tuple;
int partId;
long numRows = 0;
while (!context.isStopped() && (tuple = child.next()) != null) {
partId = partitioner.getPartition(tuple);
MemoryRowBlock rowBlock = partitionMemoryMap.get(partId);
if (rowBlock == null) {
rowBlock = new MemoryRowBlock(dataTypes, initialBufferSize, true, plan.getStorageType());
partitionMemoryMap.put(partId, rowBlock);
totalBufferCapacity += rowBlock.capacity();
}
RowWriter writer = rowBlock.getWriter();
long prevUsedMem = rowBlock.usedMem();
totalBufferCapacity -= rowBlock.capacity();
writer.addTuple(tuple);
numRows++;
totalBufferCapacity += rowBlock.capacity(); // calculate resizeable buffer capacity
usedBufferSize += (rowBlock.usedMem() - prevUsedMem);
// if total buffer capacity are required more than maxBufferSize,
// all partitions are flushed and the buffers are released
if (totalBufferCapacity > maxBufferSize) {
if (LOG.isDebugEnabled()) {
LOG.debug(
String.format(
"Too low buffer usage. threshold: %s, total capacity: %s, used: %s",
FileUtil.humanReadableByteCount(maxBufferSize, false),
FileUtil.humanReadableByteCount(totalBufferCapacity, false),
FileUtil.humanReadableByteCount(usedBufferSize, false)));
}
// flush and release buffer
flushBuffer(partitionMemoryMap, true);
writtenBytes += usedBufferSize;
totalBufferCapacity = usedBufferSize = 0;
} else if (usedBufferSize > bufferThreshold) {
// flush and reuse buffer
flushBuffer(partitionMemoryMap, false);
writtenBytes += usedBufferSize;
usedBufferSize = 0;
}
}
// flush remaining buffers
flushBuffer(partitionMemoryMap, true);
writtenBytes += usedBufferSize;
usedBufferSize = totalBufferCapacity = 0;
TableStats aggregated = (TableStats) child.getInputStats().clone();
aggregated.setNumBytes(writtenBytes);
aggregated.setNumRows(numRows);
context.setResultStats(aggregated);
return null;
} catch (RuntimeException e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
}
}
