@Override public Tuple next() throws IOException { try { Tuple tuple; int partId; long numRows = 0; while (!context.isStopped() && (tuple = child.next()) != null) { partId = partitioner.getPartition(tuple); MemoryRowBlock rowBlock = partitionMemoryMap.get(partId); if (rowBlock == null) { rowBlock = new MemoryRowBlock(dataTypes, initialBufferSize, true, plan.getStorageType()); partitionMemoryMap.put(partId, rowBlock); totalBufferCapacity += rowBlock.capacity(); } RowWriter writer = rowBlock.getWriter(); long prevUsedMem = rowBlock.usedMem(); totalBufferCapacity -= rowBlock.capacity(); writer.addTuple(tuple); numRows++; totalBufferCapacity += rowBlock.capacity(); // calculate resizeable buffer capacity usedBufferSize += (rowBlock.usedMem() - prevUsedMem); // if total buffer capacity are required more than maxBufferSize, // all partitions are flushed and the buffers are released if (totalBufferCapacity > maxBufferSize) { if (LOG.isDebugEnabled()) { LOG.debug( String.format( "Too low buffer usage. threshold: %s, total capacity: %s, used: %s", FileUtil.humanReadableByteCount(maxBufferSize, false), FileUtil.humanReadableByteCount(totalBufferCapacity, false), FileUtil.humanReadableByteCount(usedBufferSize, false))); } // flush and release buffer flushBuffer(partitionMemoryMap, true); writtenBytes += usedBufferSize; totalBufferCapacity = usedBufferSize = 0; } else if (usedBufferSize > bufferThreshold) { // flush and reuse buffer flushBuffer(partitionMemoryMap, false); writtenBytes += usedBufferSize; usedBufferSize = 0; } } // flush remaining buffers flushBuffer(partitionMemoryMap, true); writtenBytes += usedBufferSize; usedBufferSize = totalBufferCapacity = 0; TableStats aggregated = (TableStats) child.getInputStats().clone(); aggregated.setNumBytes(writtenBytes); aggregated.setNumRows(numRows); context.setResultStats(aggregated); return null; } catch (RuntimeException e) { LOG.error(e.getMessage(), e); throw new IOException(e); } catch (Throwable e) { LOG.error(e.getMessage(), e); throw new IOException(e); } }