@Override
public Result next() throws IOException {
// If the scanner is closed and there's nothing left in the cache, next is a
// no-op.
if (cache.size() == 0 && this.closed) {
return null;
}
if (cache.size() == 0) {
Result[] values = null;
long remainingResultSize = maxScannerResultSize;
int countdown = this.caching;
boolean currentRegionDone = false;
// Values == null means server-side filter has determined we must STOP
while (remainingResultSize > 0
&& countdown > 0
&& nextScanner(countdown, values == null, currentRegionDone)) {
// Server returns a null values if scanning is to stop. Else,
// returns an empty array if scanning is to go on and we've just
// exhausted current region.
values = smallScanCallable.withRetries();
this.currentRegion = smallScanCallable.getHRegionInfo();
long currentTime = System.currentTimeMillis();
if (this.scanMetrics != null) {
this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime - lastNext);
}
lastNext = currentTime;
if (values != null && values.length > 0) {
for (int i = 0; i < values.length; i++) {
Result rs = values[i];
if (i == 0
&& this.skipRowOfFirstResult != null
&& Bytes.equals(skipRowOfFirstResult, rs.getRow())) {
// Skip the first result
continue;
}
cache.add(rs);
for (KeyValue kv : rs.raw()) {
remainingResultSize -= kv.heapSize();
}
countdown--;
this.lastResult = rs;
}
}
currentRegionDone = countdown > 0;
}
}
if (cache.size() > 0) {
return cache.poll();
}
// if we exhausted this scanner before calling close, write out the scan
// metrics
writeScanMetrics();
return null;
}
@Override
protected void reduce(K row, Iterable<Put> vals, Context context)
throws IOException, InterruptedException {
// Using HeapSize to create an upper bound on the memory size of
// the puts and flush some portion of the content while looping. This
// flush could result in multiple Puts for a single rowkey. That is
// acceptable because Combiner is run as an optimization and it's not
// critical that all Puts are grouped perfectly.
long threshold =
context.getConfiguration().getLong("putcombiner.row.threshold", 1L * (1 << 30));
int cnt = 0;
long curSize = 0;
Put put = null;
Map<byte[], List<Cell>> familyMap = null;
for (Put p : vals) {
cnt++;
if (put == null) {
put = p;
familyMap = put.getFamilyCellMap();
} else {
for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap().entrySet()) {
List<Cell> cells = familyMap.get(entry.getKey());
List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null;
for (Cell cell : entry.getValue()) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
curSize += kv.heapSize();
if (kvs != null) {
kvs.add(kv);
}
}
if (cells == null) {
familyMap.put(entry.getKey(), entry.getValue());
}
}
if (cnt % 10 == 0) context.setStatus("Combine " + cnt);
if (curSize > threshold) {
LOG.info(String.format("Combined %d Put(s) into %d.", cnt, 1));
context.write(row, put);
put = null;
cnt = 0;
}
}
}
if (put != null) {
LOG.info(String.format("Combined %d Put(s) into %d.", cnt, 1));
context.write(row, put);
}
}
