@Override public Result next() throws IOException { // If the scanner is closed and there's nothing left in the cache, next is a // no-op. if (cache.size() == 0 && this.closed) { return null; } if (cache.size() == 0) { Result[] values = null; long remainingResultSize = maxScannerResultSize; int countdown = this.caching; boolean currentRegionDone = false; // Values == null means server-side filter has determined we must STOP while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null, currentRegionDone)) { // Server returns a null values if scanning is to stop. Else, // returns an empty array if scanning is to go on and we've just // exhausted current region. values = smallScanCallable.withRetries(); this.currentRegion = smallScanCallable.getHRegionInfo(); long currentTime = System.currentTimeMillis(); if (this.scanMetrics != null) { this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime - lastNext); } lastNext = currentTime; if (values != null && values.length > 0) { for (int i = 0; i < values.length; i++) { Result rs = values[i]; if (i == 0 && this.skipRowOfFirstResult != null && Bytes.equals(skipRowOfFirstResult, rs.getRow())) { // Skip the first result continue; } cache.add(rs); for (KeyValue kv : rs.raw()) { remainingResultSize -= kv.heapSize(); } countdown--; this.lastResult = rs; } } currentRegionDone = countdown > 0; } } if (cache.size() > 0) { return cache.poll(); } // if we exhausted this scanner before calling close, write out the scan // metrics writeScanMetrics(); return null; }
@Override protected void reduce(K row, Iterable<Put> vals, Context context) throws IOException, InterruptedException { // Using HeapSize to create an upper bound on the memory size of // the puts and flush some portion of the content while looping. This // flush could result in multiple Puts for a single rowkey. That is // acceptable because Combiner is run as an optimization and it's not // critical that all Puts are grouped perfectly. long threshold = context.getConfiguration().getLong("putcombiner.row.threshold", 1L * (1 << 30)); int cnt = 0; long curSize = 0; Put put = null; Map<byte[], List<Cell>> familyMap = null; for (Put p : vals) { cnt++; if (put == null) { put = p; familyMap = put.getFamilyCellMap(); } else { for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap().entrySet()) { List<Cell> cells = familyMap.get(entry.getKey()); List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null; for (Cell cell : entry.getValue()) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); curSize += kv.heapSize(); if (kvs != null) { kvs.add(kv); } } if (cells == null) { familyMap.put(entry.getKey(), entry.getValue()); } } if (cnt % 10 == 0) context.setStatus("Combine " + cnt); if (curSize > threshold) { LOG.info(String.format("Combined %d Put(s) into %d.", cnt, 1)); context.write(row, put); put = null; cnt = 0; } } } if (put != null) { LOG.info(String.format("Combined %d Put(s) into %d.", cnt, 1)); context.write(row, put); } }