protected final void trackDelete(Cell cell) {
// If keepDeletedCells is true, then we only remove cells by versions or TTL during
// compaction, so we do not need to track delete here.
// If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
// minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
// need to track the delete marker to see if it masks some cells.
if (keepDeletedCells == KeepDeletedCells.FALSE
|| (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
deletes.add(cell);
}
}
@Override
protected void reset() {
deletes.reset();
}
private Result filter(
TransactionState state, Result result, long startTimestamp, int localVersions)
throws IOException {
if (result == null) {
return null;
}
List<KeyValue> kvs = result.list();
if (kvs == null) {
return result;
}
Map<ByteArray, Map<ByteArray, Integer>> occurrences =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Integer>>();
Map<ByteArray, Map<ByteArray, Long>> minTimestamp =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Long>>();
List<KeyValue> nonDeletes = new ArrayList<KeyValue>();
List<KeyValue> filtered = new ArrayList<KeyValue>();
Map<ByteArray, Set<ByteArray>> read = new HashMap<ByteArray, Set<ByteArray>>();
DeleteTracker tracker = new DeleteTracker();
for (KeyValue kv : kvs) {
ByteArray family = new ByteArray(kv.getFamily());
ByteArray qualifier = new ByteArray(kv.getQualifier());
Set<ByteArray> readQualifiers = read.get(family);
if (readQualifiers == null) {
readQualifiers = new HashSet<TransactionalTable.ByteArray>();
read.put(family, readQualifiers);
} else if (readQualifiers.contains(qualifier)) continue;
// RowKey rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(kv.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(kv)) nonDeletes.add(kv);
{
// Read valid value
readQualifiers.add(qualifier);
// statistics
// elementsGotten++;
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Integer times = null;
if (occurrencesCols != null) {
times = occurrencesCols.get(qualifier);
}
if (times != null) {
// elementsRead += times;
versionsAvg = times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
// extraVersionsAvg = times > extraVersionsAvg ? times : alpha *
// extraVersionsAvg + (1 - alpha) * times;
} else {
// elementsRead++;
versionsAvg = alpha * versionsAvg + (1 - alpha);
// extraVersionsAvg = alpha * extraVersionsAvg + (1 - alpha);
}
}
} else {
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Map<ByteArray, Long> minTimestampCols = minTimestamp.get(family);
if (occurrencesCols == null) {
occurrencesCols = new HashMap<TransactionalTable.ByteArray, Integer>();
minTimestampCols = new HashMap<TransactionalTable.ByteArray, Long>();
occurrences.put(family, occurrencesCols);
minTimestamp.put(family, minTimestampCols);
}
Integer times = occurrencesCols.get(qualifier);
Long timestamp = minTimestampCols.get(qualifier);
if (times == null) {
times = 0;
timestamp = kv.getTimestamp();
}
times++;
timestamp = Math.min(timestamp, kv.getTimestamp());
if (times == localVersions) {
// We need to fetch more versions
Get get = new Get(kv.getRow());
get.addColumn(kv.getFamily(), kv.getQualifier());
get.setMaxVersions(localVersions);
Result r;
GOTRESULT:
do {
extraGetsPerformed++;
get.setTimeRange(0, timestamp);
r = this.get(get);
List<KeyValue> list = r.list();
if (list == null) break;
for (KeyValue t : list) {
times++;
timestamp = Math.min(timestamp, t.getTimestamp());
// rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(t.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(t)) nonDeletes.add(t);
readQualifiers.add(qualifier);
elementsGotten++;
elementsRead += times;
versionsAvg =
times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
extraVersionsAvg =
times > extraVersionsAvg
? times
: alpha * extraVersionsAvg + (1 - alpha) * times;
break GOTRESULT;
}
}
} while (r.size() == localVersions);
} else {
occurrencesCols.put(qualifier, times);
minTimestampCols.put(qualifier, timestamp);
}
}
}
for (KeyValue kv : nonDeletes) {
if (!tracker.isDeleted(kv)) {
filtered.add(kv);
}
}
// cacheVersions = (int) versionsAvg;
if (filtered.isEmpty()) {
return null;
}
return new Result(filtered);
}