protected final void trackDelete(Cell cell) {
   // If keepDeletedCells is true, then we only remove cells by versions or TTL during
   // compaction, so we do not need to track delete here.
   // If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
   // minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
   // need to track the delete marker to see if it masks some cells.
   if (keepDeletedCells == KeepDeletedCells.FALSE
       || (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
     deletes.add(cell);
   }
 }
 @Override
 protected void reset() {
   deletes.reset();
 }
Example #3
0
  private Result filter(
      TransactionState state, Result result, long startTimestamp, int localVersions)
      throws IOException {
    if (result == null) {
      return null;
    }
    List<KeyValue> kvs = result.list();
    if (kvs == null) {
      return result;
    }
    Map<ByteArray, Map<ByteArray, Integer>> occurrences =
        new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Integer>>();
    Map<ByteArray, Map<ByteArray, Long>> minTimestamp =
        new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Long>>();
    List<KeyValue> nonDeletes = new ArrayList<KeyValue>();
    List<KeyValue> filtered = new ArrayList<KeyValue>();
    Map<ByteArray, Set<ByteArray>> read = new HashMap<ByteArray, Set<ByteArray>>();
    DeleteTracker tracker = new DeleteTracker();
    for (KeyValue kv : kvs) {
      ByteArray family = new ByteArray(kv.getFamily());
      ByteArray qualifier = new ByteArray(kv.getQualifier());
      Set<ByteArray> readQualifiers = read.get(family);
      if (readQualifiers == null) {
        readQualifiers = new HashSet<TransactionalTable.ByteArray>();
        read.put(family, readQualifiers);
      } else if (readQualifiers.contains(qualifier)) continue;
      //         RowKey rk = new RowKey(kv.getRow(), getTableName());
      if (state.tsoclient.validRead(kv.getTimestamp(), startTimestamp)) {
        if (!tracker.addDeleted(kv)) nonDeletes.add(kv);
        {
          // Read valid value
          readQualifiers.add(qualifier);

          //                statistics
          //               elementsGotten++;
          Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
          Integer times = null;
          if (occurrencesCols != null) {
            times = occurrencesCols.get(qualifier);
          }
          if (times != null) {
            //                  elementsRead += times;
            versionsAvg = times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
            //                  extraVersionsAvg = times > extraVersionsAvg ? times : alpha *
            // extraVersionsAvg + (1 - alpha) * times;
          } else {
            //                  elementsRead++;
            versionsAvg = alpha * versionsAvg + (1 - alpha);
            //                  extraVersionsAvg = alpha * extraVersionsAvg + (1 - alpha);
          }
        }
      } else {
        Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
        Map<ByteArray, Long> minTimestampCols = minTimestamp.get(family);
        if (occurrencesCols == null) {
          occurrencesCols = new HashMap<TransactionalTable.ByteArray, Integer>();
          minTimestampCols = new HashMap<TransactionalTable.ByteArray, Long>();
          occurrences.put(family, occurrencesCols);
          minTimestamp.put(family, minTimestampCols);
        }
        Integer times = occurrencesCols.get(qualifier);
        Long timestamp = minTimestampCols.get(qualifier);
        if (times == null) {
          times = 0;
          timestamp = kv.getTimestamp();
        }
        times++;
        timestamp = Math.min(timestamp, kv.getTimestamp());
        if (times == localVersions) {
          // We need to fetch more versions
          Get get = new Get(kv.getRow());
          get.addColumn(kv.getFamily(), kv.getQualifier());
          get.setMaxVersions(localVersions);
          Result r;
          GOTRESULT:
          do {
            extraGetsPerformed++;
            get.setTimeRange(0, timestamp);
            r = this.get(get);
            List<KeyValue> list = r.list();
            if (list == null) break;
            for (KeyValue t : list) {
              times++;
              timestamp = Math.min(timestamp, t.getTimestamp());
              //                     rk = new RowKey(kv.getRow(), getTableName());
              if (state.tsoclient.validRead(t.getTimestamp(), startTimestamp)) {
                if (!tracker.addDeleted(t)) nonDeletes.add(t);
                readQualifiers.add(qualifier);
                elementsGotten++;
                elementsRead += times;
                versionsAvg =
                    times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
                extraVersionsAvg =
                    times > extraVersionsAvg
                        ? times
                        : alpha * extraVersionsAvg + (1 - alpha) * times;
                break GOTRESULT;
              }
            }
          } while (r.size() == localVersions);
        } else {
          occurrencesCols.put(qualifier, times);
          minTimestampCols.put(qualifier, timestamp);
        }
      }
    }
    for (KeyValue kv : nonDeletes) {
      if (!tracker.isDeleted(kv)) {
        filtered.add(kv);
      }
    }
    //      cacheVersions = (int) versionsAvg;
    if (filtered.isEmpty()) {
      return null;
    }
    return new Result(filtered);
  }