Esempio n. 1
0
 // Warning: scrubTimestamp MUST be the start timestamp of the hard delete transaction that
 // triggers
 // the scrubbing; we need this start timestamp to check whether the hard delete transaction was
 // actually committed before we do any scrubbing
 /* package */ void queueCellsForScrubbing(
     Multimap<String, Cell> tableNameToCell, long scrubTimestamp) {
   if (tableNameToCell.isEmpty()) {
     return;
   }
   scrubberStore.queueCellsForScrubbing(tableNameToCell, scrubTimestamp, batchSizeSupplier.get());
 }
Esempio n. 2
0
  /** @return number of cells read from _scrub table */
  private int scrubSomeCells(
      final TransactionManager txManager, int maxCellsToScrub, long maxScrubTimestamp) {
    SortedMap<Long, Multimap<String, Cell>> scrubTimestampToTableNameToCell =
        scrubberStore.getCellsToScrub(maxCellsToScrub, maxScrubTimestamp);

    // Don't call expensive toString() if trace logging is off
    if (log.isTraceEnabled()) {
      log.trace("Attempting to scrub cells: " + scrubTimestampToTableNameToCell);
    }

    if (log.isInfoEnabled()) {
      Set<String> tables = Sets.newHashSet();
      for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) {
        tables.addAll(v.keySet());
      }
      log.info(
          "Attempting to scrub "
              + scrubTimestampToTableNameToCell.size()
              + " cells from tables "
              + tables);
    }

    if (scrubTimestampToTableNameToCell.size() == 0) {
      return 0; // No cells left to scrub
    }

    Multimap<Long, Cell> toRemoveFromScrubQueue = HashMultimap.create();

    int numCellsReadFromScrubTable = 0;
    for (Map.Entry<Long, Multimap<String, Cell>> entry :
        scrubTimestampToTableNameToCell.entrySet()) {
      final long scrubTimestamp = entry.getKey();
      final Multimap<String, Cell> tableNameToCell = entry.getValue();

      numCellsReadFromScrubTable += tableNameToCell.size();

      long commitTimestamp = getCommitTimestampRollBackIfNecessary(scrubTimestamp, tableNameToCell);
      if (commitTimestamp >= maxScrubTimestamp) {
        // We cannot scrub this yet because not all transactions can read this value.
        continue;
      } else if (commitTimestamp != TransactionConstants.FAILED_COMMIT_TS) {
        // This is CRITICAL; don't scrub if the hard delete transaction didn't actually finish
        // (we still remove it from the _scrub table with the call to markCellsAsScrubbed though),
        // or else we could cause permanent data loss if the hard delete transaction failed after
        // queuing cells to scrub but before successfully committing
        List<Future<Void>> scrubFutures = Lists.newArrayList();
        for (final List<Entry<String, Cell>> batch :
            Iterables.partition(tableNameToCell.entries(), batchSizeSupplier.get())) {
          final Multimap<String, Cell> batchMultimap = HashMultimap.create();
          for (Entry<String, Cell> e : batch) {
            batchMultimap.put(e.getKey(), e.getValue());
          }
          scrubFutures.add(
              exec.submit(
                  new Callable<Void>() {
                    @Override
                    public Void call() throws Exception {
                      scrubCells(
                          txManager,
                          batchMultimap,
                          scrubTimestamp,
                          aggressiveScrub
                              ? TransactionType.AGGRESSIVE_HARD_DELETE
                              : TransactionType.HARD_DELETE);
                      return null;
                    }
                  }));
        }
        for (Future<Void> future : scrubFutures) {
          Futures.getUnchecked(future);
        }
      }
      toRemoveFromScrubQueue.putAll(scrubTimestamp, tableNameToCell.values());
    }

    Multimap<Cell, Long> cellToScrubTimestamp = HashMultimap.create();
    scrubberStore.markCellsAsScrubbed(
        Multimaps.invertFrom(toRemoveFromScrubQueue, cellToScrubTimestamp),
        batchSizeSupplier.get());

    if (log.isTraceEnabled()) {
      log.trace("Finished scrubbing cells: " + scrubTimestampToTableNameToCell);
    }

    if (log.isInfoEnabled()) {
      Set<String> tables = Sets.newHashSet();
      for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) {
        tables.addAll(v.keySet());
      }
      log.info(
          "Finished scrubbing "
              + scrubTimestampToTableNameToCell.size()
              + " cells from tables "
              + tables);
    }

    return numCellsReadFromScrubTable;
  }