Esempio n. 1
1
 private void scrubCells(
     TransactionManager txManager,
     Multimap<String, Cell> tableNameToCells,
     long scrubTimestamp,
     Transaction.TransactionType transactionType) {
   for (Entry<String, Collection<Cell>> entry : tableNameToCells.asMap().entrySet()) {
     String tableName = entry.getKey();
     if (log.isInfoEnabled()) {
       log.info(
           "Attempting to immediately scrub "
               + entry.getValue().size()
               + " cells from table "
               + tableName);
     }
     for (List<Cell> cells : Iterables.partition(entry.getValue(), batchSizeSupplier.get())) {
       Multimap<Cell, Long> timestampsToDelete =
           HashMultimap.create(
               keyValueService.getAllTimestamps(
                   tableName, ImmutableSet.copyOf(cells), scrubTimestamp));
       for (Cell cell : ImmutableList.copyOf(timestampsToDelete.keySet())) {
         // Don't scrub garbage collection sentinels
         timestampsToDelete.remove(cell, Value.INVALID_VALUE_TIMESTAMP);
       }
       // If transactionType == TransactionType.AGGRESSIVE_HARD_DELETE this might
       // force other transactions to abort or retry
       deleteCellsAtTimestamps(txManager, tableName, timestampsToDelete, transactionType);
     }
     if (log.isInfoEnabled()) {
       log.info(
           "Immediately scrubbed " + entry.getValue().size() + " cells from table " + tableName);
     }
   }
 }
  /**
   * Only instances of {@code DynamicPartitionMapImpl} are supported by this implementation.
   *
   * @param partitionMap Must be instance of {@code DynamicPartitionMapImpl}.
   */
  @Override
  public synchronized void updateMap(DynamicPartitionMap partitionMap) {
    if (!(partitionMap instanceof DynamicPartitionMapImpl)) {
      throw new IllegalArgumentException(
          "Only instances of DynamicPartitionMapImpl are supported!");
    }

    DynamicPartitionMapImpl dpmi = (DynamicPartitionMapImpl) partitionMap;
    storage.createTable(PARTITION_MAP_TABLE, 1234);
    storage.truncateTable(PARTITION_MAP_TABLE);
    storage.put(PARTITION_MAP_TABLE, dpmi.toTable(), 0L);
  }
Esempio n. 3
0
 private void deleteCellsAtTimestamps(
     TransactionManager txManager,
     String tableName,
     Multimap<Cell, Long> cellToTimestamp,
     Transaction.TransactionType transactionType) {
   if (!cellToTimestamp.isEmpty()) {
     for (Follower follower : followers) {
       follower.run(txManager, tableName, cellToTimestamp.keySet(), transactionType);
     }
     keyValueService.addGarbageCollectionSentinelValues(tableName, cellToTimestamp.keySet());
     keyValueService.delete(tableName, cellToTimestamp);
   }
 }
Esempio n. 4
0
  private void sweepCells(
      String tableName, Multimap<Cell, Long> cellTsPairsToSweep, Set<Cell> sentinelsToAdd) {
    if (cellTsPairsToSweep.isEmpty()) {
      return;
    }

    for (Follower follower : followers) {
      follower.run(txManager, tableName, cellTsPairsToSweep.keySet(), TransactionType.HARD_DELETE);
    }
    if (!sentinelsToAdd.isEmpty()) {
      keyValueService.addGarbageCollectionSentinelValues(tableName, sentinelsToAdd);
    }
    keyValueService.delete(tableName, cellTsPairsToSweep);
  }
 private InKvsPartitionMapService(DynamicPartitionMapImpl partitionMap, KeyValueService storage) {
   this.storage = storage;
   storage.createTable(PARTITION_MAP_TABLE, 1234);
   if (partitionMap != null) {
     updateMap(partitionMap);
   }
 }
Esempio n. 6
0
 private long getCommitTimestampRollBackIfNecessary(
     long startTimestamp, Multimap<String, Cell> tableNameToCell) {
   Long commitTimestamp = transactionService.get(startTimestamp);
   if (commitTimestamp == null) {
     // Roll back this transaction (note that rolling back arbitrary transactions
     // can never cause correctness issues, only liveness issues)
     try {
       transactionService.putUnlessExists(startTimestamp, TransactionConstants.FAILED_COMMIT_TS);
     } catch (KeyAlreadyExistsException e) {
       String msg =
           "Could not roll back transaction with start timestamp "
               + startTimestamp
               + "; either"
               + " it was already rolled back (by a different transaction), or it committed successfully"
               + " before we could roll it back.";
       log.error(
           "This isn't a bug but it should be very infrequent. " + msg,
           new TransactionFailedRetriableException(msg, e));
     }
     commitTimestamp = transactionService.get(startTimestamp);
   }
   if (commitTimestamp == null) {
     throw new RuntimeException(
         "expected commit timestamp to be non-null for startTs: " + startTimestamp);
   }
   if (commitTimestamp == TransactionConstants.FAILED_COMMIT_TS) {
     for (String table : tableNameToCell.keySet()) {
       Map<Cell, Long> toDelete =
           Maps2.createConstantValueMap(tableNameToCell.get(table), startTimestamp);
       keyValueService.delete(table, Multimaps.forMap(toDelete));
     }
   }
   return commitTimestamp;
 }
 @Override
 public synchronized DynamicPartitionMapImpl getMap() {
   ClosableIterator<RowResult<Value>> iterator =
       storage.getRange(PARTITION_MAP_TABLE, RangeRequest.all(), 1L);
   Map<Cell, byte[]> cells = Maps.newHashMap();
   while (iterator.hasNext()) {
     RowResult<Value> row = iterator.next();
     for (Entry<Cell, Value> entry : row.getCells()) {
       assert !cells.containsKey(entry.getKey());
       cells.put(entry.getKey(), entry.getValue().getContents());
     }
   }
   iterator.close();
   return DynamicPartitionMapImpl.fromTable(cells);
 }
Esempio n. 8
0
  @Override
  public SweepResults run(String tableName, int batchSize, @Nullable byte[] startRow) {
    Preconditions.checkNotNull(tableName);
    Preconditions.checkState(!AtlasDbConstants.hiddenTables.contains(tableName));

    if (tableName.startsWith(AtlasDbConstants.NAMESPACE_PREFIX)) {
      // this happens sometimes; I think it's because some places in the code can
      // start this sweeper without doing the full normally ordered KVSModule startup.
      // I did check and sweep.stats did contain the FQ table name for all of the tables,
      // so it is at least broken in some way that still allows namespaced tables to eventually be
      // swept.
      log.warn("The sweeper should not be run on tables passed through namespace mapping.");
      return SweepResults.EMPTY_SWEEP;
    }

    // Earliest start timestamp of any currently open transaction, with two caveats:
    // (1) unreadableTimestamps are calculated via wall-clock time, and so may not be correct
    //     under pathological clock conditions
    // (2) immutableTimestamps do not account for locks have timed out after checking their locks;
    //     such a transaction may have a start timestamp less than the immutableTimestamp, and it
    //     could still get successfully committed (its commit timestamp may or may not be less than
    //     the immutableTimestamp
    // Note that this is fine, because we'll either
    // (1) force old readers to abort (if they read a garbage collection sentinel), or
    // (2) force old writers to retry (note that we must roll back any uncommitted transactions that
    //     we encounter
    SweepStrategy sweepStrategy = sweepStrategyManager.get().get(tableName);
    if (sweepStrategy == null) {
      sweepStrategy = SweepStrategy.CONSERVATIVE;
    } else if (sweepStrategy == SweepStrategy.NOTHING) {
      return SweepResults.EMPTY_SWEEP;
    }
    if (startRow == null) {
      startRow = new byte[0];
    }
    RangeRequest rangeRequest =
        RangeRequest.builder().startRowInclusive(startRow).batchHint(batchSize).build();

    long sweepTimestamp = getSweepTimestamp(sweepStrategy);
    ClosableIterator<RowResult<Value>> valueResults;
    if (sweepStrategy == SweepStrategy.CONSERVATIVE) {
      valueResults = ClosableIterators.wrap(ImmutableList.<RowResult<Value>>of().iterator());
    } else {
      valueResults = keyValueService.getRange(tableName, rangeRequest, sweepTimestamp);
    }

    ClosableIterator<RowResult<Set<Long>>> rowResults =
        keyValueService.getRangeOfTimestamps(tableName, rangeRequest, sweepTimestamp);

    try {
      List<RowResult<Set<Long>>> rowResultTimestamps =
          ImmutableList.copyOf(Iterators.limit(rowResults, batchSize));
      PeekingIterator<RowResult<Value>> peekingValues = Iterators.peekingIterator(valueResults);
      Set<Cell> sentinelsToAdd = Sets.newHashSet();
      Multimap<Cell, Long> rowTimestamps =
          getTimestampsFromRowResults(rowResultTimestamps, sweepStrategy);
      Multimap<Cell, Long> cellTsPairsToSweep =
          getCellTsPairsToSweep(
              rowTimestamps, peekingValues, sweepTimestamp, sweepStrategy, sentinelsToAdd);
      sweepCells(tableName, cellTsPairsToSweep, sentinelsToAdd);
      byte[] nextRow =
          rowResultTimestamps.size() < batchSize
              ? null
              : RangeRequests.getNextStartRow(
                  false, Iterables.getLast(rowResultTimestamps).getRowName());
      return new SweepResults(nextRow, rowResultTimestamps.size(), cellTsPairsToSweep.size());
    } finally {
      rowResults.close();
      valueResults.close();
    }
  }