コード例 #1
0
  /**
   * When L1 is enabled this test should not be ran when a previous value is present as it will
   * cause timeouts. Due to how locking works with L1 this cannot occur when the previous value
   * exists.
   *
   * @param op
   * @throws Exception
   */
  protected void doTestWhereCommitOccursAfterStateTransferBeginsBeforeCompletion(
      final TestWriteOperation op) throws Exception {
    if (l1Enabled() && op.getPreviousValue() != null) {
      fail("This test cannot be ran with L1 when a previous value is set");
    }
    // Test scenario:
    // cache0,1,2 are in the cluster, an owner leaves
    // Key k is in the cache, and is transferred to the non owner
    // A user operation also modifies key k causing an invalidation
    // on the non owner which is getting the state transfer
    final AdvancedCache<Object, Object> primaryOwnerCache = cache(0, cacheName).getAdvancedCache();
    final AdvancedCache<Object, Object> backupOwnerCache = cache(1, cacheName).getAdvancedCache();
    final AdvancedCache<Object, Object> nonOwnerCache = cache(2, cacheName).getAdvancedCache();

    final MagicKey key = new MagicKey(primaryOwnerCache, backupOwnerCache);

    // Prepare for replace/remove: put a previous value in cache0
    final Object previousValue = op.getPreviousValue();
    if (previousValue != null) {
      primaryOwnerCache.put(key, previousValue);
      assertEquals(previousValue, primaryOwnerCache.get(key));
      log.tracef("Previous value inserted: %s = %s", key, previousValue);

      assertEquals(previousValue, nonOwnerCache.get(key));

      if (l1Enabled()) {
        assertIsInL1(nonOwnerCache, key);
      }
    }

    int preJoinTopologyId =
        primaryOwnerCache
            .getComponentRegistry()
            .getStateTransferManager()
            .getCacheTopology()
            .getTopologyId();

    // Block any state response commands on cache0
    CheckPoint checkPoint = new CheckPoint();
    ControlledRpcManager blockingRpcManager0 = blockStateResponseCommand(primaryOwnerCache);

    // Block the rebalance confirmation on cache0
    blockRebalanceConfirmation(primaryOwnerCache.getCacheManager(), checkPoint);

    assertEquals(
        primaryOwnerCache.getCacheManager().getCoordinator(),
        primaryOwnerCache.getCacheManager().getAddress());

    // Remove the leaver
    log.trace("Stopping the cache");
    backupOwnerCache.getCacheManager().stop();

    int rebalanceTopologyId = preJoinTopologyId + 2;

    // Wait for the write CH to contain the joiner everywhere
    eventually(
        new Condition() {
          @Override
          public boolean isSatisfied() throws Exception {
            return primaryOwnerCache.getRpcManager().getMembers().size() == 2
                && nonOwnerCache.getRpcManager().getMembers().size() == 2;
          }
        });

    assertEquals(
        primaryOwnerCache.getCacheManager().getCoordinator(),
        primaryOwnerCache.getCacheManager().getAddress());

    // Wait for cache0 to collect the state to send to cache1 (including our previous value).
    blockingRpcManager0.waitForCommandToBlock();

    // Every PutKeyValueCommand will be blocked before committing the entry on cache1
    CyclicBarrier beforeCommitCache1Barrier = new CyclicBarrier(2);
    BlockingInterceptor blockingInterceptor1 =
        new BlockingInterceptor(beforeCommitCache1Barrier, op.getCommandClass(), true);
    nonOwnerCache.addInterceptorAfter(blockingInterceptor1, EntryWrappingInterceptor.class);

    // Put/Replace/Remove from cache0 with cache0 as primary owner, cache1 will become a backup
    // owner for the retry
    // The put command will be blocked on cache1 just before committing the entry.
    Future<Object> future =
        fork(
            new Callable<Object>() {
              @Override
              public Object call() throws Exception {
                return op.perform(primaryOwnerCache, key);
              }
            });

    // Wait for the entry to be wrapped on cache1
    beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);

    // Remove the interceptor so we don't mess up any other state transfer puts
    removeAllBlockingInterceptorsFromCache(nonOwnerCache);

    // Allow the state to be applied on cache1 (writing the old value for our entry)
    blockingRpcManager0.stopBlocking();

    // Wait for second in line to finish applying the state, but don't allow the rebalance
    // confirmation to be processed.
    // (It would change the topology and it would trigger a retry for the command.)
    checkPoint.awaitStrict(
        "pre_rebalance_confirmation_"
            + rebalanceTopologyId
            + "_from_"
            + primaryOwnerCache.getCacheManager().getAddress(),
        10,
        SECONDS);

    // Now allow the command to commit on cache1
    beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);

    // Wait for the command to finish and check that it didn't fail
    Object result = future.get(10, TimeUnit.SECONDS);
    assertEquals(op.getReturnValue(), result);
    log.tracef("%s operation is done", op);

    // Allow the rebalance confirmation to proceed and wait for the topology to change everywhere
    checkPoint.trigger(
        "resume_rebalance_confirmation_"
            + rebalanceTopologyId
            + "_from_"
            + primaryOwnerCache.getCacheManager().getAddress());
    checkPoint.trigger(
        "resume_rebalance_confirmation_"
            + rebalanceTopologyId
            + "_from_"
            + nonOwnerCache.getCacheManager().getAddress());
    TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);

    switch (op) {
      case REMOVE:
      case REMOVE_EXACT:
        break;
      default:
        assertIsInContainerImmortal(primaryOwnerCache, key);
        assertIsInContainerImmortal(nonOwnerCache, key);
        break;
    }

    // Check the value to make sure data container contains correct value
    assertEquals(op.getValue(), primaryOwnerCache.get(key));
    assertEquals(op.getValue(), nonOwnerCache.get(key));
  }
コード例 #2
0
  private void doL1InvalidationOldTopologyComesAfterRebalance(final TestWriteOperation op)
      throws Exception {
    final String key = getClass().getName() + "-key";
    // Test scenario:
    // cache0,1,2 are in the cluster, an owner leaves
    // Key k is in the cache, and is transferred to the non owner
    // A user operation also modifies key k causing an invalidation
    // on the non owner which is getting the state transfer
    final AdvancedCache<Object, Object> primaryOwnerCache = getFirstOwner(key).getAdvancedCache();
    final AdvancedCache<Object, Object> backupOwnerCache = getOwners(key)[1].getAdvancedCache();
    final AdvancedCache<Object, Object> nonOwnerCache = getFirstNonOwner(key).getAdvancedCache();

    // Prepare for replace/remove: put a previous value in cache0
    final Object previousValue = op.getPreviousValue();
    if (previousValue != null) {
      primaryOwnerCache.put(key, previousValue);
      assertEquals(previousValue, primaryOwnerCache.get(key));
      log.tracef("Previous value inserted: %s = %s", key, previousValue);

      assertEquals(previousValue, nonOwnerCache.get(key));

      if (l1Enabled()) {
        assertIsInL1(nonOwnerCache, key);
      }
    }

    // Block on the interceptor right after ST which should now have the soon to be old topology id
    CyclicBarrier beforeCommitCache1Barrier = new CyclicBarrier(2);
    BlockingInterceptor blockingInterceptor1 =
        new BlockingInterceptor(beforeCommitCache1Barrier, getVisitableCommand(op), false);
    primaryOwnerCache.addInterceptorAfter(blockingInterceptor1, StateTransferInterceptor.class);

    // Put/Replace/Remove from primary owner.  This will block before it is committing on remote
    // nodes
    Future<Object> future =
        fork(
            new Callable<Object>() {
              @Override
              public Object call() throws Exception {
                try {
                  return op.perform(primaryOwnerCache, key);
                } finally {
                  log.tracef("%s operation is done", op);
                }
              }
            });

    beforeCommitCache1Barrier.await(10, SECONDS);

    // Remove blocking interceptor now since we have blocked
    removeAllBlockingInterceptorsFromCache(primaryOwnerCache);

    // Remove the leaver
    log.tracef("Stopping the cache");
    backupOwnerCache.getCacheManager().stop();

    // Wait for the write CH to contain the joiner everywhere
    eventually(
        new Condition() {
          @Override
          public boolean isSatisfied() throws Exception {
            return primaryOwnerCache.getRpcManager().getMembers().size() == 2
                && nonOwnerCache.getRpcManager().getMembers().size() == 2;
          }
        });

    TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);

    // Now let the update go through
    beforeCommitCache1Barrier.await(10, SECONDS);

    // Run the update now that we are in the middle of a rebalance
    assertEquals(op.getReturnValue(), future.get(10, SECONDS));
    log.tracef("%s operation is done", op);

    switch (op) {
      case REMOVE:
      case REMOVE_EXACT:
        break;
      default:
        assertIsInContainerImmortal(primaryOwnerCache, key);
        assertIsInContainerImmortal(nonOwnerCache, key);
        break;
    }

    // Check the value to make sure data container contains correct value
    assertEquals(op.getValue(), primaryOwnerCache.get(key));
    assertEquals(op.getValue(), nonOwnerCache.get(key));
  }
コード例 #3
0
  protected void doStateTransferInBetweenPrepareCommit(
      final TestWriteOperation op, final boolean additionalValueOnNonOwner) throws Exception {
    final String key = getClass().getName() + "-key";
    // Test scenario:
    // cache0,1,2 are in the cluster, an owner leaves
    // Key k is in the cache, and is transferred to the non owner
    // A user operation also modifies key k causing an invalidation
    // on the non owner which is getting the state transfer
    final AdvancedCache<Object, Object> primaryOwnerCache = getFirstOwner(key).getAdvancedCache();
    final AdvancedCache<Object, Object> backupOwnerCache = getOwners(key)[1].getAdvancedCache();
    final AdvancedCache<Object, Object> nonOwnerCache = getFirstNonOwner(key).getAdvancedCache();

    // Prepare for replace/remove: put a previous value in cache0
    final Object previousValue = op.getPreviousValue();
    if (previousValue != null) {
      primaryOwnerCache.put(key, previousValue);
      assertEquals(previousValue, primaryOwnerCache.get(key));
      log.tracef("Previous value inserted: %s = %s", key, previousValue);

      assertEquals(previousValue, nonOwnerCache.get(key));

      if (l1Enabled()) {
        assertIsInL1(nonOwnerCache, key);
      }
    }

    // Need to block after Prepare command was sent after it clears the StateTransferInterceptor
    final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);

    try {
      TransactionManager tm = primaryOwnerCache.getTransactionManager();
      Future<Object> future =
          fork(
              runWithTx(
                  tm,
                  new Callable<Object>() {

                    @Override
                    public Object call() throws Exception {
                      if (additionalValueOnNonOwner) {
                        MagicKey mk = new MagicKey("placeholder", nonOwnerCache);
                        String value = "somevalue";
                        primaryOwnerCache.put(mk, value);
                        log.tracef(
                            "Adding additional value on nonOwner value inserted: %s = %s",
                            mk, value);
                      }
                      primaryOwnerCache
                          .getAdvancedCache()
                          .addInterceptorBefore(
                              new BlockingInterceptor(cyclicBarrier, getVisitableCommand(op), true),
                              StateTransferInterceptor.class);
                      return op.perform(primaryOwnerCache, key);
                    }
                  }));

      cyclicBarrier.await(10, SECONDS);

      // Block the rebalance confirmation on nonOwnerCache
      CheckPoint checkPoint = new CheckPoint();
      log.trace("Adding proxy to state transfer");
      waitUntilStateBeingTransferred(nonOwnerCache, checkPoint);

      backupOwnerCache.getCacheManager().stop();

      // Wait for non owner to just about get state
      checkPoint.awaitStrict("pre_state_apply_invoked_for_" + nonOwnerCache, 10, SECONDS);

      // let prepare complete and thus commit command invalidating on nonOwner
      cyclicBarrier.await(10, SECONDS);

      assertEquals(op.getReturnValue(), future.get(10, SECONDS));

      // let state transfer go
      checkPoint.trigger("pre_state_apply_release_for_" + nonOwnerCache);

      TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);

      switch (op) {
        case REMOVE:
        case REMOVE_EXACT:
          break;
        default:
          assertIsInContainerImmortal(primaryOwnerCache, key);
          assertIsInContainerImmortal(nonOwnerCache, key);
          break;
      }

      // Check the value to make sure data container contains correct value
      assertEquals(op.getValue(), primaryOwnerCache.get(key));
      assertEquals(op.getValue(), nonOwnerCache.get(key));
    } finally {
      removeAllBlockingInterceptorsFromCache(primaryOwnerCache);
    }
  }