/**
* When L1 is enabled this test should not be ran when a previous value is present as it will
* cause timeouts. Due to how locking works with L1 this cannot occur when the previous value
* exists.
*
* @param op
* @throws Exception
*/
protected void doTestWhereCommitOccursAfterStateTransferBeginsBeforeCompletion(
final TestWriteOperation op) throws Exception {
if (l1Enabled() && op.getPreviousValue() != null) {
fail("This test cannot be ran with L1 when a previous value is set");
}
// Test scenario:
// cache0,1,2 are in the cluster, an owner leaves
// Key k is in the cache, and is transferred to the non owner
// A user operation also modifies key k causing an invalidation
// on the non owner which is getting the state transfer
final AdvancedCache<Object, Object> primaryOwnerCache = cache(0, cacheName).getAdvancedCache();
final AdvancedCache<Object, Object> backupOwnerCache = cache(1, cacheName).getAdvancedCache();
final AdvancedCache<Object, Object> nonOwnerCache = cache(2, cacheName).getAdvancedCache();
final MagicKey key = new MagicKey(primaryOwnerCache, backupOwnerCache);
// Prepare for replace/remove: put a previous value in cache0
final Object previousValue = op.getPreviousValue();
if (previousValue != null) {
primaryOwnerCache.put(key, previousValue);
assertEquals(previousValue, primaryOwnerCache.get(key));
log.tracef("Previous value inserted: %s = %s", key, previousValue);
assertEquals(previousValue, nonOwnerCache.get(key));
if (l1Enabled()) {
assertIsInL1(nonOwnerCache, key);
}
}
int preJoinTopologyId =
primaryOwnerCache
.getComponentRegistry()
.getStateTransferManager()
.getCacheTopology()
.getTopologyId();
// Block any state response commands on cache0
CheckPoint checkPoint = new CheckPoint();
ControlledRpcManager blockingRpcManager0 = blockStateResponseCommand(primaryOwnerCache);
// Block the rebalance confirmation on cache0
blockRebalanceConfirmation(primaryOwnerCache.getCacheManager(), checkPoint);
assertEquals(
primaryOwnerCache.getCacheManager().getCoordinator(),
primaryOwnerCache.getCacheManager().getAddress());
// Remove the leaver
log.trace("Stopping the cache");
backupOwnerCache.getCacheManager().stop();
int rebalanceTopologyId = preJoinTopologyId + 2;
// Wait for the write CH to contain the joiner everywhere
eventually(
new Condition() {
@Override
public boolean isSatisfied() throws Exception {
return primaryOwnerCache.getRpcManager().getMembers().size() == 2
&& nonOwnerCache.getRpcManager().getMembers().size() == 2;
}
});
assertEquals(
primaryOwnerCache.getCacheManager().getCoordinator(),
primaryOwnerCache.getCacheManager().getAddress());
// Wait for cache0 to collect the state to send to cache1 (including our previous value).
blockingRpcManager0.waitForCommandToBlock();
// Every PutKeyValueCommand will be blocked before committing the entry on cache1
CyclicBarrier beforeCommitCache1Barrier = new CyclicBarrier(2);
BlockingInterceptor blockingInterceptor1 =
new BlockingInterceptor(beforeCommitCache1Barrier, op.getCommandClass(), true);
nonOwnerCache.addInterceptorAfter(blockingInterceptor1, EntryWrappingInterceptor.class);
// Put/Replace/Remove from cache0 with cache0 as primary owner, cache1 will become a backup
// owner for the retry
// The put command will be blocked on cache1 just before committing the entry.
Future<Object> future =
fork(
new Callable<Object>() {
@Override
public Object call() throws Exception {
return op.perform(primaryOwnerCache, key);
}
});
// Wait for the entry to be wrapped on cache1
beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);
// Remove the interceptor so we don't mess up any other state transfer puts
removeAllBlockingInterceptorsFromCache(nonOwnerCache);
// Allow the state to be applied on cache1 (writing the old value for our entry)
blockingRpcManager0.stopBlocking();
// Wait for second in line to finish applying the state, but don't allow the rebalance
// confirmation to be processed.
// (It would change the topology and it would trigger a retry for the command.)
checkPoint.awaitStrict(
"pre_rebalance_confirmation_"
+ rebalanceTopologyId
+ "_from_"
+ primaryOwnerCache.getCacheManager().getAddress(),
10,
SECONDS);
// Now allow the command to commit on cache1
beforeCommitCache1Barrier.await(10, TimeUnit.SECONDS);
// Wait for the command to finish and check that it didn't fail
Object result = future.get(10, TimeUnit.SECONDS);
assertEquals(op.getReturnValue(), result);
log.tracef("%s operation is done", op);
// Allow the rebalance confirmation to proceed and wait for the topology to change everywhere
checkPoint.trigger(
"resume_rebalance_confirmation_"
+ rebalanceTopologyId
+ "_from_"
+ primaryOwnerCache.getCacheManager().getAddress());
checkPoint.trigger(
"resume_rebalance_confirmation_"
+ rebalanceTopologyId
+ "_from_"
+ nonOwnerCache.getCacheManager().getAddress());
TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);
switch (op) {
case REMOVE:
case REMOVE_EXACT:
break;
default:
assertIsInContainerImmortal(primaryOwnerCache, key);
assertIsInContainerImmortal(nonOwnerCache, key);
break;
}
// Check the value to make sure data container contains correct value
assertEquals(op.getValue(), primaryOwnerCache.get(key));
assertEquals(op.getValue(), nonOwnerCache.get(key));
}
private void doL1InvalidationOldTopologyComesAfterRebalance(final TestWriteOperation op)
throws Exception {
final String key = getClass().getName() + "-key";
// Test scenario:
// cache0,1,2 are in the cluster, an owner leaves
// Key k is in the cache, and is transferred to the non owner
// A user operation also modifies key k causing an invalidation
// on the non owner which is getting the state transfer
final AdvancedCache<Object, Object> primaryOwnerCache = getFirstOwner(key).getAdvancedCache();
final AdvancedCache<Object, Object> backupOwnerCache = getOwners(key)[1].getAdvancedCache();
final AdvancedCache<Object, Object> nonOwnerCache = getFirstNonOwner(key).getAdvancedCache();
// Prepare for replace/remove: put a previous value in cache0
final Object previousValue = op.getPreviousValue();
if (previousValue != null) {
primaryOwnerCache.put(key, previousValue);
assertEquals(previousValue, primaryOwnerCache.get(key));
log.tracef("Previous value inserted: %s = %s", key, previousValue);
assertEquals(previousValue, nonOwnerCache.get(key));
if (l1Enabled()) {
assertIsInL1(nonOwnerCache, key);
}
}
// Block on the interceptor right after ST which should now have the soon to be old topology id
CyclicBarrier beforeCommitCache1Barrier = new CyclicBarrier(2);
BlockingInterceptor blockingInterceptor1 =
new BlockingInterceptor(beforeCommitCache1Barrier, getVisitableCommand(op), false);
primaryOwnerCache.addInterceptorAfter(blockingInterceptor1, StateTransferInterceptor.class);
// Put/Replace/Remove from primary owner. This will block before it is committing on remote
// nodes
Future<Object> future =
fork(
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
return op.perform(primaryOwnerCache, key);
} finally {
log.tracef("%s operation is done", op);
}
}
});
beforeCommitCache1Barrier.await(10, SECONDS);
// Remove blocking interceptor now since we have blocked
removeAllBlockingInterceptorsFromCache(primaryOwnerCache);
// Remove the leaver
log.tracef("Stopping the cache");
backupOwnerCache.getCacheManager().stop();
// Wait for the write CH to contain the joiner everywhere
eventually(
new Condition() {
@Override
public boolean isSatisfied() throws Exception {
return primaryOwnerCache.getRpcManager().getMembers().size() == 2
&& nonOwnerCache.getRpcManager().getMembers().size() == 2;
}
});
TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);
// Now let the update go through
beforeCommitCache1Barrier.await(10, SECONDS);
// Run the update now that we are in the middle of a rebalance
assertEquals(op.getReturnValue(), future.get(10, SECONDS));
log.tracef("%s operation is done", op);
switch (op) {
case REMOVE:
case REMOVE_EXACT:
break;
default:
assertIsInContainerImmortal(primaryOwnerCache, key);
assertIsInContainerImmortal(nonOwnerCache, key);
break;
}
// Check the value to make sure data container contains correct value
assertEquals(op.getValue(), primaryOwnerCache.get(key));
assertEquals(op.getValue(), nonOwnerCache.get(key));
}
protected void doStateTransferInBetweenPrepareCommit(
final TestWriteOperation op, final boolean additionalValueOnNonOwner) throws Exception {
final String key = getClass().getName() + "-key";
// Test scenario:
// cache0,1,2 are in the cluster, an owner leaves
// Key k is in the cache, and is transferred to the non owner
// A user operation also modifies key k causing an invalidation
// on the non owner which is getting the state transfer
final AdvancedCache<Object, Object> primaryOwnerCache = getFirstOwner(key).getAdvancedCache();
final AdvancedCache<Object, Object> backupOwnerCache = getOwners(key)[1].getAdvancedCache();
final AdvancedCache<Object, Object> nonOwnerCache = getFirstNonOwner(key).getAdvancedCache();
// Prepare for replace/remove: put a previous value in cache0
final Object previousValue = op.getPreviousValue();
if (previousValue != null) {
primaryOwnerCache.put(key, previousValue);
assertEquals(previousValue, primaryOwnerCache.get(key));
log.tracef("Previous value inserted: %s = %s", key, previousValue);
assertEquals(previousValue, nonOwnerCache.get(key));
if (l1Enabled()) {
assertIsInL1(nonOwnerCache, key);
}
}
// Need to block after Prepare command was sent after it clears the StateTransferInterceptor
final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
try {
TransactionManager tm = primaryOwnerCache.getTransactionManager();
Future<Object> future =
fork(
runWithTx(
tm,
new Callable<Object>() {
@Override
public Object call() throws Exception {
if (additionalValueOnNonOwner) {
MagicKey mk = new MagicKey("placeholder", nonOwnerCache);
String value = "somevalue";
primaryOwnerCache.put(mk, value);
log.tracef(
"Adding additional value on nonOwner value inserted: %s = %s",
mk, value);
}
primaryOwnerCache
.getAdvancedCache()
.addInterceptorBefore(
new BlockingInterceptor(cyclicBarrier, getVisitableCommand(op), true),
StateTransferInterceptor.class);
return op.perform(primaryOwnerCache, key);
}
}));
cyclicBarrier.await(10, SECONDS);
// Block the rebalance confirmation on nonOwnerCache
CheckPoint checkPoint = new CheckPoint();
log.trace("Adding proxy to state transfer");
waitUntilStateBeingTransferred(nonOwnerCache, checkPoint);
backupOwnerCache.getCacheManager().stop();
// Wait for non owner to just about get state
checkPoint.awaitStrict("pre_state_apply_invoked_for_" + nonOwnerCache, 10, SECONDS);
// let prepare complete and thus commit command invalidating on nonOwner
cyclicBarrier.await(10, SECONDS);
assertEquals(op.getReturnValue(), future.get(10, SECONDS));
// let state transfer go
checkPoint.trigger("pre_state_apply_release_for_" + nonOwnerCache);
TestingUtil.waitForRehashToComplete(primaryOwnerCache, nonOwnerCache);
switch (op) {
case REMOVE:
case REMOVE_EXACT:
break;
default:
assertIsInContainerImmortal(primaryOwnerCache, key);
assertIsInContainerImmortal(nonOwnerCache, key);
break;
}
// Check the value to make sure data container contains correct value
assertEquals(op.getValue(), primaryOwnerCache.get(key));
assertEquals(op.getValue(), nonOwnerCache.get(key));
} finally {
removeAllBlockingInterceptorsFromCache(primaryOwnerCache);
}
}