public void testPrimaryOwnerCrash() throws Exception {
// cache 0 is the originator and backup, cache 1 is the primary owner
StateSequencer ss = new StateSequencer();
ss.logicalThread("main", "block_prepare", "crash_primary", "resume_prepare");
tm(0).begin();
cache(0).put("k", "v1");
DummyTransaction tx1 = (DummyTransaction) tm(0).suspend();
tx1.runPrepare();
advanceOnInboundRpc(ss, cache(1), matchCommand(PrepareCommand.class).build())
.before("block_prepare", "resume_prepare");
Future<DummyTransaction> tx2Future =
fork(
() -> {
tm(0).begin();
cache(0).put("k", "v2");
DummyTransaction tx2 = (DummyTransaction) tm(0).suspend();
tx2.runPrepare();
return tx2;
});
ss.enter("crash_primary");
killMember(1);
ss.exit("crash_primary");
DummyTransaction tx2 = tx2Future.get(10, SECONDS);
try {
tx2.runCommit(false);
fail("tx2 should not be able to commit");
} catch (Exception e) {
log.tracef(e, "Received expected exception");
}
tx1.runCommit(false);
}
public void testCommitDoesntWriteAfterRollback() throws Exception {
// Start a tx on A: put(k, v1), owners(k) = [B (primary) and C (backup)]
// Block the commit on C so that it times out
// Wait for the rollback command to be executed on B and C, and for the tx to end
// Check that locks are released on B
// Start another transaction on A: put(k, v2) with the same key
// Check that the new transaction writes successfully
// Allow the commit to proceed on C
// Check that k=v2 everywhere
StateSequencer sequencer = new StateSequencer();
sequencer.logicalThread(
"tx1",
"tx1:begin",
"tx1:block_commit_on_backup",
"tx1:after_rollback_on_primary",
"tx1:after_rollback_on_backup",
"tx1:resume_commit_on_backup",
"tx1:after_commit_on_backup",
"tx1:check");
sequencer.logicalThread("tx2", "tx2:begin", "tx2:end");
sequencer.order(
"tx1:after_rollback_on_backup", "tx2:begin", "tx2:end", "tx1:resume_commit_on_backup");
advanceOnInterceptor(
sequencer,
cache(2),
StateTransferInterceptor.class,
matchCommand(CommitCommand.class).matchCount(0).build())
.before("tx1:block_commit_on_backup", "tx1:resume_commit_on_backup")
.after("tx1:after_commit_on_backup");
advanceOnInterceptor(
sequencer,
cache(1),
StateTransferInterceptor.class,
matchCommand(RollbackCommand.class).build())
.after("tx1:after_rollback_on_primary");
advanceOnInterceptor(
sequencer,
cache(2),
StateTransferInterceptor.class,
matchCommand(RollbackCommand.class).build())
.after("tx1:after_rollback_on_backup");
assertEquals(
Arrays.asList(address(1), address(2)),
advancedCache(0).getDistributionManager().locate(TEST_KEY));
sequencer.advance("tx1:begin");
tm(0).begin();
cache(0).put(TEST_KEY, TX1_VALUE);
try {
tm(0).commit();
} catch (RollbackException e) {
log.debugf("Commit timed out as expected", e);
}
sequencer.advance("tx2:begin");
LockManager lockManager1 = TestingUtil.extractLockManager(cache(1));
assertFalse(lockManager1.isLocked(TEST_KEY));
tm(0).begin();
cache(0).put(TEST_KEY, TX2_VALUE);
tm(0).commit();
checkValue();
sequencer.advance("tx2:end");
sequencer.advance("tx1:check");
checkValue();
}
@Test(
enabled = false,
description = "Fix for this scenario is not implemented yet - rollback is asynchronous")
public void testCommitDoesntWriteAfterTxEnd() throws Exception {
// Start a tx on A: put(k, v1), owners(k) = [B (primary) and C (backup)]
// Block the commit on C so that it times out
// Wait for the rollback command to be executed on B and block before it executes on C
// Check that k is still locked on B
// Allow the commit to proceed on C
// Allow the rollback to proceed on C
// Check that k=v1 everywhere
// Check that locks are released on B
final StateSequencer sequencer = new StateSequencer();
sequencer.logicalThread(
"tx1",
"tx1:begin",
"tx1:block_commit_on_backup",
"tx1:after_rollback_on_primary",
"tx1:block_rollback_on_backup",
"tx1:resume_commit_on_backup",
"tx1:after_commit_on_backup",
"tx1:resume_rollback_on_backup",
"tx1:after_rollback_on_backup",
"tx1:check");
advanceOnInterceptor(
sequencer,
cache(2),
StateTransferInterceptor.class,
matchCommand(CommitCommand.class).matchCount(0).build())
.before("tx1:block_commit_on_backup", "tx1:resume_commit_on_backup")
.after("tx1:after_commit_on_backup");
advanceOnInterceptor(
sequencer,
cache(1),
StateTransferInterceptor.class,
matchCommand(RollbackCommand.class).build())
.after("tx1:after_rollback_on_primary");
advanceOnInterceptor(
sequencer,
cache(2),
StateTransferInterceptor.class,
matchCommand(RollbackCommand.class).build())
.before("tx1:block_rollback_on_backup")
.after("tx1:after_rollback_on_backup");
assertEquals(
Arrays.asList(address(1), address(2)),
advancedCache(0).getDistributionManager().locate(TEST_KEY));
Future<Object> lockCheckFuture =
fork(
new Callable<Object>() {
@Override
public Object call() throws Exception {
sequencer.enter("tx1:resume_rollback_on_backup");
try {
assertTrue(TestingUtil.extractLockManager(cache(1)).isLocked(TEST_KEY));
} finally {
sequencer.exit("tx1:resume_rollback_on_backup");
}
return null;
}
});
sequencer.advance("tx1:begin");
tm(0).begin();
cache(0).put(TEST_KEY, TX1_VALUE);
tm(0).commit();
sequencer.advance("tx1:check");
assertFalse(TestingUtil.extractLockManager(cache(1)).isLocked(TEST_KEY));
lockCheckFuture.get(10, TimeUnit.SECONDS);
}