public void testPrimaryOwnerCrash() throws Exception {
    // cache 0 is the originator and backup, cache 1 is the primary owner
    StateSequencer ss = new StateSequencer();
    ss.logicalThread("main", "block_prepare", "crash_primary", "resume_prepare");

    tm(0).begin();
    cache(0).put("k", "v1");
    DummyTransaction tx1 = (DummyTransaction) tm(0).suspend();
    tx1.runPrepare();

    advanceOnInboundRpc(ss, cache(1), matchCommand(PrepareCommand.class).build())
        .before("block_prepare", "resume_prepare");

    Future<DummyTransaction> tx2Future =
        fork(
            () -> {
              tm(0).begin();
              cache(0).put("k", "v2");
              DummyTransaction tx2 = (DummyTransaction) tm(0).suspend();
              tx2.runPrepare();
              return tx2;
            });

    ss.enter("crash_primary");
    killMember(1);
    ss.exit("crash_primary");

    DummyTransaction tx2 = tx2Future.get(10, SECONDS);
    try {
      tx2.runCommit(false);
      fail("tx2 should not be able to commit");
    } catch (Exception e) {
      log.tracef(e, "Received expected exception");
    }

    tx1.runCommit(false);
  }
  public void testCommitDoesntWriteAfterRollback() throws Exception {
    // Start a tx on A: put(k, v1), owners(k) = [B (primary) and C (backup)]
    // Block the commit on C so that it times out
    // Wait for the rollback command to be executed on B and C, and for the tx to end
    // Check that locks are released on B
    // Start another transaction on A: put(k, v2) with the same key
    // Check that the new transaction writes successfully
    // Allow the commit to proceed on C
    // Check that k=v2 everywhere
    StateSequencer sequencer = new StateSequencer();
    sequencer.logicalThread(
        "tx1",
        "tx1:begin",
        "tx1:block_commit_on_backup",
        "tx1:after_rollback_on_primary",
        "tx1:after_rollback_on_backup",
        "tx1:resume_commit_on_backup",
        "tx1:after_commit_on_backup",
        "tx1:check");
    sequencer.logicalThread("tx2", "tx2:begin", "tx2:end");

    sequencer.order(
        "tx1:after_rollback_on_backup", "tx2:begin", "tx2:end", "tx1:resume_commit_on_backup");

    advanceOnInterceptor(
            sequencer,
            cache(2),
            StateTransferInterceptor.class,
            matchCommand(CommitCommand.class).matchCount(0).build())
        .before("tx1:block_commit_on_backup", "tx1:resume_commit_on_backup")
        .after("tx1:after_commit_on_backup");

    advanceOnInterceptor(
            sequencer,
            cache(1),
            StateTransferInterceptor.class,
            matchCommand(RollbackCommand.class).build())
        .after("tx1:after_rollback_on_primary");

    advanceOnInterceptor(
            sequencer,
            cache(2),
            StateTransferInterceptor.class,
            matchCommand(RollbackCommand.class).build())
        .after("tx1:after_rollback_on_backup");

    assertEquals(
        Arrays.asList(address(1), address(2)),
        advancedCache(0).getDistributionManager().locate(TEST_KEY));
    sequencer.advance("tx1:begin");

    tm(0).begin();
    cache(0).put(TEST_KEY, TX1_VALUE);
    try {
      tm(0).commit();
    } catch (RollbackException e) {
      log.debugf("Commit timed out as expected", e);
    }

    sequencer.advance("tx2:begin");
    LockManager lockManager1 = TestingUtil.extractLockManager(cache(1));
    assertFalse(lockManager1.isLocked(TEST_KEY));

    tm(0).begin();
    cache(0).put(TEST_KEY, TX2_VALUE);
    tm(0).commit();

    checkValue();
    sequencer.advance("tx2:end");

    sequencer.advance("tx1:check");
    checkValue();
  }
  @Test(
      enabled = false,
      description = "Fix for this scenario is not implemented yet - rollback is asynchronous")
  public void testCommitDoesntWriteAfterTxEnd() throws Exception {
    // Start a tx on A: put(k, v1), owners(k) = [B (primary) and C (backup)]
    // Block the commit on C so that it times out
    // Wait for the rollback command to be executed on B and block before it executes on C
    // Check that k is still locked on B
    // Allow the commit to proceed on C
    // Allow the rollback to proceed on C
    // Check that k=v1 everywhere
    // Check that locks are released on B
    final StateSequencer sequencer = new StateSequencer();
    sequencer.logicalThread(
        "tx1",
        "tx1:begin",
        "tx1:block_commit_on_backup",
        "tx1:after_rollback_on_primary",
        "tx1:block_rollback_on_backup",
        "tx1:resume_commit_on_backup",
        "tx1:after_commit_on_backup",
        "tx1:resume_rollback_on_backup",
        "tx1:after_rollback_on_backup",
        "tx1:check");

    advanceOnInterceptor(
            sequencer,
            cache(2),
            StateTransferInterceptor.class,
            matchCommand(CommitCommand.class).matchCount(0).build())
        .before("tx1:block_commit_on_backup", "tx1:resume_commit_on_backup")
        .after("tx1:after_commit_on_backup");

    advanceOnInterceptor(
            sequencer,
            cache(1),
            StateTransferInterceptor.class,
            matchCommand(RollbackCommand.class).build())
        .after("tx1:after_rollback_on_primary");

    advanceOnInterceptor(
            sequencer,
            cache(2),
            StateTransferInterceptor.class,
            matchCommand(RollbackCommand.class).build())
        .before("tx1:block_rollback_on_backup")
        .after("tx1:after_rollback_on_backup");

    assertEquals(
        Arrays.asList(address(1), address(2)),
        advancedCache(0).getDistributionManager().locate(TEST_KEY));
    Future<Object> lockCheckFuture =
        fork(
            new Callable<Object>() {
              @Override
              public Object call() throws Exception {
                sequencer.enter("tx1:resume_rollback_on_backup");
                try {
                  assertTrue(TestingUtil.extractLockManager(cache(1)).isLocked(TEST_KEY));
                } finally {
                  sequencer.exit("tx1:resume_rollback_on_backup");
                }
                return null;
              }
            });

    sequencer.advance("tx1:begin");

    tm(0).begin();
    cache(0).put(TEST_KEY, TX1_VALUE);
    tm(0).commit();

    sequencer.advance("tx1:check");
    assertFalse(TestingUtil.extractLockManager(cache(1)).isLocked(TEST_KEY));
    lockCheckFuture.get(10, TimeUnit.SECONDS);
  }