/** Crash the current master, and wait until the group elects a new one. */
private ReplicatedEnvironment crashMasterAndElectNewMaster(
ReplicatedEnvironment master, RepEnvInfo[] repEnvInfo) {
int masterIndex = RepInternal.getNodeId(master) - 1;
logger.info("Crashing " + master.getNodeName());
repEnvInfo[masterIndex].abnormalCloseEnv();
logger.info("Rejoining");
ReplicatedEnvironment newMaster = RepTestUtils.openRepEnvsJoin(repEnvInfo);
logger.info("New master = " + newMaster.getNodeName());
return newMaster;
}
/**
* Create 3 nodes and replicate operations. Kill off the master, and make the other two resume.
* This will require a syncup and a rollback of any operations after the matchpoint.
*/
private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
/*
* Run a workload against the master. Sync up the group and check
* that all nodes have the same contents. This first workload must
* end with in-progress, uncommitted transactions.
*/
workload.beforeMasterCrash(master);
VLSN lastVLSN = VLSN.NULL_VLSN;
if (workload.noLockConflict()) {
lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload);
}
/*
* Crash the master, find a new master.
*/
RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
master = crashMasterAndElectNewMaster(master, repEnvInfo);
RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
logger.severe("newmaster=" + master);
RepEnvInfo alwaysReplica = null;
for (RepEnvInfo info : repEnvInfo) {
if ((info != oldMaster) && (info != newMaster)) {
alwaysReplica = info;
break;
}
}
/*
* Check that the remaining two nodes only contain committed
* updates.
* TODO: check that the number of group members is 2.
*/
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Do some work against the new master, while the old master is
* asleep. Note that the first workload may have contained
* in-flight transactions, so this may result in the rollback of
* some transactions in the first workload.
*/
workload.afterMasterCrashBeforeResumption(master);
/*
* The intent of this test is that the work after crash will end on
* an incomplete txn. Check for that.
*/
lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);
/* Now bring up the old master. */
logger.info("Bring up old master");
oldMaster.openEnv();
logger.info("Old master joined");
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
logger.info("Old master synced");
/*
* Check that all nodes only contain committed updates.
*/
workload.releaseDbLocks();
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Now crash the node that has never been a master. Do some work
* without it, then recover that node, then do a verification
* check. This exercises the recovery of a log that has syncups in
* it.
*/
alwaysReplica.abnormalCloseEnv();
workload.afterReplicaCrash(master);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN);
alwaysReplica.openEnv();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
RepTestUtils.checkUtilizationProfile(repEnvInfo);
workload.close();
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
for (RepEnvInfo repi : repEnvInfo) {
repi.abnormalCloseEnv();
}
/*
* Open and verify the environments one last time, to ensure that
* rollbacks in the recovery interval don't cause problems.
*/
master = RepTestUtils.restartGroup(repEnvInfo);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/* Final close. */
for (RepEnvInfo repi : repEnvInfo) {
repi.closeEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}