/* Insert 100 records begins with the beginKey. */
private void doWork(Environment master, String dbName, int beginKey) throws Exception {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
/* Insert/Update the records of the database. */
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 0; i < 100; i++) {
IntegerBinding.intToEntry(beginKey + i, key);
StringBinding.stringToEntry("herococo", data);
db.put(null, key, data);
}
db.close();
/*
* Do a sync at the end of the stage to make sure master and
* replica have the same data set.
*/
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
}
/**
* [#18882] Before this bug fix, this test would result in a java.io.FileNotFoundException out of
* FeederReader$SwitchWindow.fillNext.
*/
@Test
public void testDataInWriteQueue() throws Exception {
openGroup();
ExecutorService appThreads = Executors.newFixedThreadPool(numThreads);
int opsPerThread = numRecords / numThreads;
for (int i = 0; i < numThreads; i++) {
appThreads.execute(new AppWork(i, opsPerThread));
}
appThreads.shutdown();
appThreads.awaitTermination(6000, TimeUnit.SECONDS);
VLSN vlsn = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
RepTestUtils.checkNodeEquality(vlsn, verbose, repEnvInfo);
closeGroup();
}
/**
* Since the master never dies in this test, no rollbacks should occur, but no data should be lost
* either.
*
* <p>TODO: Should the workload param be of a different class (not a RollbackWorkload), since its
* masterSteadyWork method is only called here?
*/
private void replicasDieAndRejoin(RollbackWorkload workload, int numIterations) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group. Assign identities. */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
RepEnvInfo replicaA = null;
RepEnvInfo replicaB = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv().getState().isMaster()) {
continue;
}
if (replicaA == null) {
replicaA = info;
} else {
replicaB = info;
}
}
/*
* For the sake of easy test writing, make sure numIterations is an
* even number.
*/
assertTrue((numIterations % 2) == 0);
replicaA.abnormalCloseEnv();
for (int i = 0; i < numIterations; i++) {
workload.masterSteadyWork(master);
waitForReplicaToSync(master, repEnvInfo);
if ((i % 2) == 0) {
flushLogAndCrash(replicaB);
replicaA.openEnv();
} else {
flushLogAndCrash(replicaA);
replicaB.openEnv();
}
waitForReplicaToSync(master, repEnvInfo);
}
replicaA.openEnv();
VLSN lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
assertTrue(workload.containsAllData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
workload.close();
for (RepEnvInfo repi : repEnvInfo) {
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
repi.abnormalCloseEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
/**
* Create 3 nodes and replicate operations. Kill off the master, and make the other two resume.
* This will require a syncup and a rollback of any operations after the matchpoint.
*/
private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
/*
* Run a workload against the master. Sync up the group and check
* that all nodes have the same contents. This first workload must
* end with in-progress, uncommitted transactions.
*/
workload.beforeMasterCrash(master);
VLSN lastVLSN = VLSN.NULL_VLSN;
if (workload.noLockConflict()) {
lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload);
}
/*
* Crash the master, find a new master.
*/
RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
master = crashMasterAndElectNewMaster(master, repEnvInfo);
RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
logger.severe("newmaster=" + master);
RepEnvInfo alwaysReplica = null;
for (RepEnvInfo info : repEnvInfo) {
if ((info != oldMaster) && (info != newMaster)) {
alwaysReplica = info;
break;
}
}
/*
* Check that the remaining two nodes only contain committed
* updates.
* TODO: check that the number of group members is 2.
*/
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Do some work against the new master, while the old master is
* asleep. Note that the first workload may have contained
* in-flight transactions, so this may result in the rollback of
* some transactions in the first workload.
*/
workload.afterMasterCrashBeforeResumption(master);
/*
* The intent of this test is that the work after crash will end on
* an incomplete txn. Check for that.
*/
lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);
/* Now bring up the old master. */
logger.info("Bring up old master");
oldMaster.openEnv();
logger.info("Old master joined");
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
logger.info("Old master synced");
/*
* Check that all nodes only contain committed updates.
*/
workload.releaseDbLocks();
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Now crash the node that has never been a master. Do some work
* without it, then recover that node, then do a verification
* check. This exercises the recovery of a log that has syncups in
* it.
*/
alwaysReplica.abnormalCloseEnv();
workload.afterReplicaCrash(master);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN);
alwaysReplica.openEnv();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
RepTestUtils.checkUtilizationProfile(repEnvInfo);
workload.close();
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
for (RepEnvInfo repi : repEnvInfo) {
repi.abnormalCloseEnv();
}
/*
* Open and verify the environments one last time, to ensure that
* rollbacks in the recovery interval don't cause problems.
*/
master = RepTestUtils.restartGroup(repEnvInfo);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/* Final close. */
for (RepEnvInfo repi : repEnvInfo) {
repi.closeEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
/**
* This is really multiple tests in one. It tests network restore with a replica in each of the
* following three states:
*
* <p>1) A brand new node joining the group and needing a network restore.
*
* <p>2) An existing node with its own unique log needing a network restore.
*
* <p>3) Repeated network restores, reflecting a mature node.
*/
@Test
public void testBasic() throws DatabaseException, Exception {
/*
* The cleaner thread can see InsufficientLogExceptions so just stifle
* those exceptions from stderr.
*/
DaemonThread.stifleExceptionChatter = true;
configureForMaxCleaning(2);
final RepEnvInfo info1 = repEnvInfo[0];
RepEnvInfo info2 = repEnvInfo[1];
ReplicatedEnvironment masterRep = info1.openEnv();
Environment menv = masterRep;
EnvironmentMutableConfig mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
menv.setMutableConfig(mconfig);
/*
* Have just the master join first. We do this to test the special case
* of a brand new node joining a group and needing VLSN 1. The same
* node then rejoins with its VLSN > 1 to test subsequent rejoins
* where the node has already participated in the replication.
*/
populateDB(masterRep, TEST_DB_NAME, 100);
mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true");
menv.setMutableConfig(mconfig);
File cenvDir = info2.getEnvHome();
final int cid = 2;
for (int i = 0; i < RESTORE_CYCLES; i++) {
leaveGroupAllButMaster();
shiftVLSNRight(masterRep);
RepNodeImpl memberPrev =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/* Node1 is not known on the first iteration. */
final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN();
try {
/* Should force a network restore. */
setExceptionListener(info2);
info2.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
RepNodeImpl member =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/*
* The sync state should have been advanced to help contribute
* to the global CBVLSN and prevent it from advancing.
*/
final VLSN currSync = member.getBarrierState().getLastCBVLSN();
assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0);
NetworkRestore networkRestore = new NetworkRestore();
networkRestore.execute(e, new NetworkRestoreConfig());
final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
assertThat(stats.getExpectedBytes(), greaterThan(0));
assertThat(stats.getTransferredBytes(), greaterThan(0));
/* Create a replacement replicator. */
info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1);
setExceptionListener(info2);
info2.openEnv();
}
/* Verify that we can continue with the "restored" log files. */
populateDB(masterRep, TEST_DB_NAME, 100, 100);
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
info2.closeEnv();
}
}