@Test
public void testConfigError() {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
RepEnvInfo nrInfo = repEnvInfo[1];
nrInfo.closeEnv();
shiftVLSNRight(repEnvInfo[0].getEnv());
try {
setExceptionListener(nrInfo);
nrInfo.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
NetworkRestore networkRestore = new NetworkRestore();
final ReplicationConfig repConfig = repEnvInfo[0].getRepConfig();
// bad node name
repConfig.setNodeName("badname");
ReplicationNode restoreNode = new RepNodeImpl(repConfig);
final NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setLogProviders(Arrays.asList(restoreNode));
try {
networkRestore.execute(e, config);
fail("exception expected");
} catch (IllegalArgumentException iae) {
// Expected
}
}
}
@Test
public void testLogProviders() throws Exception {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
/* Add a secondary node */
repEnvInfo = RepTestUtils.setupExtendEnvInfo(repEnvInfo, 1);
final RepEnvInfo sInfo = repEnvInfo[repEnvInfo.length - 1];
sInfo.getRepConfig().setNodeType(NodeType.SECONDARY);
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
/* The node that will be use for network restores. */
RepEnvInfo nrInfo = repEnvInfo[1];
/* restore from master. */
doAndCheckRestore(nrInfo, minfo);
/* Check restore from specific Replica. */
doAndCheckRestore(nrInfo, repEnvInfo[2]);
/* restore from self should fail. */
try {
doAndCheckRestore(nrInfo, repEnvInfo[1]);
fail("exception expected");
} catch (EnvironmentFailureException e) {
// Expected. Cannot restore from just yourself.
}
/* Restore secondary */
doAndCheckRestore(sInfo, minfo);
/* Restore from secondary */
doAndCheckRestore(nrInfo, sInfo);
}
@Test
public void testDefaultJoinGroupHelper() throws UnknownMasterException, DatabaseException {
for (int i = 0; i < repEnvInfo.length; i++) {
RepEnvInfo ri = repEnvInfo[i];
if ((i + 1) == repEnvInfo.length) {
/* Use a non-master helper for the last replicator. */
ReplicationConfig config = RepTestUtils.createRepConfig((short) (i + 1));
String hpPairs = "";
// Skip the master, use all the other nodes
for (int j = 1; j < i; j++) {
hpPairs += "," + repEnvInfo[j].getRepConfig().getNodeHostPort();
}
hpPairs = hpPairs.substring(1);
config.setHelperHosts(hpPairs);
File envHome = ri.getEnvHome();
ri =
repEnvInfo[i] =
new RepEnvInfo(
envHome, config, RepTestUtils.createEnvConfig(Durability.COMMIT_SYNC));
}
ri.openEnv();
State state = ri.getEnv().getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
}
}
@Test
public void testDefaultJoinGroup() throws UnknownMasterException, DatabaseException {
createGroup();
ReplicatedEnvironment masterRep = repEnvInfo[0].getEnv();
assertEquals(State.MASTER, masterRep.getState());
leaveGroupAllButMaster();
/* Populates just the master. */
CommitToken ct = populateDB(masterRep, TEST_DB_NAME, 100);
/* Replicas should have caught up when they re-open their handles. */
for (RepEnvInfo ri : repEnvInfo) {
ReplicatedEnvironment rep = (ri.getEnv() == null) ? ri.openEnv() : ri.getEnv();
VLSN repVLSN = RepInternal.getRepImpl(rep).getVLSNIndex().getRange().getLast();
assertTrue(new VLSN(ct.getVLSN()).compareTo(repVLSN) <= 0);
}
}
/**
* Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
* specified, the bounced node will also be upgraded.
*/
private void bounceMaster(final int appVersion) throws Exception {
/* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == masterEnv) {
/*
* Sync up the replication group so that node2 doesn't do
* hard recovery.
*/
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Disable replay on replicas. */
shutdownFeeder(info.getRepNode(), replicaEnv1);
shutdownFeeder(info.getRepNode(), replicaEnv2);
/* Close the master. */
masterApp.close();
masterApp = null;
info.closeEnv();
masterEnv = null;
/* Force repEnvInfo[2] to the master. */
WaitForMasterListener masterWaiter = new WaitForMasterListener();
replicaEnv2.setStateChangeListener(masterWaiter);
RepNode repNode = repEnvInfo[2].getRepNode();
repNode.forceMaster(true);
/* Enable the LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
masterWaiter.awaitMastership();
assertTrue(repNode.isMaster());
masterEnv = replicaEnv2;
/* Replica2 was elected, swap names with replica1. */
final ReplicatedEnvironment tmpEnv = replicaEnv1;
replicaEnv1 = replicaEnv2;
replicaEnv2 = tmpEnv;
final AppInterface tmpApp = replicaApp1;
replicaApp1 = replicaApp2;
replicaApp2 = tmpApp;
/* Replica1 (or 2, see above) has been elected master. */
masterApp = newAppObject(appVersion);
masterApp.adopt(replicaApp1);
/* Former master (just upgraded) becomes replica1. */
replicaEnv1 = info.openEnv();
replicaApp1.open(replicaEnv1);
break;
}
}
assertNotNull(masterApp);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
/**
* Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
* node will also be upgraded.
*/
private void bounceReplica2(final int appVersion) throws Exception {
replicaApp2.close();
replicaApp2 = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == replicaEnv2) {
info.closeEnv();
replicaEnv2 = info.openEnv();
replicaApp2 = newAppObject(appVersion);
replicaApp2.open(replicaEnv2);
break;
}
}
assertNotNull(replicaApp2);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
@Override
public void run() {
try {
ReplicatedEnvironment repenv =
replicator.openEnv(
new CommitPointConsistencyPolicy(
commitToken, RepTestUtils.MINUTE_MS, TimeUnit.MILLISECONDS));
assertEquals(ReplicatedEnvironment.State.REPLICA, repenv.getState());
ReplicatedEnvironmentStats stats = replicator.getEnv().getRepStats(StatsConfig.DEFAULT);
assertEquals(1, stats.getTrackerVLSNConsistencyWaits());
assertTrue(stats.getTrackerVLSNConsistencyWaitMs() > 0);
} catch (UnknownMasterException e) {
testException = e;
throw new RuntimeException(e);
} catch (DatabaseException e) {
testException = e;
throw new RuntimeException(e);
}
}
public void openEnv() {
try {
Durability durability =
new Durability(
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.ReplicaAckPolicy.ALL);
EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
ReplicationConfig repConfig = RepTestUtils.createRepConfig(1);
repEnvInfo = RepTestUtils.setupEnvInfo(envHome, envConfig, repConfig, null);
repEnvInfo.openEnv();
Thread.sleep(sleepTime);
} catch (EnvironmentLockedException e) {
/*
* Exit the process with value 1, don't print out the exception
* since it's expected.
*/
System.exit(1);
} catch (UnsupportedOperationException e) {
/*
* Exit the process with value 2, don't print out the exception
* since it's expected.
*
* Note: this exception thrown because we can't start a
* replicated Environment on an existed standalone Environment.
*/
System.exit(2);
} catch (Exception e) {
/* Dump unexpected exceptions, exit processs with value 3. */
e.printStackTrace();
System.exit(3);
} finally {
if (repEnvInfo.getEnv() != null) {
repEnvInfo.closeEnv();
}
}
}
private void flushLogAndCrash(RepEnvInfo replica) {
DbInternal.getEnvironmentImpl(replica.getEnv()).getLogManager().flush();
replica.abnormalCloseEnv();
}
/**
* Since the master never dies in this test, no rollbacks should occur, but no data should be lost
* either.
*
* <p>TODO: Should the workload param be of a different class (not a RollbackWorkload), since its
* masterSteadyWork method is only called here?
*/
private void replicasDieAndRejoin(RollbackWorkload workload, int numIterations) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group. Assign identities. */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
RepEnvInfo replicaA = null;
RepEnvInfo replicaB = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv().getState().isMaster()) {
continue;
}
if (replicaA == null) {
replicaA = info;
} else {
replicaB = info;
}
}
/*
* For the sake of easy test writing, make sure numIterations is an
* even number.
*/
assertTrue((numIterations % 2) == 0);
replicaA.abnormalCloseEnv();
for (int i = 0; i < numIterations; i++) {
workload.masterSteadyWork(master);
waitForReplicaToSync(master, repEnvInfo);
if ((i % 2) == 0) {
flushLogAndCrash(replicaB);
replicaA.openEnv();
} else {
flushLogAndCrash(replicaA);
replicaB.openEnv();
}
waitForReplicaToSync(master, repEnvInfo);
}
replicaA.openEnv();
VLSN lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
assertTrue(workload.containsAllData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
workload.close();
for (RepEnvInfo repi : repEnvInfo) {
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
repi.abnormalCloseEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}