@Test
public void testConfigError() {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
RepEnvInfo nrInfo = repEnvInfo[1];
nrInfo.closeEnv();
shiftVLSNRight(repEnvInfo[0].getEnv());
try {
setExceptionListener(nrInfo);
nrInfo.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
NetworkRestore networkRestore = new NetworkRestore();
final ReplicationConfig repConfig = repEnvInfo[0].getRepConfig();
// bad node name
repConfig.setNodeName("badname");
ReplicationNode restoreNode = new RepNodeImpl(repConfig);
final NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setLogProviders(Arrays.asList(restoreNode));
try {
networkRestore.execute(e, config);
fail("exception expected");
} catch (IllegalArgumentException iae) {
// Expected
}
}
}
private void doReplicaHasGapNetworkRestore(boolean multiGaps) throws Throwable {
Durability noAck =
new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
db = null;
try {
Environment master = setupLogWithCleanedGaps(multiGaps);
int masterIdx = findMasterIndex(master);
/*
* Write a record, so that we are sure that there will be a
* network restore, because we have to cross a checkpoint.
*/
generateData(master, 1, noAck, false);
CheckpointConfig cc = new CheckpointConfig();
master.checkpoint(cc);
EnvironmentStats stats = master.getStats(clearConfig);
assertEquals(0, stats.getCleanerBacklog());
if (multiGaps) {
logger.info("Multigap: deletion backlog = " + stats.getFileDeletionBacklog());
} else {
assertEquals(0, stats.getFileDeletionBacklog());
}
db.close();
db = null;
repEnvInfo[masterIdx].closeEnv();
/* Start up the two replicas */
openReplicas(masterIdx);
/* Start the node that had been the master */
try {
repEnvInfo[masterIdx].openEnv();
fail("Should be a network restore");
} catch (InsufficientLogException ile) {
repEnvInfo[masterIdx].closeEnv();
NetworkRestore restore = new NetworkRestore();
NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setRetainLogFiles(true);
restore.execute(ile, config);
repEnvInfo[masterIdx].openEnv();
}
/* Check its last VLSN and size. */
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
/*
* Creates conditions for a network restore at nrInfo and then restores the
* node from a specific member.
*/
private void doAndCheckRestore(RepEnvInfo nrInfo, RepEnvInfo restoreFromInfo) {
nrInfo.closeEnv();
shiftVLSNRight(repEnvInfo[0].getEnv());
try {
nrInfo.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
NetworkRestore networkRestore = new NetworkRestore();
ReplicationNode restoreNode = new RepNodeImpl(restoreFromInfo.getRepConfig());
final NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setLogProviders(Arrays.asList(restoreNode));
networkRestore.execute(e, config);
assertEquals(restoreNode, networkRestore.getLogProvider());
final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
assertThat(stats.getExpectedBytes(), greaterThan(0));
assertThat(stats.getTransferredBytes(), greaterThan(0));
nrInfo.openEnv();
}
}
@Test
public void testReplicaHasGap() throws Throwable {
db = null;
try {
Environment master = setupLogWithCleanedGaps(false);
int masterIdx = findMasterIndex(master);
db.close();
db = null;
repEnvInfo[masterIdx].closeEnv();
/* Start up the two replicas */
openReplicas(masterIdx);
/* Start the master */
try {
repEnvInfo[masterIdx].openEnv();
} catch (InsufficientLogException ile) {
repEnvInfo[masterIdx].closeEnv();
NetworkRestore restore = new NetworkRestore();
NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setRetainLogFiles(true);
restore.execute(ile, config);
repEnvInfo[masterIdx].openEnv();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
} finally {
if (db != null) {
db.close();
}
RepTestUtils.shutdownRepEnvs(repEnvInfo);
}
}
/**
* This is really multiple tests in one. It tests network restore with a replica in each of the
* following three states:
*
* <p>1) A brand new node joining the group and needing a network restore.
*
* <p>2) An existing node with its own unique log needing a network restore.
*
* <p>3) Repeated network restores, reflecting a mature node.
*/
@Test
public void testBasic() throws DatabaseException, Exception {
/*
* The cleaner thread can see InsufficientLogExceptions so just stifle
* those exceptions from stderr.
*/
DaemonThread.stifleExceptionChatter = true;
configureForMaxCleaning(2);
final RepEnvInfo info1 = repEnvInfo[0];
RepEnvInfo info2 = repEnvInfo[1];
ReplicatedEnvironment masterRep = info1.openEnv();
Environment menv = masterRep;
EnvironmentMutableConfig mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
menv.setMutableConfig(mconfig);
/*
* Have just the master join first. We do this to test the special case
* of a brand new node joining a group and needing VLSN 1. The same
* node then rejoins with its VLSN > 1 to test subsequent rejoins
* where the node has already participated in the replication.
*/
populateDB(masterRep, TEST_DB_NAME, 100);
mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true");
menv.setMutableConfig(mconfig);
File cenvDir = info2.getEnvHome();
final int cid = 2;
for (int i = 0; i < RESTORE_CYCLES; i++) {
leaveGroupAllButMaster();
shiftVLSNRight(masterRep);
RepNodeImpl memberPrev =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/* Node1 is not known on the first iteration. */
final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN();
try {
/* Should force a network restore. */
setExceptionListener(info2);
info2.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
RepNodeImpl member =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/*
* The sync state should have been advanced to help contribute
* to the global CBVLSN and prevent it from advancing.
*/
final VLSN currSync = member.getBarrierState().getLastCBVLSN();
assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0);
NetworkRestore networkRestore = new NetworkRestore();
networkRestore.execute(e, new NetworkRestoreConfig());
final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
assertThat(stats.getExpectedBytes(), greaterThan(0));
assertThat(stats.getTransferredBytes(), greaterThan(0));
/* Create a replacement replicator. */
info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1);
setExceptionListener(info2);
info2.openEnv();
}
/* Verify that we can continue with the "restored" log files. */
populateDB(masterRep, TEST_DB_NAME, 100, 100);
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
info2.closeEnv();
}
}