@Test
public void testConfigError() {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
RepEnvInfo nrInfo = repEnvInfo[1];
nrInfo.closeEnv();
shiftVLSNRight(repEnvInfo[0].getEnv());
try {
setExceptionListener(nrInfo);
nrInfo.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
NetworkRestore networkRestore = new NetworkRestore();
final ReplicationConfig repConfig = repEnvInfo[0].getRepConfig();
// bad node name
repConfig.setNodeName("badname");
ReplicationNode restoreNode = new RepNodeImpl(repConfig);
final NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setLogProviders(Arrays.asList(restoreNode));
try {
networkRestore.execute(e, config);
fail("exception expected");
} catch (IllegalArgumentException iae) {
// Expected
}
}
}
@Test
public void testDefaultJoinGroupHelper() throws UnknownMasterException, DatabaseException {
for (int i = 0; i < repEnvInfo.length; i++) {
RepEnvInfo ri = repEnvInfo[i];
if ((i + 1) == repEnvInfo.length) {
/* Use a non-master helper for the last replicator. */
ReplicationConfig config = RepTestUtils.createRepConfig((short) (i + 1));
String hpPairs = "";
// Skip the master, use all the other nodes
for (int j = 1; j < i; j++) {
hpPairs += "," + repEnvInfo[j].getRepConfig().getNodeHostPort();
}
hpPairs = hpPairs.substring(1);
config.setHelperHosts(hpPairs);
File envHome = ri.getEnvHome();
ri =
repEnvInfo[i] =
new RepEnvInfo(
envHome, config, RepTestUtils.createEnvConfig(Durability.COMMIT_SYNC));
}
ri.openEnv();
State state = ri.getEnv().getState();
assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
}
}
@Test
public void testLogProviders() throws Exception {
configureForMaxCleaning(5);
final RepEnvInfo minfo = repEnvInfo[0];
/* Add a secondary node */
repEnvInfo = RepTestUtils.setupExtendEnvInfo(repEnvInfo, 1);
final RepEnvInfo sInfo = repEnvInfo[repEnvInfo.length - 1];
sInfo.getRepConfig().setNodeType(NodeType.SECONDARY);
createGroup();
populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
/* The node that will be use for network restores. */
RepEnvInfo nrInfo = repEnvInfo[1];
/* restore from master. */
doAndCheckRestore(nrInfo, minfo);
/* Check restore from specific Replica. */
doAndCheckRestore(nrInfo, repEnvInfo[2]);
/* restore from self should fail. */
try {
doAndCheckRestore(nrInfo, repEnvInfo[1]);
fail("exception expected");
} catch (EnvironmentFailureException e) {
// Expected. Cannot restore from just yourself.
}
/* Restore secondary */
doAndCheckRestore(sInfo, minfo);
/* Restore from secondary */
doAndCheckRestore(nrInfo, sInfo);
}
/**
* Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
* specified, the bounced node will also be upgraded.
*/
private void bounceMaster(final int appVersion) throws Exception {
/* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == masterEnv) {
/*
* Sync up the replication group so that node2 doesn't do
* hard recovery.
*/
RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
/* Disable replay on replicas. */
shutdownFeeder(info.getRepNode(), replicaEnv1);
shutdownFeeder(info.getRepNode(), replicaEnv2);
/* Close the master. */
masterApp.close();
masterApp = null;
info.closeEnv();
masterEnv = null;
/* Force repEnvInfo[2] to the master. */
WaitForMasterListener masterWaiter = new WaitForMasterListener();
replicaEnv2.setStateChangeListener(masterWaiter);
RepNode repNode = repEnvInfo[2].getRepNode();
repNode.forceMaster(true);
/* Enable the LocalCBVLSN updates. */
LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
masterWaiter.awaitMastership();
assertTrue(repNode.isMaster());
masterEnv = replicaEnv2;
/* Replica2 was elected, swap names with replica1. */
final ReplicatedEnvironment tmpEnv = replicaEnv1;
replicaEnv1 = replicaEnv2;
replicaEnv2 = tmpEnv;
final AppInterface tmpApp = replicaApp1;
replicaApp1 = replicaApp2;
replicaApp2 = tmpApp;
/* Replica1 (or 2, see above) has been elected master. */
masterApp = newAppObject(appVersion);
masterApp.adopt(replicaApp1);
/* Former master (just upgraded) becomes replica1. */
replicaEnv1 = info.openEnv();
replicaApp1.open(replicaEnv1);
break;
}
}
assertNotNull(masterApp);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
/**
* Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
* node will also be upgraded.
*/
private void bounceReplica2(final int appVersion) throws Exception {
replicaApp2.close();
replicaApp2 = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv() == replicaEnv2) {
info.closeEnv();
replicaEnv2 = info.openEnv();
replicaApp2 = newAppObject(appVersion);
replicaApp2.open(replicaEnv2);
break;
}
}
assertNotNull(replicaApp2);
assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
}
@Test
public void testDefaultJoinGroup() throws UnknownMasterException, DatabaseException {
createGroup();
ReplicatedEnvironment masterRep = repEnvInfo[0].getEnv();
assertEquals(State.MASTER, masterRep.getState());
leaveGroupAllButMaster();
/* Populates just the master. */
CommitToken ct = populateDB(masterRep, TEST_DB_NAME, 100);
/* Replicas should have caught up when they re-open their handles. */
for (RepEnvInfo ri : repEnvInfo) {
ReplicatedEnvironment rep = (ri.getEnv() == null) ? ri.openEnv() : ri.getEnv();
VLSN repVLSN = RepInternal.getRepImpl(rep).getVLSNIndex().getRange().getLast();
assertTrue(new VLSN(ct.getVLSN()).compareTo(repVLSN) <= 0);
}
}
/*
* Tests internal node removal APIs.
*/
@Test
public void testRemoveMember() {
createGroup(groupSize);
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
assertTrue(master.getState().isMaster());
RepNode masterRep = repEnvInfo[0].getRepNode();
/* Reduce the group size all the way down to one. */
for (int i = 1; i < groupSize; i++) {
assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
masterRep.removeMember(repEnvInfo[i].getEnv().getNodeName());
assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
}
/* Close the replica handles*/
for (int i = groupSize - 1; i > 0; i--) {
repEnvInfo[i].closeEnv();
}
/* Attempting to re-open them with the same node names should fail. */
for (int i = 1; i < groupSize; i++) {
try {
repEnvInfo[i].openEnv();
fail("Exception expected");
} catch (EnvironmentFailureException e) {
/* Expected, the master should reject the attempt. */
assertEquals(EnvironmentFailureReason.HANDSHAKE_ERROR, e.getReason());
}
}
/* Doing the same but with different node names should be ok. */
for (int i = 1; i < groupSize; i++) {
final RepEnvInfo ri = repEnvInfo[i];
final ReplicationConfig repConfig = ri.getRepConfig();
TestUtils.removeLogFiles("RemoveRepEnvironments", ri.getEnvHome(), false);
repConfig.setNodeName("ReplaceNode_" + i);
ri.openEnv();
assertEquals(i + 1, masterRep.getGroup().getElectableGroupSize());
}
master.close();
}
/*
* Creates conditions for a network restore at nrInfo and then restores the
* node from a specific member.
*/
private void doAndCheckRestore(RepEnvInfo nrInfo, RepEnvInfo restoreFromInfo) {
nrInfo.closeEnv();
shiftVLSNRight(repEnvInfo[0].getEnv());
try {
nrInfo.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
NetworkRestore networkRestore = new NetworkRestore();
ReplicationNode restoreNode = new RepNodeImpl(restoreFromInfo.getRepConfig());
final NetworkRestoreConfig config = new NetworkRestoreConfig();
config.setLogProviders(Arrays.asList(restoreNode));
networkRestore.execute(e, config);
assertEquals(restoreNode, networkRestore.getLogProvider());
final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
assertThat(stats.getExpectedBytes(), greaterThan(0));
assertThat(stats.getTransferredBytes(), greaterThan(0));
nrInfo.openEnv();
}
}
@Override
public void run() {
try {
ReplicatedEnvironment repenv =
replicator.openEnv(
new CommitPointConsistencyPolicy(
commitToken, RepTestUtils.MINUTE_MS, TimeUnit.MILLISECONDS));
assertEquals(ReplicatedEnvironment.State.REPLICA, repenv.getState());
ReplicatedEnvironmentStats stats = replicator.getEnv().getRepStats(StatsConfig.DEFAULT);
assertEquals(1, stats.getTrackerVLSNConsistencyWaits());
assertTrue(stats.getTrackerVLSNConsistencyWaitMs() > 0);
} catch (UnknownMasterException e) {
testException = e;
throw new RuntimeException(e);
} catch (DatabaseException e) {
testException = e;
throw new RuntimeException(e);
}
}
public void openEnv() {
try {
Durability durability =
new Durability(
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.SyncPolicy.WRITE_NO_SYNC,
Durability.ReplicaAckPolicy.ALL);
EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
ReplicationConfig repConfig = RepTestUtils.createRepConfig(1);
repEnvInfo = RepTestUtils.setupEnvInfo(envHome, envConfig, repConfig, null);
repEnvInfo.openEnv();
Thread.sleep(sleepTime);
} catch (EnvironmentLockedException e) {
/*
* Exit the process with value 1, don't print out the exception
* since it's expected.
*/
System.exit(1);
} catch (UnsupportedOperationException e) {
/*
* Exit the process with value 2, don't print out the exception
* since it's expected.
*
* Note: this exception thrown because we can't start a
* replicated Environment on an existed standalone Environment.
*/
System.exit(2);
} catch (Exception e) {
/* Dump unexpected exceptions, exit processs with value 3. */
e.printStackTrace();
System.exit(3);
} finally {
if (repEnvInfo.getEnv() != null) {
repEnvInfo.closeEnv();
}
}
}
private void close(boolean normalShutdown) {
try {
if (normalShutdown) {
replicaApp1.close();
replicaApp2.close();
masterApp.close();
RepTestUtils.shutdownRepEnvs(repEnvInfo);
} else {
for (RepEnvInfo info : repEnvInfo) {
info.abnormalCloseEnv();
}
}
} finally {
repEnvInfo = null;
masterEnv = null;
replicaEnv1 = null;
replicaEnv2 = null;
masterApp = null;
replicaApp1 = null;
replicaApp2 = null;
}
}
private void flushLogAndCrash(RepEnvInfo replica) {
DbInternal.getEnvironmentImpl(replica.getEnv()).getLogManager().flush();
replica.abnormalCloseEnv();
}
/**
* Since the master never dies in this test, no rollbacks should occur, but no data should be lost
* either.
*
* <p>TODO: Should the workload param be of a different class (not a RollbackWorkload), since its
* masterSteadyWork method is only called here?
*/
private void replicasDieAndRejoin(RollbackWorkload workload, int numIterations) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group. Assign identities. */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
RepEnvInfo replicaA = null;
RepEnvInfo replicaB = null;
for (RepEnvInfo info : repEnvInfo) {
if (info.getEnv().getState().isMaster()) {
continue;
}
if (replicaA == null) {
replicaA = info;
} else {
replicaB = info;
}
}
/*
* For the sake of easy test writing, make sure numIterations is an
* even number.
*/
assertTrue((numIterations % 2) == 0);
replicaA.abnormalCloseEnv();
for (int i = 0; i < numIterations; i++) {
workload.masterSteadyWork(master);
waitForReplicaToSync(master, repEnvInfo);
if ((i % 2) == 0) {
flushLogAndCrash(replicaB);
replicaA.openEnv();
} else {
flushLogAndCrash(replicaA);
replicaB.openEnv();
}
waitForReplicaToSync(master, repEnvInfo);
}
replicaA.openEnv();
VLSN lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
assertTrue(workload.containsAllData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
workload.close();
for (RepEnvInfo repi : repEnvInfo) {
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
repi.abnormalCloseEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
/**
* Create 3 nodes and replicate operations. Kill off the master, and make the other two resume.
* This will require a syncup and a rollback of any operations after the matchpoint.
*/
private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable {
RepEnvInfo[] repEnvInfo = null;
try {
/* Create a 3 node group */
repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
logger.severe("master=" + master);
/*
* Run a workload against the master. Sync up the group and check
* that all nodes have the same contents. This first workload must
* end with in-progress, uncommitted transactions.
*/
workload.beforeMasterCrash(master);
VLSN lastVLSN = VLSN.NULL_VLSN;
if (workload.noLockConflict()) {
lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload);
}
/*
* Crash the master, find a new master.
*/
RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
master = crashMasterAndElectNewMaster(master, repEnvInfo);
RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
logger.severe("newmaster=" + master);
RepEnvInfo alwaysReplica = null;
for (RepEnvInfo info : repEnvInfo) {
if ((info != oldMaster) && (info != newMaster)) {
alwaysReplica = info;
break;
}
}
/*
* Check that the remaining two nodes only contain committed
* updates.
* TODO: check that the number of group members is 2.
*/
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Do some work against the new master, while the old master is
* asleep. Note that the first workload may have contained
* in-flight transactions, so this may result in the rollback of
* some transactions in the first workload.
*/
workload.afterMasterCrashBeforeResumption(master);
/*
* The intent of this test is that the work after crash will end on
* an incomplete txn. Check for that.
*/
lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);
/* Now bring up the old master. */
logger.info("Bring up old master");
oldMaster.openEnv();
logger.info("Old master joined");
RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
logger.info("Old master synced");
/*
* Check that all nodes only contain committed updates.
*/
workload.releaseDbLocks();
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/*
* Now crash the node that has never been a master. Do some work
* without it, then recover that node, then do a verification
* check. This exercises the recovery of a log that has syncups in
* it.
*/
alwaysReplica.abnormalCloseEnv();
workload.afterReplicaCrash(master);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN);
alwaysReplica.openEnv();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
assertTrue(workload.containsSavedData(master));
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
RepTestUtils.checkUtilizationProfile(repEnvInfo);
workload.close();
/*
* We're done with the test. Bringing down these replicators
* forcibly, without closing transactions and whatnot.
*/
for (RepEnvInfo repi : repEnvInfo) {
repi.abnormalCloseEnv();
}
/*
* Open and verify the environments one last time, to ensure that
* rollbacks in the recovery interval don't cause problems.
*/
master = RepTestUtils.restartGroup(repEnvInfo);
lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
/* Final close. */
for (RepEnvInfo repi : repEnvInfo) {
repi.closeEnv();
}
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
@SuppressWarnings("null")
@Test
public void testBasic() throws InterruptedException {
final int dataNodeSize = groupSize - 1;
final int electableNodeSize = groupSize - 2;
final int persistentNodeSize = groupSize - 1;
final ReplicationConfig sConfig = repEnvInfo[groupSize - 2].getRepConfig();
sConfig.setNodeType(NodeType.SECONDARY);
createGroup(dataNodeSize);
ReplicationConfig rConfig = repEnvInfo[groupSize - 1].getRepConfig();
/* RepNetConfig needs to come from an open environment */
ReplicationConfig r0Config = repEnvInfo[0].getEnv().getRepConfig();
rConfig.setNodeType(NodeType.MONITOR);
MonitorConfig monConfig = new MonitorConfig();
monConfig.setNodeName(rConfig.getNodeName());
monConfig.setGroupName(rConfig.getGroupName());
monConfig.setNodeHostPort(rConfig.getNodeHostPort());
monConfig.setHelperHosts(rConfig.getHelperHosts());
monConfig.setRepNetConfig(r0Config.getRepNetConfig());
new Monitor(monConfig).register();
for (int i = 0; i < dataNodeSize; i++) {
final ReplicatedEnvironment env = repEnvInfo[i].getEnv();
final boolean isMaster = (env.getState() == MASTER);
final int targetGroupSize = isMaster ? groupSize : persistentNodeSize;
ReplicationGroup group = null;
for (int j = 0; j < 100; j++) {
group = env.getGroup();
if (group.getNodes().size() == targetGroupSize) {
break;
}
/* Wait for the replica to catch up. */
Thread.sleep(1000);
}
assertEquals("Nodes", targetGroupSize, group.getNodes().size());
assertEquals(RepTestUtils.TEST_REP_GROUP_NAME, group.getName());
logger.info(group.toString());
for (RepEnvInfo rinfo : repEnvInfo) {
final ReplicationConfig repConfig = rinfo.getRepConfig();
ReplicationNode member = group.getMember(repConfig.getNodeName());
if (!isMaster && repConfig.getNodeType().isSecondary()) {
assertNull("Member", member);
} else {
assertNotNull("Member", member);
assertEquals(repConfig.getNodeName(), member.getName());
assertEquals(repConfig.getNodeType(), member.getType());
assertEquals(repConfig.getNodeSocketAddress(), member.getSocketAddress());
}
}
final Set<ReplicationNode> electableNodes = group.getElectableNodes();
for (final ReplicationNode n : electableNodes) {
assertEquals(NodeType.ELECTABLE, n.getType());
}
assertEquals("Electable nodes", electableNodeSize, electableNodes.size());
final Set<ReplicationNode> monitorNodes = group.getMonitorNodes();
for (final ReplicationNode n : monitorNodes) {
assertEquals(NodeType.MONITOR, n.getType());
}
assertEquals("Monitor nodes", 1, monitorNodes.size());
final Set<ReplicationNode> secondaryNodes = group.getSecondaryNodes();
for (final ReplicationNode n : secondaryNodes) {
assertEquals(NodeType.SECONDARY, n.getType());
}
assertEquals("Secondary nodes", isMaster ? 1 : 0, secondaryNodes.size());
final Set<ReplicationNode> dataNodes = group.getDataNodes();
for (final ReplicationNode n : dataNodes) {
if (isMaster) {
assertThat(n.getType(), anyOf(is(NodeType.ELECTABLE), is(NodeType.SECONDARY)));
} else {
assertEquals(NodeType.ELECTABLE, n.getType());
}
}
assertEquals("Data nodes", isMaster ? dataNodeSize : electableNodeSize, dataNodes.size());
}
}
private void setExceptionListener(final RepEnvInfo info) {
EnvironmentConfig infoConfig = info.getEnvConfig();
infoConfig.setExceptionListener(new NetworkTestExceptionListener());
info.setEnvConfig(infoConfig);
}
/**
* This is really multiple tests in one. It tests network restore with a replica in each of the
* following three states:
*
* <p>1) A brand new node joining the group and needing a network restore.
*
* <p>2) An existing node with its own unique log needing a network restore.
*
* <p>3) Repeated network restores, reflecting a mature node.
*/
@Test
public void testBasic() throws DatabaseException, Exception {
/*
* The cleaner thread can see InsufficientLogExceptions so just stifle
* those exceptions from stderr.
*/
DaemonThread.stifleExceptionChatter = true;
configureForMaxCleaning(2);
final RepEnvInfo info1 = repEnvInfo[0];
RepEnvInfo info2 = repEnvInfo[1];
ReplicatedEnvironment masterRep = info1.openEnv();
Environment menv = masterRep;
EnvironmentMutableConfig mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
menv.setMutableConfig(mconfig);
/*
* Have just the master join first. We do this to test the special case
* of a brand new node joining a group and needing VLSN 1. The same
* node then rejoins with its VLSN > 1 to test subsequent rejoins
* where the node has already participated in the replication.
*/
populateDB(masterRep, TEST_DB_NAME, 100);
mconfig = menv.getMutableConfig();
mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true");
menv.setMutableConfig(mconfig);
File cenvDir = info2.getEnvHome();
final int cid = 2;
for (int i = 0; i < RESTORE_CYCLES; i++) {
leaveGroupAllButMaster();
shiftVLSNRight(masterRep);
RepNodeImpl memberPrev =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/* Node1 is not known on the first iteration. */
final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN();
try {
/* Should force a network restore. */
setExceptionListener(info2);
info2.openEnv();
fail("exception expected");
} catch (InsufficientLogException e) {
RepNodeImpl member =
info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
/*
* The sync state should have been advanced to help contribute
* to the global CBVLSN and prevent it from advancing.
*/
final VLSN currSync = member.getBarrierState().getLastCBVLSN();
assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0);
NetworkRestore networkRestore = new NetworkRestore();
networkRestore.execute(e, new NetworkRestoreConfig());
final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
assertThat(stats.getExpectedBytes(), greaterThan(0));
assertThat(stats.getTransferredBytes(), greaterThan(0));
/* Create a replacement replicator. */
info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1);
setExceptionListener(info2);
info2.openEnv();
}
/* Verify that we can continue with the "restored" log files. */
populateDB(masterRep, TEST_DB_NAME, 100, 100);
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
info2.closeEnv();
}
}