@Test
  public void testConfigError() {
    configureForMaxCleaning(5);
    final RepEnvInfo minfo = repEnvInfo[0];

    createGroup();
    populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
    RepEnvInfo nrInfo = repEnvInfo[1];
    nrInfo.closeEnv();
    shiftVLSNRight(repEnvInfo[0].getEnv());
    try {
      setExceptionListener(nrInfo);
      nrInfo.openEnv();
      fail("exception expected");
    } catch (InsufficientLogException e) {
      NetworkRestore networkRestore = new NetworkRestore();

      final ReplicationConfig repConfig = repEnvInfo[0].getRepConfig();
      // bad node name
      repConfig.setNodeName("badname");
      ReplicationNode restoreNode = new RepNodeImpl(repConfig);
      final NetworkRestoreConfig config = new NetworkRestoreConfig();
      config.setLogProviders(Arrays.asList(restoreNode));
      try {
        networkRestore.execute(e, config);
        fail("exception expected");
      } catch (IllegalArgumentException iae) {
        // Expected
      }
    }
  }
  @Test
  public void testDefaultJoinGroupHelper() throws UnknownMasterException, DatabaseException {

    for (int i = 0; i < repEnvInfo.length; i++) {
      RepEnvInfo ri = repEnvInfo[i];
      if ((i + 1) == repEnvInfo.length) {
        /* Use a non-master helper for the last replicator. */
        ReplicationConfig config = RepTestUtils.createRepConfig((short) (i + 1));
        String hpPairs = "";
        // Skip the master, use all the other nodes
        for (int j = 1; j < i; j++) {
          hpPairs += "," + repEnvInfo[j].getRepConfig().getNodeHostPort();
        }
        hpPairs = hpPairs.substring(1);
        config.setHelperHosts(hpPairs);
        File envHome = ri.getEnvHome();
        ri =
            repEnvInfo[i] =
                new RepEnvInfo(
                    envHome, config, RepTestUtils.createEnvConfig(Durability.COMMIT_SYNC));
      }
      ri.openEnv();
      State state = ri.getEnv().getState();
      assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
    }
  }
  @Test
  public void testLogProviders() throws Exception {

    configureForMaxCleaning(5);
    final RepEnvInfo minfo = repEnvInfo[0];

    /* Add a secondary node */
    repEnvInfo = RepTestUtils.setupExtendEnvInfo(repEnvInfo, 1);
    final RepEnvInfo sInfo = repEnvInfo[repEnvInfo.length - 1];
    sInfo.getRepConfig().setNodeType(NodeType.SECONDARY);

    createGroup();
    populateDB(minfo.getEnv(), TEST_DB_NAME, 100);

    /* The node that will be use for network restores. */
    RepEnvInfo nrInfo = repEnvInfo[1];

    /* restore from master. */
    doAndCheckRestore(nrInfo, minfo);
    /* Check restore from specific Replica. */
    doAndCheckRestore(nrInfo, repEnvInfo[2]);
    /* restore from self should fail. */
    try {
      doAndCheckRestore(nrInfo, repEnvInfo[1]);
      fail("exception expected");
    } catch (EnvironmentFailureException e) {
      // Expected. Cannot restore from just yourself.
    }

    /* Restore secondary */
    doAndCheckRestore(sInfo, minfo);

    /* Restore from secondary */
    doAndCheckRestore(nrInfo, sInfo);
  }
예제 #4
0
  /**
   * Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
   * specified, the bounced node will also be upgraded.
   */
  private void bounceMaster(final int appVersion) throws Exception {

    /* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
    LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);

    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == masterEnv) {

        /*
         * Sync up the replication group so that node2 doesn't do
         * hard recovery.
         */
        RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
        /* Disable replay on replicas. */
        shutdownFeeder(info.getRepNode(), replicaEnv1);
        shutdownFeeder(info.getRepNode(), replicaEnv2);

        /* Close the master. */
        masterApp.close();
        masterApp = null;
        info.closeEnv();
        masterEnv = null;

        /* Force repEnvInfo[2] to the master. */
        WaitForMasterListener masterWaiter = new WaitForMasterListener();
        replicaEnv2.setStateChangeListener(masterWaiter);
        RepNode repNode = repEnvInfo[2].getRepNode();
        repNode.forceMaster(true);
        /* Enable the LocalCBVLSN updates. */
        LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
        masterWaiter.awaitMastership();
        assertTrue(repNode.isMaster());
        masterEnv = replicaEnv2;

        /* Replica2 was elected, swap names with replica1. */
        final ReplicatedEnvironment tmpEnv = replicaEnv1;
        replicaEnv1 = replicaEnv2;
        replicaEnv2 = tmpEnv;
        final AppInterface tmpApp = replicaApp1;
        replicaApp1 = replicaApp2;
        replicaApp2 = tmpApp;

        /* Replica1 (or 2, see above) has been elected master. */
        masterApp = newAppObject(appVersion);
        masterApp.adopt(replicaApp1);
        /* Former master (just upgraded) becomes replica1. */
        replicaEnv1 = info.openEnv();
        replicaApp1.open(replicaEnv1);
        break;
      }
    }
    assertNotNull(masterApp);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
예제 #5
0
  /**
   * Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
   * node will also be upgraded.
   */
  private void bounceReplica2(final int appVersion) throws Exception {

    replicaApp2.close();
    replicaApp2 = null;
    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == replicaEnv2) {
        info.closeEnv();
        replicaEnv2 = info.openEnv();
        replicaApp2 = newAppObject(appVersion);
        replicaApp2.open(replicaEnv2);
        break;
      }
    }
    assertNotNull(replicaApp2);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
  @Test
  public void testDefaultJoinGroup() throws UnknownMasterException, DatabaseException {

    createGroup();
    ReplicatedEnvironment masterRep = repEnvInfo[0].getEnv();
    assertEquals(State.MASTER, masterRep.getState());
    leaveGroupAllButMaster();
    /* Populates just the master. */
    CommitToken ct = populateDB(masterRep, TEST_DB_NAME, 100);

    /* Replicas should have caught up when they re-open their handles. */
    for (RepEnvInfo ri : repEnvInfo) {
      ReplicatedEnvironment rep = (ri.getEnv() == null) ? ri.openEnv() : ri.getEnv();
      VLSN repVLSN = RepInternal.getRepImpl(rep).getVLSNIndex().getRange().getLast();
      assertTrue(new VLSN(ct.getVLSN()).compareTo(repVLSN) <= 0);
    }
  }
  /*
   * Tests internal node removal APIs.
   */
  @Test
  public void testRemoveMember() {
    createGroup(groupSize);
    ReplicatedEnvironment master = repEnvInfo[0].getEnv();
    assertTrue(master.getState().isMaster());

    RepNode masterRep = repEnvInfo[0].getRepNode();

    /* Reduce the group size all the way down to one. */
    for (int i = 1; i < groupSize; i++) {
      assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
      masterRep.removeMember(repEnvInfo[i].getEnv().getNodeName());
      assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
    }

    /* Close the replica handles*/
    for (int i = groupSize - 1; i > 0; i--) {
      repEnvInfo[i].closeEnv();
    }

    /* Attempting to re-open them with the same node names should fail. */
    for (int i = 1; i < groupSize; i++) {
      try {
        repEnvInfo[i].openEnv();
        fail("Exception expected");
      } catch (EnvironmentFailureException e) {
        /* Expected, the master should reject the attempt. */
        assertEquals(EnvironmentFailureReason.HANDSHAKE_ERROR, e.getReason());
      }
    }

    /* Doing the same but with different node names should be ok. */
    for (int i = 1; i < groupSize; i++) {
      final RepEnvInfo ri = repEnvInfo[i];
      final ReplicationConfig repConfig = ri.getRepConfig();
      TestUtils.removeLogFiles("RemoveRepEnvironments", ri.getEnvHome(), false);

      repConfig.setNodeName("ReplaceNode_" + i);
      ri.openEnv();
      assertEquals(i + 1, masterRep.getGroup().getElectableGroupSize());
    }
    master.close();
  }
 /*
  * Creates conditions for a network restore at nrInfo and then restores the
  * node from a specific member.
  */
 private void doAndCheckRestore(RepEnvInfo nrInfo, RepEnvInfo restoreFromInfo) {
   nrInfo.closeEnv();
   shiftVLSNRight(repEnvInfo[0].getEnv());
   try {
     nrInfo.openEnv();
     fail("exception expected");
   } catch (InsufficientLogException e) {
     NetworkRestore networkRestore = new NetworkRestore();
     ReplicationNode restoreNode = new RepNodeImpl(restoreFromInfo.getRepConfig());
     final NetworkRestoreConfig config = new NetworkRestoreConfig();
     config.setLogProviders(Arrays.asList(restoreNode));
     networkRestore.execute(e, config);
     assertEquals(restoreNode, networkRestore.getLogProvider());
     final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
     assertThat(stats.getExpectedBytes(), greaterThan(0));
     assertThat(stats.getTransferredBytes(), greaterThan(0));
     nrInfo.openEnv();
   }
 }
    @Override
    public void run() {
      try {
        ReplicatedEnvironment repenv =
            replicator.openEnv(
                new CommitPointConsistencyPolicy(
                    commitToken, RepTestUtils.MINUTE_MS, TimeUnit.MILLISECONDS));
        assertEquals(ReplicatedEnvironment.State.REPLICA, repenv.getState());
        ReplicatedEnvironmentStats stats = replicator.getEnv().getRepStats(StatsConfig.DEFAULT);

        assertEquals(1, stats.getTrackerVLSNConsistencyWaits());
        assertTrue(stats.getTrackerVLSNConsistencyWaitMs() > 0);
      } catch (UnknownMasterException e) {
        testException = e;
        throw new RuntimeException(e);
      } catch (DatabaseException e) {
        testException = e;
        throw new RuntimeException(e);
      }
    }
    public void openEnv() {
      try {
        Durability durability =
            new Durability(
                Durability.SyncPolicy.WRITE_NO_SYNC,
                Durability.SyncPolicy.WRITE_NO_SYNC,
                Durability.ReplicaAckPolicy.ALL);
        EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
        ReplicationConfig repConfig = RepTestUtils.createRepConfig(1);
        repEnvInfo = RepTestUtils.setupEnvInfo(envHome, envConfig, repConfig, null);
        repEnvInfo.openEnv();
        Thread.sleep(sleepTime);
      } catch (EnvironmentLockedException e) {

        /*
         * Exit the process with value 1, don't print out the exception
         * since it's expected.
         */
        System.exit(1);
      } catch (UnsupportedOperationException e) {

        /*
         * Exit the process with value 2, don't print out the exception
         * since it's expected.
         *
         * Note: this exception thrown because we can't start a
         * replicated Environment on an existed standalone Environment.
         */
        System.exit(2);
      } catch (Exception e) {
        /* Dump unexpected exceptions, exit processs with value 3. */
        e.printStackTrace();
        System.exit(3);
      } finally {
        if (repEnvInfo.getEnv() != null) {
          repEnvInfo.closeEnv();
        }
      }
    }
예제 #11
0
 private void close(boolean normalShutdown) {
   try {
     if (normalShutdown) {
       replicaApp1.close();
       replicaApp2.close();
       masterApp.close();
       RepTestUtils.shutdownRepEnvs(repEnvInfo);
     } else {
       for (RepEnvInfo info : repEnvInfo) {
         info.abnormalCloseEnv();
       }
     }
   } finally {
     repEnvInfo = null;
     masterEnv = null;
     replicaEnv1 = null;
     replicaEnv2 = null;
     masterApp = null;
     replicaApp1 = null;
     replicaApp2 = null;
   }
 }
예제 #12
0
 private void flushLogAndCrash(RepEnvInfo replica) {
   DbInternal.getEnvironmentImpl(replica.getEnv()).getLogManager().flush();
   replica.abnormalCloseEnv();
 }
예제 #13
0
  /**
   * Since the master never dies in this test, no rollbacks should occur, but no data should be lost
   * either.
   *
   * <p>TODO: Should the workload param be of a different class (not a RollbackWorkload), since its
   * masterSteadyWork method is only called here?
   */
  private void replicasDieAndRejoin(RollbackWorkload workload, int numIterations) throws Throwable {

    RepEnvInfo[] repEnvInfo = null;

    try {
      /* Create a  3 node group. Assign identities. */
      repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      logger.severe("master=" + master);

      RepEnvInfo replicaA = null;
      RepEnvInfo replicaB = null;

      for (RepEnvInfo info : repEnvInfo) {
        if (info.getEnv().getState().isMaster()) {
          continue;
        }

        if (replicaA == null) {
          replicaA = info;
        } else {
          replicaB = info;
        }
      }

      /*
       * For the sake of easy test writing, make sure numIterations is an
       * even number.
       */
      assertTrue((numIterations % 2) == 0);
      replicaA.abnormalCloseEnv();
      for (int i = 0; i < numIterations; i++) {
        workload.masterSteadyWork(master);
        waitForReplicaToSync(master, repEnvInfo);
        if ((i % 2) == 0) {
          flushLogAndCrash(replicaB);
          replicaA.openEnv();
        } else {
          flushLogAndCrash(replicaA);
          replicaB.openEnv();
        }
        waitForReplicaToSync(master, repEnvInfo);
      }
      replicaA.openEnv();

      VLSN lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);

      assertTrue(workload.containsAllData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      workload.close();
      for (RepEnvInfo repi : repEnvInfo) {
        /*
         * We're done with the test. Bringing down these replicators
         * forcibly, without closing transactions and whatnot.
         */
        repi.abnormalCloseEnv();
      }
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }
  }
예제 #14
0
  /**
   * Create 3 nodes and replicate operations. Kill off the master, and make the other two resume.
   * This will require a syncup and a rollback of any operations after the matchpoint.
   */
  private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable {

    RepEnvInfo[] repEnvInfo = null;

    try {
      /* Create a  3 node group */
      repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      logger.severe("master=" + master);

      /*
       * Run a workload against the master. Sync up the group and check
       * that all nodes have the same contents. This first workload must
       * end with in-progress, uncommitted transactions.
       */
      workload.beforeMasterCrash(master);
      VLSN lastVLSN = VLSN.NULL_VLSN;
      if (workload.noLockConflict()) {
        lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload);
      }

      /*
       * Crash the master, find a new master.
       */
      RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
      master = crashMasterAndElectNewMaster(master, repEnvInfo);
      RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
      logger.severe("newmaster=" + master);
      RepEnvInfo alwaysReplica = null;
      for (RepEnvInfo info : repEnvInfo) {
        if ((info != oldMaster) && (info != newMaster)) {
          alwaysReplica = info;
          break;
        }
      }

      /*
       * Check that the remaining two nodes only contain committed
       * updates.
       * TODO: check that the number of group members is 2.
       */
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /*
       * Do some work against the new master, while the old master is
       * asleep. Note that the first workload may have contained
       * in-flight transactions, so this may result in the rollback of
       * some transactions in the first workload.
       */
      workload.afterMasterCrashBeforeResumption(master);

      /*
       * The intent of this test is that the work after crash will end on
       * an incomplete txn. Check for that.
       */
      lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);

      /* Now bring up the old master. */
      logger.info("Bring up old master");
      oldMaster.openEnv();

      logger.info("Old master joined");
      RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
      logger.info("Old master synced");

      /*
       * Check that all nodes only contain committed updates.
       */
      workload.releaseDbLocks();
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /*
       * Now crash the node that has never been a master. Do some work
       * without it, then recover that node, then do a verification
       * check.  This exercises the recovery of a log that has syncups in
       * it.
       */
      alwaysReplica.abnormalCloseEnv();
      workload.afterReplicaCrash(master);

      lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN);
      alwaysReplica.openEnv();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
      RepTestUtils.checkUtilizationProfile(repEnvInfo);

      workload.close();

      /*
       * We're done with the test. Bringing down these replicators
       * forcibly, without closing transactions and whatnot.
       */
      for (RepEnvInfo repi : repEnvInfo) {
        repi.abnormalCloseEnv();
      }

      /*
       * Open and verify the environments one last time, to ensure that
       * rollbacks in the recovery interval don't cause problems.
       */
      master = RepTestUtils.restartGroup(repEnvInfo);
      lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /* Final close. */
      for (RepEnvInfo repi : repEnvInfo) {
        repi.closeEnv();
      }
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }
  }
  @SuppressWarnings("null")
  @Test
  public void testBasic() throws InterruptedException {

    final int dataNodeSize = groupSize - 1;
    final int electableNodeSize = groupSize - 2;
    final int persistentNodeSize = groupSize - 1;

    final ReplicationConfig sConfig = repEnvInfo[groupSize - 2].getRepConfig();
    sConfig.setNodeType(NodeType.SECONDARY);

    createGroup(dataNodeSize);

    ReplicationConfig rConfig = repEnvInfo[groupSize - 1].getRepConfig();
    /* RepNetConfig needs to come from an open environment */
    ReplicationConfig r0Config = repEnvInfo[0].getEnv().getRepConfig();
    rConfig.setNodeType(NodeType.MONITOR);
    MonitorConfig monConfig = new MonitorConfig();
    monConfig.setNodeName(rConfig.getNodeName());
    monConfig.setGroupName(rConfig.getGroupName());
    monConfig.setNodeHostPort(rConfig.getNodeHostPort());
    monConfig.setHelperHosts(rConfig.getHelperHosts());
    monConfig.setRepNetConfig(r0Config.getRepNetConfig());

    new Monitor(monConfig).register();

    for (int i = 0; i < dataNodeSize; i++) {
      final ReplicatedEnvironment env = repEnvInfo[i].getEnv();
      final boolean isMaster = (env.getState() == MASTER);
      final int targetGroupSize = isMaster ? groupSize : persistentNodeSize;
      ReplicationGroup group = null;
      for (int j = 0; j < 100; j++) {
        group = env.getGroup();
        if (group.getNodes().size() == targetGroupSize) {
          break;
        }
        /* Wait for the replica to catch up. */
        Thread.sleep(1000);
      }
      assertEquals("Nodes", targetGroupSize, group.getNodes().size());
      assertEquals(RepTestUtils.TEST_REP_GROUP_NAME, group.getName());
      logger.info(group.toString());

      for (RepEnvInfo rinfo : repEnvInfo) {
        final ReplicationConfig repConfig = rinfo.getRepConfig();
        ReplicationNode member = group.getMember(repConfig.getNodeName());
        if (!isMaster && repConfig.getNodeType().isSecondary()) {
          assertNull("Member", member);
        } else {
          assertNotNull("Member", member);
          assertEquals(repConfig.getNodeName(), member.getName());
          assertEquals(repConfig.getNodeType(), member.getType());
          assertEquals(repConfig.getNodeSocketAddress(), member.getSocketAddress());
        }
      }

      final Set<ReplicationNode> electableNodes = group.getElectableNodes();
      for (final ReplicationNode n : electableNodes) {
        assertEquals(NodeType.ELECTABLE, n.getType());
      }
      assertEquals("Electable nodes", electableNodeSize, electableNodes.size());

      final Set<ReplicationNode> monitorNodes = group.getMonitorNodes();
      for (final ReplicationNode n : monitorNodes) {
        assertEquals(NodeType.MONITOR, n.getType());
      }
      assertEquals("Monitor nodes", 1, monitorNodes.size());

      final Set<ReplicationNode> secondaryNodes = group.getSecondaryNodes();
      for (final ReplicationNode n : secondaryNodes) {
        assertEquals(NodeType.SECONDARY, n.getType());
      }
      assertEquals("Secondary nodes", isMaster ? 1 : 0, secondaryNodes.size());

      final Set<ReplicationNode> dataNodes = group.getDataNodes();
      for (final ReplicationNode n : dataNodes) {
        if (isMaster) {
          assertThat(n.getType(), anyOf(is(NodeType.ELECTABLE), is(NodeType.SECONDARY)));
        } else {
          assertEquals(NodeType.ELECTABLE, n.getType());
        }
      }
      assertEquals("Data nodes", isMaster ? dataNodeSize : electableNodeSize, dataNodes.size());
    }
  }
 private void setExceptionListener(final RepEnvInfo info) {
   EnvironmentConfig infoConfig = info.getEnvConfig();
   infoConfig.setExceptionListener(new NetworkTestExceptionListener());
   info.setEnvConfig(infoConfig);
 }
  /**
   * This is really multiple tests in one. It tests network restore with a replica in each of the
   * following three states:
   *
   * <p>1) A brand new node joining the group and needing a network restore.
   *
   * <p>2) An existing node with its own unique log needing a network restore.
   *
   * <p>3) Repeated network restores, reflecting a mature node.
   */
  @Test
  public void testBasic() throws DatabaseException, Exception {

    /*
     * The cleaner thread can see InsufficientLogExceptions so just stifle
     * those exceptions from stderr.
     */
    DaemonThread.stifleExceptionChatter = true;

    configureForMaxCleaning(2);

    final RepEnvInfo info1 = repEnvInfo[0];
    RepEnvInfo info2 = repEnvInfo[1];

    ReplicatedEnvironment masterRep = info1.openEnv();
    Environment menv = masterRep;
    EnvironmentMutableConfig mconfig = menv.getMutableConfig();
    mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
    menv.setMutableConfig(mconfig);

    /*
     * Have just the master join first. We do this to test the special case
     * of a brand new node joining a group and needing VLSN 1. The same
     * node then rejoins with its VLSN > 1 to test subsequent rejoins
     * where the node has already participated in the replication.
     */
    populateDB(masterRep, TEST_DB_NAME, 100);

    mconfig = menv.getMutableConfig();
    mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true");
    menv.setMutableConfig(mconfig);

    File cenvDir = info2.getEnvHome();
    final int cid = 2;

    for (int i = 0; i < RESTORE_CYCLES; i++) {

      leaveGroupAllButMaster();
      shiftVLSNRight(masterRep);
      RepNodeImpl memberPrev =
          info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
      /* Node1 is not known on the first iteration. */
      final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN();
      try {
        /* Should force a network restore. */
        setExceptionListener(info2);
        info2.openEnv();
        fail("exception expected");
      } catch (InsufficientLogException e) {
        RepNodeImpl member =
            info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());

        /*
         * The sync state should have been advanced to help contribute
         * to the global CBVLSN and prevent it from advancing.
         */
        final VLSN currSync = member.getBarrierState().getLastCBVLSN();
        assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0);

        NetworkRestore networkRestore = new NetworkRestore();
        networkRestore.execute(e, new NetworkRestoreConfig());
        final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
        assertThat(stats.getExpectedBytes(), greaterThan(0));
        assertThat(stats.getTransferredBytes(), greaterThan(0));
        /* Create a replacement replicator. */
        info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1);
        setExceptionListener(info2);
        info2.openEnv();
      }
      /* Verify that we can continue with the "restored" log files. */
      populateDB(masterRep, TEST_DB_NAME, 100, 100);
      VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2);
      RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
      info2.closeEnv();
    }
  }