@Test
  public void testConfigError() {
    configureForMaxCleaning(5);
    final RepEnvInfo minfo = repEnvInfo[0];

    createGroup();
    populateDB(minfo.getEnv(), TEST_DB_NAME, 100);
    RepEnvInfo nrInfo = repEnvInfo[1];
    nrInfo.closeEnv();
    shiftVLSNRight(repEnvInfo[0].getEnv());
    try {
      setExceptionListener(nrInfo);
      nrInfo.openEnv();
      fail("exception expected");
    } catch (InsufficientLogException e) {
      NetworkRestore networkRestore = new NetworkRestore();

      final ReplicationConfig repConfig = repEnvInfo[0].getRepConfig();
      // bad node name
      repConfig.setNodeName("badname");
      ReplicationNode restoreNode = new RepNodeImpl(repConfig);
      final NetworkRestoreConfig config = new NetworkRestoreConfig();
      config.setLogProviders(Arrays.asList(restoreNode));
      try {
        networkRestore.execute(e, config);
        fail("exception expected");
      } catch (IllegalArgumentException iae) {
        // Expected
      }
    }
  }
Exemplo n.º 2
0
  /**
   * Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
   * specified, the bounced node will also be upgraded.
   */
  private void bounceMaster(final int appVersion) throws Exception {

    /* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
    LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);

    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == masterEnv) {

        /*
         * Sync up the replication group so that node2 doesn't do
         * hard recovery.
         */
        RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
        /* Disable replay on replicas. */
        shutdownFeeder(info.getRepNode(), replicaEnv1);
        shutdownFeeder(info.getRepNode(), replicaEnv2);

        /* Close the master. */
        masterApp.close();
        masterApp = null;
        info.closeEnv();
        masterEnv = null;

        /* Force repEnvInfo[2] to the master. */
        WaitForMasterListener masterWaiter = new WaitForMasterListener();
        replicaEnv2.setStateChangeListener(masterWaiter);
        RepNode repNode = repEnvInfo[2].getRepNode();
        repNode.forceMaster(true);
        /* Enable the LocalCBVLSN updates. */
        LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
        masterWaiter.awaitMastership();
        assertTrue(repNode.isMaster());
        masterEnv = replicaEnv2;

        /* Replica2 was elected, swap names with replica1. */
        final ReplicatedEnvironment tmpEnv = replicaEnv1;
        replicaEnv1 = replicaEnv2;
        replicaEnv2 = tmpEnv;
        final AppInterface tmpApp = replicaApp1;
        replicaApp1 = replicaApp2;
        replicaApp2 = tmpApp;

        /* Replica1 (or 2, see above) has been elected master. */
        masterApp = newAppObject(appVersion);
        masterApp.adopt(replicaApp1);
        /* Former master (just upgraded) becomes replica1. */
        replicaEnv1 = info.openEnv();
        replicaApp1.open(replicaEnv1);
        break;
      }
    }
    assertNotNull(masterApp);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
Exemplo n.º 3
0
  /**
   * Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
   * node will also be upgraded.
   */
  private void bounceReplica2(final int appVersion) throws Exception {

    replicaApp2.close();
    replicaApp2 = null;
    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == replicaEnv2) {
        info.closeEnv();
        replicaEnv2 = info.openEnv();
        replicaApp2 = newAppObject(appVersion);
        replicaApp2.open(replicaEnv2);
        break;
      }
    }
    assertNotNull(replicaApp2);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
 /*
  * Creates conditions for a network restore at nrInfo and then restores the
  * node from a specific member.
  */
 private void doAndCheckRestore(RepEnvInfo nrInfo, RepEnvInfo restoreFromInfo) {
   nrInfo.closeEnv();
   shiftVLSNRight(repEnvInfo[0].getEnv());
   try {
     nrInfo.openEnv();
     fail("exception expected");
   } catch (InsufficientLogException e) {
     NetworkRestore networkRestore = new NetworkRestore();
     ReplicationNode restoreNode = new RepNodeImpl(restoreFromInfo.getRepConfig());
     final NetworkRestoreConfig config = new NetworkRestoreConfig();
     config.setLogProviders(Arrays.asList(restoreNode));
     networkRestore.execute(e, config);
     assertEquals(restoreNode, networkRestore.getLogProvider());
     final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
     assertThat(stats.getExpectedBytes(), greaterThan(0));
     assertThat(stats.getTransferredBytes(), greaterThan(0));
     nrInfo.openEnv();
   }
 }
    public void openEnv() {
      try {
        Durability durability =
            new Durability(
                Durability.SyncPolicy.WRITE_NO_SYNC,
                Durability.SyncPolicy.WRITE_NO_SYNC,
                Durability.ReplicaAckPolicy.ALL);
        EnvironmentConfig envConfig = RepTestUtils.createEnvConfig(durability);
        ReplicationConfig repConfig = RepTestUtils.createRepConfig(1);
        repEnvInfo = RepTestUtils.setupEnvInfo(envHome, envConfig, repConfig, null);
        repEnvInfo.openEnv();
        Thread.sleep(sleepTime);
      } catch (EnvironmentLockedException e) {

        /*
         * Exit the process with value 1, don't print out the exception
         * since it's expected.
         */
        System.exit(1);
      } catch (UnsupportedOperationException e) {

        /*
         * Exit the process with value 2, don't print out the exception
         * since it's expected.
         *
         * Note: this exception thrown because we can't start a
         * replicated Environment on an existed standalone Environment.
         */
        System.exit(2);
      } catch (Exception e) {
        /* Dump unexpected exceptions, exit processs with value 3. */
        e.printStackTrace();
        System.exit(3);
      } finally {
        if (repEnvInfo.getEnv() != null) {
          repEnvInfo.closeEnv();
        }
      }
    }
  /**
   * Create 3 nodes and replicate operations. Kill off the master, and make the other two resume.
   * This will require a syncup and a rollback of any operations after the matchpoint.
   */
  private void masterDiesAndRejoins(RollbackWorkload workload) throws Throwable {

    RepEnvInfo[] repEnvInfo = null;

    try {
      /* Create a  3 node group */
      repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3);
      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      logger.severe("master=" + master);

      /*
       * Run a workload against the master. Sync up the group and check
       * that all nodes have the same contents. This first workload must
       * end with in-progress, uncommitted transactions.
       */
      workload.beforeMasterCrash(master);
      VLSN lastVLSN = VLSN.NULL_VLSN;
      if (workload.noLockConflict()) {
        lastVLSN = checkIfWholeGroupInSync(master, repEnvInfo, workload);
      }

      /*
       * Crash the master, find a new master.
       */
      RepEnvInfo oldMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
      master = crashMasterAndElectNewMaster(master, repEnvInfo);
      RepEnvInfo newMaster = repEnvInfo[RepInternal.getNodeId(master) - 1];
      logger.severe("newmaster=" + master);
      RepEnvInfo alwaysReplica = null;
      for (RepEnvInfo info : repEnvInfo) {
        if ((info != oldMaster) && (info != newMaster)) {
          alwaysReplica = info;
          break;
        }
      }

      /*
       * Check that the remaining two nodes only contain committed
       * updates.
       * TODO: check that the number of group members is 2.
       */
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /*
       * Do some work against the new master, while the old master is
       * asleep. Note that the first workload may have contained
       * in-flight transactions, so this may result in the rollback of
       * some transactions in the first workload.
       */
      workload.afterMasterCrashBeforeResumption(master);

      /*
       * The intent of this test is that the work after crash will end on
       * an incomplete txn. Check for that.
       */
      lastVLSN = ensureDistinctLastAndSyncVLSN(master, repEnvInfo);

      /* Now bring up the old master. */
      logger.info("Bring up old master");
      oldMaster.openEnv();

      logger.info("Old master joined");
      RepTestUtils.syncGroupToVLSN(repEnvInfo, repEnvInfo.length, lastVLSN);
      logger.info("Old master synced");

      /*
       * Check that all nodes only contain committed updates.
       */
      workload.releaseDbLocks();
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /*
       * Now crash the node that has never been a master. Do some work
       * without it, then recover that node, then do a verification
       * check.  This exercises the recovery of a log that has syncups in
       * it.
       */
      alwaysReplica.abnormalCloseEnv();
      workload.afterReplicaCrash(master);

      lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 2, lastVLSN);
      alwaysReplica.openEnv();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
      assertTrue(workload.containsSavedData(master));
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);
      RepTestUtils.checkUtilizationProfile(repEnvInfo);

      workload.close();

      /*
       * We're done with the test. Bringing down these replicators
       * forcibly, without closing transactions and whatnot.
       */
      for (RepEnvInfo repi : repEnvInfo) {
        repi.abnormalCloseEnv();
      }

      /*
       * Open and verify the environments one last time, to ensure that
       * rollbacks in the recovery interval don't cause problems.
       */
      master = RepTestUtils.restartGroup(repEnvInfo);
      lastVLSN = RepInternal.getRepImpl(master).getVLSNIndex().getRange().getLast();
      RepTestUtils.syncGroupToVLSN(repEnvInfo, 3, lastVLSN);
      RepTestUtils.checkNodeEquality(lastVLSN, verbose, repEnvInfo);

      /* Final close. */
      for (RepEnvInfo repi : repEnvInfo) {
        repi.closeEnv();
      }
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }
  }
  /**
   * This is really multiple tests in one. It tests network restore with a replica in each of the
   * following three states:
   *
   * <p>1) A brand new node joining the group and needing a network restore.
   *
   * <p>2) An existing node with its own unique log needing a network restore.
   *
   * <p>3) Repeated network restores, reflecting a mature node.
   */
  @Test
  public void testBasic() throws DatabaseException, Exception {

    /*
     * The cleaner thread can see InsufficientLogExceptions so just stifle
     * those exceptions from stderr.
     */
    DaemonThread.stifleExceptionChatter = true;

    configureForMaxCleaning(2);

    final RepEnvInfo info1 = repEnvInfo[0];
    RepEnvInfo info2 = repEnvInfo[1];

    ReplicatedEnvironment masterRep = info1.openEnv();
    Environment menv = masterRep;
    EnvironmentMutableConfig mconfig = menv.getMutableConfig();
    mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
    menv.setMutableConfig(mconfig);

    /*
     * Have just the master join first. We do this to test the special case
     * of a brand new node joining a group and needing VLSN 1. The same
     * node then rejoins with its VLSN > 1 to test subsequent rejoins
     * where the node has already participated in the replication.
     */
    populateDB(masterRep, TEST_DB_NAME, 100);

    mconfig = menv.getMutableConfig();
    mconfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "true");
    menv.setMutableConfig(mconfig);

    File cenvDir = info2.getEnvHome();
    final int cid = 2;

    for (int i = 0; i < RESTORE_CYCLES; i++) {

      leaveGroupAllButMaster();
      shiftVLSNRight(masterRep);
      RepNodeImpl memberPrev =
          info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());
      /* Node1 is not known on the first iteration. */
      final VLSN prevSync = (i == 0) ? null : memberPrev.getBarrierState().getLastCBVLSN();
      try {
        /* Should force a network restore. */
        setExceptionListener(info2);
        info2.openEnv();
        fail("exception expected");
      } catch (InsufficientLogException e) {
        RepNodeImpl member =
            info1.getRepNode().getGroup().getMember(info2.getRepConfig().getNodeName());

        /*
         * The sync state should have been advanced to help contribute
         * to the global CBVLSN and prevent it from advancing.
         */
        final VLSN currSync = member.getBarrierState().getLastCBVLSN();
        assertTrue((i == 0) || currSync.compareTo(prevSync) >= 0);

        NetworkRestore networkRestore = new NetworkRestore();
        networkRestore.execute(e, new NetworkRestoreConfig());
        final NetworkBackupStats stats = networkRestore.getNetworkBackupStats();
        assertThat(stats.getExpectedBytes(), greaterThan(0));
        assertThat(stats.getTransferredBytes(), greaterThan(0));
        /* Create a replacement replicator. */
        info2 = RepTestUtils.setupEnvInfo(cenvDir, RepTestUtils.DEFAULT_DURABILITY, cid, info1);
        setExceptionListener(info2);
        info2.openEnv();
      }
      /* Verify that we can continue with the "restored" log files. */
      populateDB(masterRep, TEST_DB_NAME, 100, 100);
      VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, 2);
      RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
      info2.closeEnv();
    }
  }