/*
   * Provoke sufficient log cleaning to move the entire VLSN right
   * sufficiently that the new VLSN range no longer overlaps the VLSN
   * range upon entry thus guaranteeing a LogFileRefreshException.
   */
  private void shiftVLSNRight(ReplicatedEnvironment menv) {
    /* Shift the vlsn range window. */

    RepImpl menvImpl = repEnvInfo[0].getRepImpl();
    final VLSNIndex vlsnIndex = menvImpl.getRepNode().getVLSNIndex();
    VLSN masterHigh = vlsnIndex.getRange().getLast();

    CheckpointConfig checkpointConfig = new CheckpointConfig();
    checkpointConfig.setForce(true);

    do {

      /*
       * Populate just the master, leaving the replica behind Re-populate
       * with the same keys to create Cleaner fodder.
       */
      populateDB(menv, TEST_DB_NAME, 100);

      /*
       * Sleep to permit the cbvlsn on the master to be updated. It's
       * done with the period: FeederManager.MASTER_CHANGE_CHECK_TIMEOUT
       */
      try {
        Thread.sleep(1000);
      } catch (InterruptedException e) {
        fail("unexpected interrupt");
      }
      menv.cleanLog();
      menv.checkpoint(checkpointConfig);
    } while (masterHigh.compareTo(vlsnIndex.getRange().getFirst()) > 0);
  }
  @Test
  public void testRemoveMemberExceptions() {
    createGroup(2);
    ReplicatedEnvironment master = repEnvInfo[0].getEnv();
    assertTrue(master.getState().isMaster());

    RepNode masterRep = repEnvInfo[0].getRepNode();
    try {
      masterRep.removeMember(master.getNodeName());
      fail("Exception expected.");
    } catch (MasterStateException e) {
      // Expected
    }

    try {
      masterRep.removeMember("unknown node foobar");
      fail("Exception expected.");
    } catch (MemberNotFoundException e) {
      // Expected
    }

    masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
    try {
      masterRep.removeMember(repEnvInfo[1].getRepNode().getNodeName());
      fail("Exception expected.");
    } catch (MemberNotFoundException e) {
      // Expected
    }
    repEnvInfo[1].closeEnv();
  }
Пример #3
0
  /**
   * Bounce the master, causing replica1 to switch roles with the master. If a higher appVersion is
   * specified, the bounced node will also be upgraded.
   */
  private void bounceMaster(final int appVersion) throws Exception {

    /* Disable updates to RepGroupDB due to LocalCBVLSN updates. */
    LocalCBVLSNUpdater.setSuppressGroupDBUpdates(true);

    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == masterEnv) {

        /*
         * Sync up the replication group so that node2 doesn't do
         * hard recovery.
         */
        RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
        /* Disable replay on replicas. */
        shutdownFeeder(info.getRepNode(), replicaEnv1);
        shutdownFeeder(info.getRepNode(), replicaEnv2);

        /* Close the master. */
        masterApp.close();
        masterApp = null;
        info.closeEnv();
        masterEnv = null;

        /* Force repEnvInfo[2] to the master. */
        WaitForMasterListener masterWaiter = new WaitForMasterListener();
        replicaEnv2.setStateChangeListener(masterWaiter);
        RepNode repNode = repEnvInfo[2].getRepNode();
        repNode.forceMaster(true);
        /* Enable the LocalCBVLSN updates. */
        LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);
        masterWaiter.awaitMastership();
        assertTrue(repNode.isMaster());
        masterEnv = replicaEnv2;

        /* Replica2 was elected, swap names with replica1. */
        final ReplicatedEnvironment tmpEnv = replicaEnv1;
        replicaEnv1 = replicaEnv2;
        replicaEnv2 = tmpEnv;
        final AppInterface tmpApp = replicaApp1;
        replicaApp1 = replicaApp2;
        replicaApp2 = tmpApp;

        /* Replica1 (or 2, see above) has been elected master. */
        masterApp = newAppObject(appVersion);
        masterApp.adopt(replicaApp1);
        /* Former master (just upgraded) becomes replica1. */
        replicaEnv1 = info.openEnv();
        replicaApp1.open(replicaEnv1);
        break;
      }
    }
    assertNotNull(masterApp);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
  /* Start the master (the helper node) first */
  @Test
  public void testGroupCreateMasterFirst() throws DatabaseException {

    for (int i = 0; i < repEnvInfo.length; i++) {
      ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
      State state = rep.getState();
      assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
      RepNode repNode = RepInternal.getRepImpl(rep).getRepNode();
      /* No elections, helper nodes or members queried for master. */
      assertEquals(0, repNode.getElections().getElectionCount());
    }
  }
  /** Crash the current master, and wait until the group elects a new one. */
  private ReplicatedEnvironment crashMasterAndElectNewMaster(
      ReplicatedEnvironment master, RepEnvInfo[] repEnvInfo) {

    int masterIndex = RepInternal.getNodeId(master) - 1;

    logger.info("Crashing " + master.getNodeName());
    repEnvInfo[masterIndex].abnormalCloseEnv();

    logger.info("Rejoining");
    ReplicatedEnvironment newMaster = RepTestUtils.openRepEnvsJoin(repEnvInfo);

    logger.info("New master = " + newMaster.getNodeName());
    return newMaster;
  }
  private void testMemberRemoveAckInteraction(final boolean delete) {
    createGroup(groupSize);
    Transaction txn;
    Database db;
    try {
      MasterTxn.setFactory(new TxnFactory(delete));
      ReplicatedEnvironment master = repEnvInfo[0].getEnv();

      txn = master.beginTransaction(null, null);
      /* Write to the environment. */
      db = master.openDatabase(txn, "random", dbconfig);
      db.close();
      txn.commit();
    } catch (InsufficientAcksException e) {
      fail("No exception expected.");
    } finally {
      MasterTxn.setFactory(null);
    }
  }
  /*
   * Tests internal node removal APIs.
   */
  @Test
  public void testRemoveMember() {
    createGroup(groupSize);
    ReplicatedEnvironment master = repEnvInfo[0].getEnv();
    assertTrue(master.getState().isMaster());

    RepNode masterRep = repEnvInfo[0].getRepNode();

    /* Reduce the group size all the way down to one. */
    for (int i = 1; i < groupSize; i++) {
      assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
      masterRep.removeMember(repEnvInfo[i].getEnv().getNodeName());
      assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
    }

    /* Close the replica handles*/
    for (int i = groupSize - 1; i > 0; i--) {
      repEnvInfo[i].closeEnv();
    }

    /* Attempting to re-open them with the same node names should fail. */
    for (int i = 1; i < groupSize; i++) {
      try {
        repEnvInfo[i].openEnv();
        fail("Exception expected");
      } catch (EnvironmentFailureException e) {
        /* Expected, the master should reject the attempt. */
        assertEquals(EnvironmentFailureReason.HANDSHAKE_ERROR, e.getReason());
      }
    }

    /* Doing the same but with different node names should be ok. */
    for (int i = 1; i < groupSize; i++) {
      final RepEnvInfo ri = repEnvInfo[i];
      final ReplicationConfig repConfig = ri.getRepConfig();
      TestUtils.removeLogFiles("RemoveRepEnvironments", ri.getEnvHome(), false);

      repConfig.setNodeName("ReplaceNode_" + i);
      ri.openEnv();
      assertEquals(i + 1, masterRep.getGroup().getElectableGroupSize());
    }
    master.close();
  }
  @Test
  public void testNoQuorum() throws DatabaseException, InterruptedException {

    for (int i = 0; i < 3; i++) {
      ReplicatedEnvironment rep = repEnvInfo[i].openEnv();
      State state = rep.getState();
      assertEquals((i == 0) ? State.MASTER : State.REPLICA, state);
    }
    RepTestUtils.syncGroupToLastCommit(repEnvInfo, 3);
    repEnvInfo[1].closeEnv();
    repEnvInfo[2].closeEnv();

    // A new node joining in the absence of a quorum must fail
    try {
      repEnvInfo[3].openEnv();
      fail("Expected exception");
    } catch (UnknownMasterException e) {
      /* Expected. */
    }
  }
  /*
   * Tests internal node deletion APIs.
   */
  @Test
  public void testDeleteMember() {
    createGroup(groupSize);
    ReplicatedEnvironment master = repEnvInfo[0].getEnv();
    assertTrue(master.getState().isMaster());

    RepNode masterRep = repEnvInfo[0].getRepNode();

    /* Reduce the group size all the way down to one. */
    for (int i = 1; i < groupSize; i++) {
      assertTrue(!RepInternal.isClosed(repEnvInfo[i].getEnv()));
      final String delName = repEnvInfo[i].getEnv().getNodeName();
      repEnvInfo[i].closeEnv();
      masterRep.removeMember(delName, true);
      assertEquals((groupSize - i), masterRep.getGroup().getElectableGroupSize());
    }

    /*
     * Attempting to re-open them with the same node names should succeed
     */
    for (int i = 1; i < groupSize; i++) {
      repEnvInfo[i].openEnv();
    }
  }
Пример #10
0
  /**
   * Bounce replica2. No election will take place. If a higher appVersion is specified, the bounced
   * node will also be upgraded.
   */
  private void bounceReplica2(final int appVersion) throws Exception {

    replicaApp2.close();
    replicaApp2 = null;
    for (RepEnvInfo info : repEnvInfo) {
      if (info.getEnv() == replicaEnv2) {
        info.closeEnv();
        replicaEnv2 = info.openEnv();
        replicaApp2 = newAppObject(appVersion);
        replicaApp2.open(replicaEnv2);
        break;
      }
    }
    assertNotNull(replicaApp2);
    assertSame(masterEnv.getState(), ReplicatedEnvironment.State.MASTER);
  }
Пример #11
0
  /*
   * Remove a feeder from the master so that no entries can be replayed on
   * this node.
   */
  private void shutdownFeeder(RepNode masterNode, ReplicatedEnvironment repEnv) throws Exception {

    String nodeName = repEnv.getRepConfig().getNodeName();
    RepNodeImpl removeNode = masterNode.getGroup().getNode(nodeName);
    masterNode.feederManager().shutdownFeeder(removeNode);
  }
  private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {

    try {
      createRepEnvInfo(sleepTime);

      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      long startTime = System.currentTimeMillis();

      StatsConfig stConfig = new StatsConfig();
      stConfig.setClear(true);

      /* Flush the existed dirty data before we do writes. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        repEnvInfo[i].getEnv().sync();
        repEnvInfo[i].getEnv().getStats(stConfig);
      }

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);

      Database db = master.openDatabase(null, dbName, dbConfig);

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      for (int i = 1; i <= 100; i++) {
        IntegerBinding.intToEntry(i, key);
        StringBinding.stringToEntry(value, data);
        db.put(null, key, data);
      }

      assertTrue(System.currentTimeMillis() - startTime < 15000);

      Thread.sleep(15000);

      long endTime = System.currentTimeMillis();

      for (int i = 0; i < repEnvInfo.length; i++) {
        EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
        LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
        if (flushBeforeCrash) {
          /* Make sure the LogFlushTask has been invoked. */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);

          /*
           * Since the log file size is not so big, we can't assure
           * all the data will be written in the same log file, but
           * we can sure that a flush does happen.
           */
          assertTrue(envStats.getNSequentialWrites() >= 1);
          assertTrue(envStats.getNLogFSyncs() == 1);
        } else {

          /*
           * Make sure the LogFlushTask is not invoked after making
           * the changes.
           */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
          assertTrue(envStats.getNSequentialWrites() == 0);
          assertTrue(envStats.getNLogFSyncs() == 0);
        }
        assertTrue(envStats.getNFSyncs() == 0);
      }

      File[] envHomes = new File[3];
      /* Close the replicas without doing a checkpoint. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        envHomes[i] = repEnvInfo[i].getEnvHome();
        repEnvInfo[i].getRepImpl().abnormalClose();
      }

      /*
       * Open a read only standalone Environment on the replicas to see
       * whether the data has been synced to the disk.
       */
      EnvironmentConfig newConfig = new EnvironmentConfig();
      newConfig.setAllowCreate(false);
      newConfig.setReadOnly(true);
      newConfig.setTransactional(true);

      for (int i = 0; i < repEnvInfo.length; i++) {
        Environment env = new Environment(envHomes[i], newConfig);

        dbConfig.setAllowCreate(false);
        dbConfig.setReadOnly(true);

        try {
          db = env.openDatabase(null, dbName, dbConfig);
        } catch (DatabaseNotFoundException e) {

          /*
           * If the system crashes before the flush, the database is
           * not synced to the disk, so this database can't be found
           * at all, it's expected.
           */
          assertFalse(flushBeforeCrash);
        }

        if (flushBeforeCrash) {
          assertTrue(db.count() == 100);
          for (int index = 1; index <= 100; index++) {
            IntegerBinding.intToEntry(index, key);
            OperationStatus status = db.get(null, key, data, null);
            if (flushBeforeCrash) {
              assertTrue(status == OperationStatus.SUCCESS);
              assertEquals(value, StringBinding.entryToString(data));
            }
          }
        }

        if (flushBeforeCrash) {
          db.close();
        }
        env.close();
      }
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  /**
   * Test that a timeout in the feeder while attempting to read the group database because other
   * feeders have it write locked causes the feeder (and replica) to fail, but allows the master to
   * continue operating. [#23822]
   */
  @Test
  public void testJoinGroupReadGroupTimeout() throws DatabaseException, InterruptedException {

    /* Start first node as master */
    ReplicatedEnvironment repEnv = repEnvInfo[0].openEnv();
    assertEquals("Master node state", State.MASTER, repEnv.getState());

    RepImpl repImpl = RepInternal.getRepImpl(repEnv);

    for (int i = 1; i <= 2; i++) {

      /* Get a write lock on the RepGroupDB */
      final MasterTxn txn =
          new MasterTxn(
              repImpl,
              new TransactionConfig()
                  .setDurability(
                      new Durability(
                          SyncPolicy.SYNC, SyncPolicy.SYNC, ReplicaAckPolicy.SIMPLE_MAJORITY)),
              repImpl.getNameIdPair());
      final DatabaseImpl groupDbImpl = repImpl.getGroupDb();
      final DatabaseEntry value = new DatabaseEntry();
      final Cursor cursor = DbInternal.makeCursor(groupDbImpl, txn, new CursorConfig());
      final OperationStatus status = cursor.getNext(RepGroupDB.groupKeyEntry, value, LockMode.RMW);
      assertEquals(i + ": Lock group result", OperationStatus.SUCCESS, status);

      /* Wait longer than the default 500 ms read timeout */
      Thread.sleep(600);

      /* Test both electable and secondary nodes */
      if (i == 2) {
        repEnvInfo[i].getRepConfig().setNodeType(NodeType.SECONDARY);
      }

      /* Create a thread that attempts to join another environment */
      RepNodeThread repNodeThread = new RepNodeThread(i, i != 1);
      repNodeThread.start();

      /* Wait for attempt to complete */
      repNodeThread.join(30000);
      assertEquals("RN thread alive", false, repNodeThread.isAlive());

      if (i == 1) {

        /* Join attempt should fail for primary */
        assertNotNull("Expected RN thread exception", repNodeThread.te);

        /* Release write lock on RepGroupDB */
        cursor.close();
        txn.abort();

        /* Second join attempt should succeed */
        repNodeThread = new RepNodeThread(1);
        repNodeThread.start();
        repNodeThread.join(30000);
        assertEquals("RN thread alive", false, repNodeThread.isAlive());
        assertEquals("RN thread exception", null, repNodeThread.te);
      } else {

        /* Join attempt should succeed for secondary */
        assertEquals("RN thread exception", null, repNodeThread.te);

        /* Release write lock on RepGroupDB */
        cursor.close();
        txn.abort();
      }
    }
  }