/**
   * Create a log that will have swathes of cleaned files that follow the replication stream, or are
   * intermingled in the replication stream.
   *
   * @return master
   */
  private Environment setupLogWithCleanedGaps(boolean multipleGaps) throws Exception {

    db = null;
    repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 3, makeEnvConfig());
    Environment master = RepTestUtils.joinGroup(repEnvInfo);
    int masterIdx = findMasterIndex(master);
    db = openDatabase(master);

    /* Write some data so there is a replication stream. */
    generateData(master, 50, Durability.COMMIT_NO_SYNC, true);

    /*
     * Make the master have a low-utilization log, and gate cleaning
     * with a non-updating global cbvlsn. Shut down the replicas so the
     * global cbvlsn remains low, and then fill the master with junk.
     * The junk will either entirely be to the right of the last VLSN,
     * or (since we can't predict RepGroupDB updates) at least within
     * the range of the active VLSN range.
     */
    closeReplicas(masterIdx);
    fillLogWithTraceMsgs(master, 50);

    if (multipleGaps) {
      Durability noAck =
          new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
      /* Write more data */
      generateData(master, 50, noAck, true);

      /* Make a second cleanup area of junk */
      fillLogWithTraceMsgs(master, 50);
    }

    CheckpointConfig cc = new CheckpointConfig();
    cc.setForce(true);
    master.checkpoint(cc);

    EnvironmentStats stats = master.getStats(clearConfig);
    stats = master.getStats(clearConfig);

    /* Clean the log */
    int totalCleaned = 0;
    int cleanedThisPass = 0;
    do {
      cleanedThisPass = cleanLog(master);
      totalCleaned += cleanedThisPass;
      master.checkpoint(cc);

      stats = master.getStats(clearConfig);
      logger.info(
          "after cleaning, cleaner backlog = "
              + stats.getCleanerBacklog()
              + " deletionBacklog="
              + stats.getFileDeletionBacklog());
    } while (cleanedThisPass > 0);

    assertTrue(totalCleaned > 0);

    return master;
  }
예제 #2
0
 private void expectNothingToClean() {
   env.cleanLog();
   final EnvironmentStats stats = env.getStats(null);
   final String msg =
       String.format(
           "%d probes, %d non-probes", stats.getNCleanerProbeRuns(), stats.getNCleanerRuns());
   assertEquals(msg, 0, stats.getNCleanerRuns() - stats.getNCleanerProbeRuns());
 }
  private void doReplicaHasGapNetworkRestore(boolean multiGaps) throws Throwable {

    Durability noAck =
        new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
    db = null;
    try {
      Environment master = setupLogWithCleanedGaps(multiGaps);
      int masterIdx = findMasterIndex(master);
      /*
       * Write a record, so that we are sure that there will be a
       * network restore, because we have to cross a checkpoint.
       */
      generateData(master, 1, noAck, false);
      CheckpointConfig cc = new CheckpointConfig();
      master.checkpoint(cc);
      EnvironmentStats stats = master.getStats(clearConfig);
      assertEquals(0, stats.getCleanerBacklog());
      if (multiGaps) {
        logger.info("Multigap: deletion backlog = " + stats.getFileDeletionBacklog());
      } else {
        assertEquals(0, stats.getFileDeletionBacklog());
      }

      db.close();
      db = null;
      repEnvInfo[masterIdx].closeEnv();

      /* Start up the two replicas */
      openReplicas(masterIdx);

      /* Start the node that had been the master */
      try {
        repEnvInfo[masterIdx].openEnv();
        fail("Should be a network restore");
      } catch (InsufficientLogException ile) {
        repEnvInfo[masterIdx].closeEnv();
        NetworkRestore restore = new NetworkRestore();
        NetworkRestoreConfig config = new NetworkRestoreConfig();
        config.setRetainLogFiles(true);
        restore.execute(ile, config);
        repEnvInfo[masterIdx].openEnv();
      }

      /* Check its last VLSN and size. */

    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    } finally {
      if (db != null) {
        db.close();
      }
      RepTestUtils.shutdownRepEnvs(repEnvInfo);
    }
  }
예제 #4
0
 private void expectBackgroundCleaning() {
   final long endTime = System.currentTimeMillis() + (30 * 1000);
   while (System.currentTimeMillis() < endTime) {
     final EnvironmentStats stats = env.getStats(null);
     if (stats.getNCleanerRuns() > 0) {
       return;
     }
   }
   close();
   fail("Cleaner did not run");
 }
  /**
   * On the master, generate a log that has section A: a lot of records packed together section B: a
   * lot of junk that gets cleaned away, creating a gap in the log section C: a new section of data
   *
   * <p>Bring the replicas down after A is replicated, but before C is written. When the replicas
   * come up, they will have to be fed by the feeder from point A.
   */
  @Test
  public void testFeederHasGap() throws Throwable {

    Durability noAck =
        new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
    db = null;
    try {
      Environment master = setupLogWithCleanedGaps(false);
      int masterIdx = findMasterIndex(master);

      /*
       * Write a single record, and then junk, so that we are sure there
       * is a new VLSN, and that the replicas will have to sync up to
       * this point, across the gap of cleaned junk.
       */
      generateData(master, 1, noAck, false);
      EnvironmentStats stats = master.getStats(clearConfig);
      assertEquals(0, stats.getCleanerBacklog());
      assertEquals(0, stats.getFileDeletionBacklog());

      /* Start up the two replicas */
      for (int i = 0; i < repEnvInfo.length; i++) {
        if (i != masterIdx) {

          repEnvInfo[i].openEnv();
          /* make sure we have up to date data */
          readData(repEnvInfo[i].getEnv(), 50);
        }
      }
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    } finally {
      if (db != null) {
        db.close();
      }
      RepTestUtils.shutdownRepEnvs(repEnvInfo);
    }
  }
예제 #6
0
 void loadStats(StatsConfig config, EnvironmentStats stats) {
   stats.setCacheDataBytes(getCacheMemoryUsage());
 }
  private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {

    try {
      createRepEnvInfo(sleepTime);

      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      long startTime = System.currentTimeMillis();

      StatsConfig stConfig = new StatsConfig();
      stConfig.setClear(true);

      /* Flush the existed dirty data before we do writes. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        repEnvInfo[i].getEnv().sync();
        repEnvInfo[i].getEnv().getStats(stConfig);
      }

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);

      Database db = master.openDatabase(null, dbName, dbConfig);

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      for (int i = 1; i <= 100; i++) {
        IntegerBinding.intToEntry(i, key);
        StringBinding.stringToEntry(value, data);
        db.put(null, key, data);
      }

      assertTrue(System.currentTimeMillis() - startTime < 15000);

      Thread.sleep(15000);

      long endTime = System.currentTimeMillis();

      for (int i = 0; i < repEnvInfo.length; i++) {
        EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
        LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
        if (flushBeforeCrash) {
          /* Make sure the LogFlushTask has been invoked. */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);

          /*
           * Since the log file size is not so big, we can't assure
           * all the data will be written in the same log file, but
           * we can sure that a flush does happen.
           */
          assertTrue(envStats.getNSequentialWrites() >= 1);
          assertTrue(envStats.getNLogFSyncs() == 1);
        } else {

          /*
           * Make sure the LogFlushTask is not invoked after making
           * the changes.
           */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
          assertTrue(envStats.getNSequentialWrites() == 0);
          assertTrue(envStats.getNLogFSyncs() == 0);
        }
        assertTrue(envStats.getNFSyncs() == 0);
      }

      File[] envHomes = new File[3];
      /* Close the replicas without doing a checkpoint. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        envHomes[i] = repEnvInfo[i].getEnvHome();
        repEnvInfo[i].getRepImpl().abnormalClose();
      }

      /*
       * Open a read only standalone Environment on the replicas to see
       * whether the data has been synced to the disk.
       */
      EnvironmentConfig newConfig = new EnvironmentConfig();
      newConfig.setAllowCreate(false);
      newConfig.setReadOnly(true);
      newConfig.setTransactional(true);

      for (int i = 0; i < repEnvInfo.length; i++) {
        Environment env = new Environment(envHomes[i], newConfig);

        dbConfig.setAllowCreate(false);
        dbConfig.setReadOnly(true);

        try {
          db = env.openDatabase(null, dbName, dbConfig);
        } catch (DatabaseNotFoundException e) {

          /*
           * If the system crashes before the flush, the database is
           * not synced to the disk, so this database can't be found
           * at all, it's expected.
           */
          assertFalse(flushBeforeCrash);
        }

        if (flushBeforeCrash) {
          assertTrue(db.count() == 100);
          for (int index = 1; index <= 100; index++) {
            IntegerBinding.intToEntry(index, key);
            OperationStatus status = db.get(null, key, data, null);
            if (flushBeforeCrash) {
              assertTrue(status == OperationStatus.SUCCESS);
              assertEquals(value, StringBinding.entryToString(data));
            }
          }
        }

        if (flushBeforeCrash) {
          db.close();
        }
        env.close();
      }
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }