Exemplo n.º 1
0
  /**
   * Do operations on the REGION_NAME's keys using keyIntervals to specify which keys get which
   * operations. This will return when all operations in all intervals have completed.
   *
   * @param availableOps - Bits which are true correspond to the operations that should be executed.
   */
  public void doOps(BitSet availableOps) {

    boolean useTransactions = getInitialImage.InitImagePrms.useTransactions();

    while (availableOps.cardinality() != 0) {
      int whichOp = getOp(availableOps, operations.length);
      boolean doneWithOps = false;

      if (useTransactions) {
        TxHelper.begin();
      }

      switch (whichOp) {
        case ADD_NEW_KEY:
          doneWithOps = addNewKey();
          break;
        case INVALIDATE:
          doneWithOps = invalidate();
          break;
        case DESTROY:
          doneWithOps = destroy();
          break;
        case UPDATE_EXISTING_KEY:
          doneWithOps = updateExistingKey();
          break;
        case GET:
          doneWithOps = get();
          break;
        case LOCAL_INVALIDATE:
          doneWithOps = localInvalidate();
          break;
        case LOCAL_DESTROY:
          doneWithOps = localDestroy();
          break;
        default:
          {
            throw new TestException("Unknown operation " + whichOp);
          }
      }

      if (useTransactions) {
        try {
          TxHelper.commit();
        } catch (CommitConflictException e) {
          // currently not expecting any conflicts ...
          throw new TestException(
              "Unexpected CommitConflictException " + TestHelper.getStackTrace(e));
        }
      }

      if (doneWithOps) {
        Log.getLogWriter().info("Done with operation " + whichOp);
        availableOps.clear(whichOp);
      }
      if (sleepBetweenOps) {
        Log.getLogWriter().info("Sleeping between ops for " + SLEEP_BETWEEN_OPS_MILLIS + " millis");
        MasterController.sleepForMs(SLEEP_BETWEEN_OPS_MILLIS);
      }
    }
  }
  /**
   * Do operations and pause when directed by the snapshotController Also, writes the snapshot to
   * the blackboard and exports snapshot to disk
   */
  protected static void snapshotResponder(RecoveryTest testInstance) throws Exception {
    long pausing = RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.pausing);
    if (pausing > 0) { // controller has signaled to pause
      if ((Boolean)
          (testInstance.threadIsPaused
              .get())) { // this thread has already paused, so don't increment counter
        Log.getLogWriter().info("Thread has paused");
      } else {
        Log.getLogWriter().info("This thread is pausing");
        testInstance.threadIsPaused.set(new Boolean(true));
        RecoveryBB.getBB().getSharedCounters().incrementAndRead(RecoveryBB.pausing);
      }
      long writeSnapshot = RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.writeSnapshot);
      if (writeSnapshot > 0) {
        long leader = RecoveryBB.getBB().getSharedCounters().incrementAndRead(RecoveryBB.leader);
        if (leader == 1) { // this is the thread to write the snapshot
          Log.getLogWriter().info("This thread is the leader; it will write the snapshot");
          testInstance.writeSnapshot(true); // include non-persistent regions
          long executionNumber =
              RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.executionNumber);

          // Add numFilterObjects entries to each region (SnapshotFilter tests)
          if (SnapshotPrms.useFilterOnExport() || SnapshotPrms.useFilterOnImport()) {
            Map allRegionsSnapshot = new HashMap();
            int numToCreate = SnapshotPrms.numFilterObjects();
            for (Region aRegion : testInstance.allRegions) {
              for (int i = 1; i <= numToCreate; i++) {
                String key = "FilterObject_" + i;
                String value =
                    "object to be filtered via snapshot.save() or snapshot.load(): this should never be a value in the cache once snapshot restored";
                aRegion.put(key, value);
              }
            }
            Log.getLogWriter()
                .info("Wrote " + numToCreate + " FilterObject entries to each region");
          }

          CacheSnapshotService snapshot = CacheHelper.getCache().getSnapshotService();
          SnapshotOptions options = snapshot.createOptions();
          if (SnapshotPrms.useFilterOnExport()) {
            options.setFilter(getSnapshotFilter());
          }

          String currDirName = System.getProperty("user.dir");
          String snapshotDirName =
              currDirName + File.separator + "cacheSnapshotDir_" + executionNumber;
          Log.getLogWriter().info("Starting cacheSnapshot to " + snapshotDirName);
          snapshot.save(new File(snapshotDirName), SnapshotFormat.GEMFIRE, options);
          Log.getLogWriter().info("Completed cacheSnapshot to " + snapshotDirName);
          RecoveryBB.getBB().getSharedCounters().increment(RecoveryBB.snapshotWritten);
        }
      }
      MasterController.sleepForMs(5000);
    } else { // not pausing
      long minTaskGranularitySec = TestConfig.tab().longAt(TestHelperPrms.minTaskGranularitySec);
      long minTaskGranularityMS = minTaskGranularitySec * TestHelper.SEC_MILLI_FACTOR;
      testInstance.doOperations(minTaskGranularityMS);
    }
  }
Exemplo n.º 3
0
 /** Wait for the given vmid to recognize a severe alert. */
 public static void waitForSevereAlert(int vmID) {
   Log.getLogWriter().info("Waiting for vmID " + vmID + " to recognize a severe alert");
   String key = ControllerBB.SevereAlertKey + vmID;
   while (true) {
     if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
       Log.getLogWriter().info("Done waiting for vmID " + vmID + " to recognize a severe alert");
       return;
     } else {
       MasterController.sleepForMs(2000);
     }
     checkForError();
   }
 }
Exemplo n.º 4
0
 /** Wait for the given vmid to become dead. */
 public static void waitForIsDead(int vmID) {
   Log.getLogWriter().info("Waiting for vmID " + vmID + " to become dead");
   String key = ControllerBB.IsDeadKey + vmID;
   while (true) {
     if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
       Log.getLogWriter().info("Done waiting for vmID " + vmID + " to become dead");
       return;
     } else {
       MasterController.sleepForMs(2000);
     }
     checkForError();
   }
 }
Exemplo n.º 5
0
 /** Wait for the given vm to restart. */
 public static void waitForRestartedVm(int vmID) {
   Log.getLogWriter().info("Waiting for vmID " + vmID + " to restart");
   String key = ControllerBB.InitIsCompleteKey + vmID;
   while (true) {
     if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
       Log.getLogWriter().info("Done Waiting for vmID " + vmID + " to restart");
       return;
     } else {
       MasterController.sleepForMs(2000);
     }
     checkForError();
   }
 }
Exemplo n.º 6
0
 /** Wait for membership failure to complete. */
 public static void waitMembershipFailureComplete(int vmID) {
   Log.getLogWriter().info("Waiting for vmID " + vmID + " to complete membership failure");
   String key = ControllerBB.MembershipFailureCompleteKey + vmID;
   while (true) {
     if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
       Log.getLogWriter()
           .info("Done waiting for vmID " + vmID + " to complete membership failure");
       return;
     } else {
       MasterController.sleepForMs(2000);
     }
     checkForError();
   }
 }
 protected void doAfterCreate(EntryEvent event) {
   if (ControllerBB.isPlayDeadEnabled()) {
     SBUtil.playDead();
   }
   if (ControllerBB.isSlowListenerEnabled()) { // slow
     logCall("afterCreate", event);
     Log.getLogWriter().info("Sleeping while slow listeners are enabled...");
     do {
       MasterController.sleepForMs(5000);
     } while (ControllerBB.isSlowListenerEnabled());
     Log.getLogWriter().info("Done being slow, returning from ForcedDiscListener.afterCreate");
   } else if (ControllerBB.isSicknessEnabled()) { // sick
     logCall("afterCreate", event);
     SBUtil.beSick();
   }
 }
  /**
   * Verify that the system is waiting for member(s) to come up before recovering.
   *
   * @param expectedWaitingForMember This specifies the vmID of the member the system should be
   *     waiting for.
   * @param membersWaiting This is a List of the members that should be waiting.
   */
  protected static void verifyWaiting(
      ClientVmInfo expectedWaitingForMember, List<ClientVmInfo> membersWaiting) {
    AdminDistributedSystem adminDS = AdminHelper.getAdminDistributedSystem();
    boolean conditionPreviouslyMet = false;
    do {
      try {
        Set<PersistentID> waitingForMembers = adminDS.getMissingPersistentMembers();
        Log.getLogWriter()
            .info(
                "System is waiting for "
                    + waitingForMembers.size()
                    + " member(s); "
                    + waitingForMembers
                    + ", test is expecting the system to be waiting for member with vmID "
                    + expectedWaitingForMember.getVmid());
        Set vmsWaitingFor = new TreeSet();
        for (PersistentID id : waitingForMembers) {
          String diskDirWaitingFor = id.getDirectory();
          String searchStr = "vm_";
          int index = diskDirWaitingFor.indexOf(searchStr) + searchStr.length();
          int index2 = diskDirWaitingFor.indexOf("_", index);
          String vmIdStr = diskDirWaitingFor.substring(index, index2);
          vmsWaitingFor.add(vmIdStr);
        }

        if (vmsWaitingFor.size() == 1) { // waiting for correct number of members
          String singleVmWaitingFor = (String) vmsWaitingFor.iterator().next();
          if (singleVmWaitingFor.equals(
              expectedWaitingForMember
                  .getVmid()
                  .toString())) { // waiting for 1 member only; it is the correct member
            // now we found what we were waiting for, but to make sure the product sticks with this
            // over a period of time; we will sleep and check again
            Log.getLogWriter().info("System is waiting for expected member " + waitingForMembers);
            if (conditionPreviouslyMet) { // this is the second time the condition has been met
              verifyMissingDiskStoresCommandLineTool(waitingForMembers);
              Log.getLogWriter()
                  .info(
                      "Verified that the system is waiting on vm ID "
                          + expectedWaitingForMember.getVmid()
                          + "; system is waiting for vm with ID "
                          + singleVmWaitingFor);
              return;
            } else { // this if the first time the correct condition has been met
              conditionPreviouslyMet = true;
              Log.getLogWriter()
                  .info(
                      "Sleeping for 180 seconds to make sure the member the system is waiting for does not change");
              MasterController.sleepForMs(180000);
            }
          } else {
            Log.getLogWriter()
                .info(
                    "System is not waiting for the expected member "
                        + expectedWaitingForMember.getVmid());
            conditionPreviouslyMet = false;
          }
        } else {
          Log.getLogWriter()
              .info(
                  "System is not waiting for the expected member "
                      + expectedWaitingForMember.getVmid());
          conditionPreviouslyMet = false;
        }
      } catch (AdminException e) {
        throw new TestException(TestHelper.getStackTrace(e));
      }

      MasterController.sleepForMs(2000);
    } while (true);
  }