/**
  * Invalidate a key in region REGION_NAME. The keys to invalidate are specified in keyIntervals.
  *
  * @return true if all keys to be invalidated have been completed.
  */
 protected boolean invalidate() {
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   long nextKey = sc.incrementAndRead(CQUtilBB.LASTKEY_INVALIDATE);
   if (!keyIntervals.keyInRange(KeyIntervals.INVALIDATE, nextKey)) {
     Log.getLogWriter().info("All existing keys invalidated; returning from invalidate");
     return true;
   }
   Object key = NameFactory.getObjectNameForCounter(nextKey);
   Log.getLogWriter().info("Invalidating " + key);
   checkContainsValueForKey(key, true, "before invalidate");
   try {
     aRegion.invalidate(key);
     Log.getLogWriter()
         .info(
             "Done invalidating "
                 + key
                 + ", num remaining: "
                 + (keyIntervals.getLastKey(KeyIntervals.INVALIDATE) - nextKey));
   } catch (TimeoutException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   } catch (EntryNotFoundException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   }
   return (nextKey >= keyIntervals.getLastKey(KeyIntervals.INVALIDATE));
 }
 /**
  * Update an existing key in region REGION_NAME. The keys to update are specified in keyIntervals.
  *
  * @return true if all keys to be updated have been completed.
  */
 protected boolean updateExistingKey() {
   long nextKey =
       CQUtilBB.getBB().getSharedCounters().incrementAndRead(CQUtilBB.LASTKEY_UPDATE_EXISTING_KEY);
   if (!keyIntervals.keyInRange(KeyIntervals.UPDATE_EXISTING_KEY, nextKey)) {
     Log.getLogWriter().info("All existing keys updated; returning from updateExistingKey");
     return true;
   }
   Object key = NameFactory.getObjectNameForCounter(nextKey);
   QueryObject existingValue = (QueryObject) aRegion.get(key);
   if (existingValue == null)
     throw new TestException("Get of key " + key + " returned unexpected " + existingValue);
   QueryObject newValue = existingValue.modifyWithNewInstance(QueryObject.NEGATE, 0, true);
   newValue.extra = key; // encode the key in the object for later validation
   if (existingValue.aPrimitiveLong < 0)
     throw new TestException(
         "Trying to update a key which was already updated: " + existingValue.toStringFull());
   Log.getLogWriter()
       .info("Updating existing key " + key + " with value " + TestHelper.toString(newValue));
   aRegion.put(key, newValue);
   Log.getLogWriter()
       .info(
           "Done updating existing key "
               + key
               + " with value "
               + TestHelper.toString(newValue)
               + ", num remaining: "
               + (keyIntervals.getLastKey(KeyIntervals.UPDATE_EXISTING_KEY) - nextKey));
   return (nextKey >= keyIntervals.getLastKey(KeyIntervals.UPDATE_EXISTING_KEY));
 }
 /**
  * Destroy a key in region REGION_NAME. The keys to destroy are specified in keyIntervals.
  *
  * @return true if all keys to be destroyed have been completed.
  */
 protected boolean destroy() {
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   long nextKey = sc.incrementAndRead(CQUtilBB.LASTKEY_DESTROY);
   if (!keyIntervals.keyInRange(KeyIntervals.DESTROY, nextKey)) {
     Log.getLogWriter().info("All destroys completed; returning from destroy");
     return true;
   }
   Object key = NameFactory.getObjectNameForCounter(nextKey);
   Log.getLogWriter().info("Destroying " + key);
   checkContainsValueForKey(key, true, "before destroy");
   try {
     aRegion.destroy(key);
     Log.getLogWriter()
         .info(
             "Done Destroying "
                 + key
                 + ", num remaining: "
                 + (keyIntervals.getLastKey(KeyIntervals.DESTROY) - nextKey));
   } catch (CacheWriterException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   } catch (TimeoutException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   } catch (EntryNotFoundException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   }
   return (nextKey >= keyIntervals.getLastKey(KeyIntervals.DESTROY));
 }
 /**
  * Do a get on a key in region REGION_NAME. Keys to get are specified in keyIntervals.
  *
  * @return true if all keys to have get performaed have been completed.
  */
 protected boolean get() {
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   long nextKey = sc.incrementAndRead(CQUtilBB.LASTKEY_GET);
   if (!keyIntervals.keyInRange(KeyIntervals.GET, nextKey)) {
     Log.getLogWriter().info("All gets completed; returning from get");
     return true;
   }
   Object key = NameFactory.getObjectNameForCounter(nextKey);
   Log.getLogWriter().info("Getting " + key);
   try {
     Object existingValue = aRegion.get(key);
     Log.getLogWriter()
         .info(
             "Done getting "
                 + key
                 + ", num remaining: "
                 + (keyIntervals.getLastKey(KeyIntervals.GET) - nextKey));
     if (existingValue == null)
       throw new TestException("Get of key " + key + " returned unexpected " + existingValue);
   } catch (TimeoutException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   } catch (CacheLoaderException e) {
     throw new TestException(TestHelper.getStackTrace(e));
   }
   return (nextKey >= keyIntervals.getLastKey(KeyIntervals.GET));
 }
 /**
  * Hydra task to verify queries on the region after all ops. This MUST be called as a batched
  * task, and will throw StopSchedulingTaskOnClientOrder when completed. It is necessary to
  * reinitialize the verify state variables if this is called a second time in this VM, however an
  * error is thrown if a second attempt it made without resetting the state variables.
  */
 public static void HydraTask_verifyQueries() {
   if (CQsOn) {
     // the region contents verify task comes first, so if we got to this
     // task, then we know the the region is a perfect snapshot of what
     // we expect; the CQTestInstance relies on a snapshot
     if (verifyQueryIterator == null) {
       testInstance.CQTestInstance.regionSnapshot = new HashMap(testInstance.aRegion);
       verifyQueryIterator = testInstance.queryMap.keySet().iterator();
     }
     if (verifyQueryIterator.hasNext()) {
       verifyQueryCount++;
       String cqName = (String) (verifyQueryIterator.next());
       Log.getLogWriter()
           .info(
               "Verifying query "
                   + verifyQueryCount
                   + " out of "
                   + testInstance.queryMap.size()
                   + " with name "
                   + cqName);
       testInstance.CQTestInstance.verifyQuery(cqName);
     } else {
       String aStr = "Done verifying " + verifyQueryCount + " queries";
       Log.getLogWriter().info(aStr);
       // set for next time; if this is an init task, then we will be set for the close task
       verifyQueryIterator = null;
       verifyQueryCount = 0;
       throw new StopSchedulingTaskOnClientOrder(aStr);
     }
   } else {
     String aStr = "Skipping verification of queries because CQUtilPrms.CQsOn is " + CQsOn;
     Log.getLogWriter().info(aStr);
     throw new StopSchedulingTaskOnClientOrder(aStr);
   }
 }
  /**
   * Do operations on the REGION_NAME's keys using keyIntervals to specify which keys get which
   * operations. This will return when all operations in all intervals have completed.
   *
   * @param availableOps - Bits which are true correspond to the operations that should be executed.
   */
  public void doOps(BitSet availableOps) {

    boolean useTransactions = getInitialImage.InitImagePrms.useTransactions();

    while (availableOps.cardinality() != 0) {
      int whichOp = getOp(availableOps, operations.length);
      boolean doneWithOps = false;

      if (useTransactions) {
        TxHelper.begin();
      }

      switch (whichOp) {
        case ADD_NEW_KEY:
          doneWithOps = addNewKey();
          break;
        case INVALIDATE:
          doneWithOps = invalidate();
          break;
        case DESTROY:
          doneWithOps = destroy();
          break;
        case UPDATE_EXISTING_KEY:
          doneWithOps = updateExistingKey();
          break;
        case GET:
          doneWithOps = get();
          break;
        case LOCAL_INVALIDATE:
          doneWithOps = localInvalidate();
          break;
        case LOCAL_DESTROY:
          doneWithOps = localDestroy();
          break;
        default:
          {
            throw new TestException("Unknown operation " + whichOp);
          }
      }

      if (useTransactions) {
        try {
          TxHelper.commit();
        } catch (CommitConflictException e) {
          // currently not expecting any conflicts ...
          throw new TestException(
              "Unexpected CommitConflictException " + TestHelper.getStackTrace(e));
        }
      }

      if (doneWithOps) {
        Log.getLogWriter().info("Done with operation " + whichOp);
        availableOps.clear(whichOp);
      }
      if (sleepBetweenOps) {
        Log.getLogWriter().info("Sleeping between ops for " + SLEEP_BETWEEN_OPS_MILLIS + " millis");
        MasterController.sleepForMs(SLEEP_BETWEEN_OPS_MILLIS);
      }
    }
  }
  /** Start all cqs running for this VM, and create a CQHistory instance for each CQ. */
  private void startCQsWithHistory() {
    initializeQueryService();
    CqAttributesFactory cqFac = new CqAttributesFactory();
    cqFac.addCqListener(new CQHistoryListener());
    cqFac.addCqListener(new CQGatherListener());
    CqAttributes cqAttr = cqFac.create();
    Iterator it = queryMap.keySet().iterator();
    while (it.hasNext()) {
      String queryName = (String) (it.next());
      String query = (String) (queryMap.get(queryName));
      try {
        CqQuery cq = qService.newCq(queryName, query, cqAttr);
        CQHistory history = new CQHistory(cq.getName());
        CQHistoryListener.recordHistory(history);
        Log.getLogWriter()
            .info(
                "Creating CQ with name " + queryName + ": " + query + ", cq attributes: " + cqAttr);
        Log.getLogWriter().info("Calling executeWithInitialResults on " + cq);
        CqResults rs = cq.executeWithInitialResults();
        SelectResults sr = CQUtil.getSelectResults(rs);
        if (sr == null) {
          throw new TestException(
              "For cq "
                  + cq
                  + " with name "
                  + cq.getName()
                  + " executeWithInitialResults returned "
                  + sr);
        }
        Log.getLogWriter()
            .info(
                "Done calling executeWithInitializResults on "
                    + cq
                    + " with name "
                    + queryName
                    + ", select results size is "
                    + sr.size());
        history.setSelectResults(sr);
        logNumOps();

        // log the select results
        List srList = sr.asList();
        StringBuffer aStr = new StringBuffer();
        aStr.append("SelectResults returned from " + queryName + " is\n");
        for (int i = 0; i < srList.size(); i++) {
          aStr.append(srList.get(i) + "\n");
        }
        Log.getLogWriter().info(aStr.toString());
      } catch (CqExistsException e) {
        throw new TestException(TestHelper.getStackTrace(e));
      } catch (RegionNotFoundException e) {
        throw new TestException(TestHelper.getStackTrace(e));
      } catch (CqException e) {
        throw new TestException(TestHelper.getStackTrace(e));
      }
    }
  }
 /**
  * Register interest with ALL_KEYS, and InterestPolicyResult = KEYS_VALUES which is equivalent to
  * a full GII.
  *
  * @param aRegion The region to register interest on.
  */
 protected static void registerInterest(Region aRegion) {
   Log.getLogWriter()
       .info("Calling registerInterest for all keys, result interest policy KEYS_VALUES");
   aRegion.registerInterest("ALL_KEYS", InterestResultPolicy.KEYS_VALUES);
   Log.getLogWriter()
       .info(
           "Done calling registerInterest for all keys, "
               + "result interest policy KEYS_VALUES, "
               + aRegion.getFullPath()
               + " size is "
               + aRegion.size());
 }
  /**
   * Hydra start task to initialize key intervals, which are ranges of keys which are to have an
   * operation done on them (invalidate, destroy, etc)
   */
  public static synchronized void StartTask_initialize() {
    int numKeys = TestConfig.tab().intAt(CQUtilPrms.numKeys);
    KeyIntervals intervals =
        new KeyIntervals(
            new int[] {
              KeyIntervals.NONE, KeyIntervals.INVALIDATE,
              KeyIntervals.DESTROY, KeyIntervals.UPDATE_EXISTING_KEY,
              KeyIntervals.GET
            },
            numKeys);
    CQUtilBB.getBB().getSharedMap().put(CQUtilBB.KeyIntervals, intervals);
    Log.getLogWriter().info("Created keyIntervals: " + intervals);

    // Set the counters for the next keys to use for each operation
    hydra.blackboard.SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
    sc.setIfLarger(CQUtilBB.LASTKEY_INVALIDATE, intervals.getFirstKey(KeyIntervals.INVALIDATE) - 1);
    sc.setIfLarger(
        CQUtilBB.LASTKEY_LOCAL_INVALIDATE,
        intervals.getFirstKey(KeyIntervals.LOCAL_INVALIDATE) - 1);
    sc.setIfLarger(CQUtilBB.LASTKEY_DESTROY, intervals.getFirstKey(KeyIntervals.DESTROY) - 1);
    sc.setIfLarger(
        CQUtilBB.LASTKEY_LOCAL_DESTROY, intervals.getFirstKey(KeyIntervals.LOCAL_DESTROY) - 1);
    sc.setIfLarger(
        CQUtilBB.LASTKEY_UPDATE_EXISTING_KEY,
        intervals.getFirstKey(KeyIntervals.UPDATE_EXISTING_KEY) - 1);
    sc.setIfLarger(CQUtilBB.LASTKEY_GET, intervals.getFirstKey(KeyIntervals.GET) - 1);

    // show the blackboard
    CQUtilBB.getBB().printSharedMap();
    CQUtilBB.getBB().printSharedCounters();
  }
 protected void initializeQueryService() {
   try {
     String usingPool = TestConfig.tab().stringAt(CQUtilPrms.QueryServiceUsingPool, "false");
     boolean queryServiceUsingPool = Boolean.valueOf(usingPool).booleanValue();
     if (queryServiceUsingPool) {
       Pool pool = PoolHelper.createPool(CQUtilPrms.getQueryServicePoolName());
       qService = pool.getQueryService();
       Log.getLogWriter()
           .info("Initializing QueryService using Pool. PoolName: " + pool.getName());
     } else {
       qService = CacheHelper.getCache().getQueryService();
       Log.getLogWriter().info("Initializing QueryService using Cache.");
     }
   } catch (Exception e) {
     throw new TestException(TestHelper.getStackTrace(e));
   }
 }
 /**
  * Load a region with keys and values. The number of keys and values is specified by the total
  * number of keys in keyIntervals. This can be invoked by several threads to accomplish the work.
  */
 public void loadRegion() {
   final long LOG_INTERVAL_MILLIS = 10000;
   int numKeysToCreate = keyIntervals.getNumKeys();
   long lastLogTime = System.currentTimeMillis();
   long startTime = System.currentTimeMillis();
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   do {
     long shouldAddCount =
         CQUtilBB.getBB().getSharedCounters().incrementAndRead(CQUtilBB.SHOULD_ADD_COUNT);
     if (shouldAddCount > numKeysToCreate) {
       String aStr =
           "In loadRegion, shouldAddCount is "
               + shouldAddCount
               + ", numOriginalKeysCreated is "
               + sc.read(CQUtilBB.NUM_ORIGINAL_KEYS_CREATED)
               + ", numKeysToCreate is "
               + numKeysToCreate
               + ", region size is "
               + aRegion.size();
       Log.getLogWriter().info(aStr);
       NameBB.getBB().printSharedCounters();
       throw new StopSchedulingTaskOnClientOrder(aStr);
     }
     Object key = NameFactory.getNextPositiveObjectName();
     QueryObject value = getValueToAdd(key);
     value.extra = key;
     Log.getLogWriter().info("Creating with put, key " + key + ", value " + value.toStringFull());
     aRegion.put(key, value);
     sc.increment(CQUtilBB.NUM_ORIGINAL_KEYS_CREATED);
     if (System.currentTimeMillis() - lastLogTime > LOG_INTERVAL_MILLIS) {
       Log.getLogWriter()
           .info(
               "Added "
                   + NameFactory.getPositiveNameCounter()
                   + " out of "
                   + numKeysToCreate
                   + " entries into "
                   + TestHelper.regionToString(aRegion, false));
       lastLogTime = System.currentTimeMillis();
     }
   } while ((minTaskGranularitySec == -1)
       || (System.currentTimeMillis() - startTime < minTaskGranularityMS));
 }
 /**
  * Verify all queries by combining the results of executeWithInitialResults and subsequent
  * cqEvents.
  */
 private void verifyQueriesCombine() {
   if (CQsOn) {
     // the region contents verify task comes first, so if we got to this
     // task, then we know the the region is a perfect snapshot of what
     // we expect; the CQTestInstance relies on a snapshot
     testInstance.CQTestInstance.regionSnapshot = new HashMap(testInstance.aRegion);
     verifyQueryResultsCombine();
   } else {
     Log.getLogWriter()
         .info("Skipping verification of queries because CQUtilPrms.CQsOn is " + CQsOn);
   }
 }
 /** Initialize fields for this instance */
 public void initializeInstance() {
   numNewKeys = TestConfig.tab().intAt(CQUtilPrms.numNewKeys, -1);
   keyIntervals = (KeyIntervals) (CQUtilBB.getBB().getSharedMap().get(CQUtilBB.KeyIntervals));
   Log.getLogWriter()
       .info("initInstance, keyIntervals read from blackboard = " + keyIntervals.toString());
   int numDestroyed = keyIntervals.getNumKeys(KeyIntervals.DESTROY);
   int numKeyIntervals = keyIntervals.getNumKeys();
   totalNumKeys = numKeyIntervals + numNewKeys - numDestroyed;
   minTaskGranularitySec = TestConfig.tab().longAt(TestHelperPrms.minTaskGranularitySec, -1);
   minTaskGranularityMS = -1;
   if (minTaskGranularitySec != -1) {
     minTaskGranularityMS = minTaskGranularitySec * TestHelper.SEC_MILLI_FACTOR;
   }
   queryDepth = TestConfig.tab().intAt(CQUtilPrms.queryDepth, 1);
   Vector bridgeNames = TestConfig.tab().vecAt(BridgePrms.names, null);
   isBridgeConfiguration = bridgeNames != null;
   CQsOn = TestConfig.tab().booleanAt(CQUtilPrms.CQsOn, true);
   CQTestInstance = new CQTest();
   CQTestInstance.initializeInstance();
   Log.getLogWriter().info("numKeyIntervals is " + numKeyIntervals);
   Log.getLogWriter().info("numNewKeys is " + numNewKeys);
   Log.getLogWriter().info("numDestroyed is " + numDestroyed);
   Log.getLogWriter().info("totalNumKeys is " + totalNumKeys);
 }
 /**
  * Add a new key to REGION_NAME.
  *
  * @return true if all new keys have been added (specified by CQUtilPrms.numNewKeys)
  */
 protected boolean addNewKey() {
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   long numNewKeysCreated = sc.incrementAndRead(CQUtilBB.NUM_NEW_KEYS_CREATED);
   if (numNewKeysCreated > numNewKeys) {
     Log.getLogWriter().info("All new keys created; returning from addNewKey");
     return true;
   }
   Object key = NameFactory.getNextPositiveObjectName();
   checkContainsValueForKey(key, false, "before addNewKey");
   QueryObject value =
       new QueryObject(
           NameFactory.getCounterForName(key), QueryObject.EQUAL_VALUES, -1, queryDepth);
   value.extra = key; // encode the key in the value for later validation
   Log.getLogWriter().info("Adding new key " + key + " with put");
   aRegion.put(key, value);
   Log.getLogWriter()
       .info(
           "Done adding new key "
               + key
               + " with put, "
               + "num remaining: "
               + (numNewKeys - numNewKeysCreated));
   return (numNewKeysCreated >= numNewKeys);
 }
 /** Log the number of ops that have completed. */
 protected void logNumOps() {
   SharedCounters sc = CQUtilBB.getBB().getSharedCounters();
   int totalOps =
       keyIntervals.getNumKeys() - keyIntervals.getNumKeys(KeyIntervals.NONE) + numNewKeys;
   long numOpsCompleted =
       sc.read(CQUtilBB.LASTKEY_INVALIDATE)
           - keyIntervals.getFirstKey(KeyIntervals.INVALIDATE)
           + sc.read(CQUtilBB.LASTKEY_DESTROY)
           - keyIntervals.getFirstKey(KeyIntervals.DESTROY)
           + sc.read(CQUtilBB.LASTKEY_UPDATE_EXISTING_KEY)
           - keyIntervals.getFirstKey(KeyIntervals.UPDATE_EXISTING_KEY)
           + sc.read(CQUtilBB.LASTKEY_GET)
           - keyIntervals.getFirstKey(KeyIntervals.GET)
           + sc.read(CQUtilBB.NUM_NEW_KEYS_CREATED);
   Log.getLogWriter()
       .info(
           "Total ops is " + totalOps + ", current number of ops completed is " + numOpsCompleted);
 }
 /**
  * Initialize a client vm.
  *
  * @param startCQsRunning true if the initialize should start the CQs running false otherwise
  */
 private void initializeClient(boolean startCQsRunning) {
   initializeRegion("clientRegion");
   initializeInstance();
   if (isBridgeConfiguration) {
     isBridgeClient = true;
     registerInterest(testInstance.aRegion);
     if (CQsOn) {
       CQTestInstance.initializeQueryService();
       queryMap = testInstance.CQTestInstance.generateQueries(queryDepth);
       CQTestInstance.queryMap = queryMap;
       CQTestInstance.aRegion = aRegion;
       if (startCQsRunning) {
         CQTestInstance.initializeCQs();
       }
     } else {
       Log.getLogWriter().info("Not creating CQs because CQUtilPrms.CQsOn is " + CQsOn);
     }
   }
 }
  /**
   * Verify the result of the CQs registered for this VM. Query results from
   * executeWithInitialResults are combined with subsequent cq events to form the final result.
   */
  protected void verifyQueryResultsCombine() {
    Log.getLogWriter().info("In verifyQueryResultsCombine");
    Iterator it = queryMap.keySet().iterator();
    int count = 0;
    while (it.hasNext()) {
      count++;
      String cqName = (String) (it.next());
      Log.getLogWriter()
          .info("Verifying query " + count + " out of " + queryMap.size() + " with name " + cqName);
      CqQuery cq = qService.getCq(cqName);
      String queryStr = cq.getQueryString();
      String readableQueryStr = CQTest.getReadableQueryString(queryStr);

      // combine the initial selectResults with history of events
      CQHistory history = CQHistoryListener.getCQHistory(cqName);
      Map combinedMap = history.getCombinedResults();
      List combinedList = new ArrayList(combinedMap.values());
      List expectedResults = CQTestInstance.getExpectedResults(queryStr);
      List missingInCombined = new ArrayList(expectedResults);
      List unexpectedInCombined = new ArrayList(combinedList);
      unexpectedInCombined.removeAll(expectedResults);
      missingInCombined.removeAll(combinedList);

      // prepare error Strings
      StringBuffer aStr = new StringBuffer();
      if (unexpectedInCombined.size() > 0) {
        String tmpStr =
            getLocationString(unexpectedInCombined, expectedResults, history)
                + "\n"
                + "Found the following "
                + unexpectedInCombined.size()
                + " unexpected elements in combined results for cq "
                + cqName
                + ", "
                + readableQueryStr
                + ": "
                + QueryObject.toStringFull(unexpectedInCombined);
        Log.getLogWriter().info(tmpStr);
        aStr.append(tmpStr);
      }
      if (missingInCombined.size() > 0) {
        String tmpStr =
            getLocationString(missingInCombined, expectedResults, history)
                + "\n"
                + "The following "
                + missingInCombined.size()
                + " elements were missing from combined results for cq "
                + cqName
                + ", "
                + readableQueryStr
                + ": "
                + QueryObject.toStringFull(missingInCombined);
        Log.getLogWriter().info(tmpStr);
        aStr.append(tmpStr);
      }

      if (aStr.length() > 0) {
        throw new TestException(
            "Probably bug 38065: For cq "
                + cqName
                + ", "
                + readableQueryStr
                + "\n"
                + aStr.toString());
      }
    }
    Log.getLogWriter().info("Done verifying " + count + " queries");
  }
  /**
   * Verify the contents of the region, taking into account the keys that were destroyed,
   * invalidated, etc (as specified in keyIntervals) Throw an error of any problems are detected.
   * This must be called repeatedly by the same thread until StopSchedulingTaskOnClientOrder is
   * thrown.
   */
  public void verifyRegionContents() {
    final long LOG_INTERVAL_MILLIS = 10000;
    // we already completed this check once; we can't do it again without reinitializing the
    // verify state variables
    if (verifyRegionContentsCompleted) {
      throw new TestException(
          "Test configuration problem; already verified region contents, "
              + "cannot call this task again without resetting batch variables");
    }

    // iterate keys
    long lastLogTime = System.currentTimeMillis();
    long minTaskGranularitySec = TestConfig.tab().longAt(TestHelperPrms.minTaskGranularitySec);
    long minTaskGranularityMS = minTaskGranularitySec * TestHelper.SEC_MILLI_FACTOR;
    long startTime = System.currentTimeMillis();
    long size = aRegion.size();
    boolean first = true;
    int numKeysToCheck = keyIntervals.getNumKeys() + numNewKeys;
    while (verifyRegionContentsIndex < numKeysToCheck) {
      verifyRegionContentsIndex++;
      if (first) {
        Log.getLogWriter()
            .info(
                "In verifyRegionContents, region has "
                    + size
                    + " keys; starting verify at verifyRegionContentsIndex "
                    + verifyRegionContentsIndex
                    + "; verifying key names with indexes through (and including) "
                    + numKeysToCheck);
        first = false;
      }

      // check region size the first time through the loop to avoid it being called
      // multiple times when this is batched
      if (verifyRegionContentsIndex == 1) {
        if (totalNumKeys != size) {
          String tmpStr = "Expected region size to be " + totalNumKeys + ", but it is size " + size;
          Log.getLogWriter().info(tmpStr);
          verifyRegionContentsErrStr.append(tmpStr + "\n");
        }
      }

      Object key = NameFactory.getObjectNameForCounter(verifyRegionContentsIndex);
      try {
        if (((verifyRegionContentsIndex >= keyIntervals.getFirstKey(KeyIntervals.NONE))
                && (verifyRegionContentsIndex <= keyIntervals.getLastKey(KeyIntervals.NONE)))
            || ((verifyRegionContentsIndex >= keyIntervals.getFirstKey(KeyIntervals.GET))
                && (verifyRegionContentsIndex <= keyIntervals.getLastKey(KeyIntervals.GET)))) {
          // this key was untouched after its creation
          checkContainsKey(key, true, "key was untouched");
          checkContainsValueForKey(key, true, "key was untouched");
          Object value = aRegion.get(key);
          checkValue(key, value);
        } else if ((verifyRegionContentsIndex >= keyIntervals.getFirstKey(KeyIntervals.INVALIDATE))
            && (verifyRegionContentsIndex <= keyIntervals.getLastKey(KeyIntervals.INVALIDATE))) {
          checkContainsKey(key, true, "key was invalidated");
          checkContainsValueForKey(key, false, "key was invalidated");
        } else if ((verifyRegionContentsIndex
                >= keyIntervals.getFirstKey(KeyIntervals.LOCAL_INVALIDATE))
            && (verifyRegionContentsIndex
                <= keyIntervals.getLastKey(KeyIntervals.LOCAL_INVALIDATE))) {
          // this key was locally invalidated
          checkContainsKey(key, true, "key was locally invalidated");
          checkContainsValueForKey(key, true, "key was locally invalidated");
          Object value = aRegion.get(key);
          checkValue(key, value);
        } else if ((verifyRegionContentsIndex >= keyIntervals.getFirstKey(KeyIntervals.DESTROY))
            && (verifyRegionContentsIndex <= keyIntervals.getLastKey(KeyIntervals.DESTROY))) {
          // this key was destroyed
          checkContainsKey(key, false, "key was destroyed");
          checkContainsValueForKey(key, false, "key was destroyed");
        } else if ((verifyRegionContentsIndex
                >= keyIntervals.getFirstKey(KeyIntervals.LOCAL_DESTROY))
            && (verifyRegionContentsIndex <= keyIntervals.getLastKey(KeyIntervals.LOCAL_DESTROY))) {
          // this key was locally destroyed
          checkContainsKey(key, true, "key was locally destroyed");
          checkContainsValueForKey(key, true, "key was locally destroyed");
          Object value = aRegion.get(key);
          checkValue(key, value);
        } else if ((verifyRegionContentsIndex
                >= keyIntervals.getFirstKey(KeyIntervals.UPDATE_EXISTING_KEY))
            && (verifyRegionContentsIndex
                <= keyIntervals.getLastKey(KeyIntervals.UPDATE_EXISTING_KEY))) {
          // this key was updated
          checkContainsKey(key, true, "key was updated");
          checkContainsValueForKey(key, true, "key was updated");
          Object value = aRegion.get(key);
          checkUpdatedValue(key, value);
        } else if (verifyRegionContentsIndex > keyIntervals.getNumKeys()) {
          // key was newly added
          checkContainsKey(key, true, "key was new");
          checkContainsValueForKey(key, true, "key was new");
          Object value = aRegion.get(key);
          checkValue(key, value);
        }
      } catch (TestException e) {
        Log.getLogWriter().info(TestHelper.getStackTrace(e));
        verifyRegionContentsErrStr.append(e.getMessage() + "\n");
      }

      if (System.currentTimeMillis() - lastLogTime > LOG_INTERVAL_MILLIS) {
        Log.getLogWriter()
            .info("Verified key " + verifyRegionContentsIndex + " out of " + totalNumKeys);
        lastLogTime = System.currentTimeMillis();
      }

      if (System.currentTimeMillis() - startTime >= minTaskGranularityMS) {
        Log.getLogWriter()
            .info(
                "In HydraTask_verifyRegionContents, returning before completing verify "
                    + "because of task granularity (this task must be batched to complete); last key verified is "
                    + key);
        return; // task is batched; we are done with this batch
      }
    }
    verifyRegionContentsCompleted = true;
    if (verifyRegionContentsErrStr.length() > 0) {
      throw new TestException(verifyRegionContentsErrStr.toString());
    }
    String aStr =
        "In HydraTask_verifyRegionContents, verified " + verifyRegionContentsIndex + " keys/values";
    Log.getLogWriter().info(aStr);
    throw new StopSchedulingTaskOnClientOrder(aStr);
  }