/**
* Do operations on the REGION_NAME's keys using keyIntervals to specify which keys get which
* operations. This will return when all operations in all intervals have completed.
*
* @param availableOps - Bits which are true correspond to the operations that should be executed.
*/
public void doOps(BitSet availableOps) {
boolean useTransactions = getInitialImage.InitImagePrms.useTransactions();
while (availableOps.cardinality() != 0) {
int whichOp = getOp(availableOps, operations.length);
boolean doneWithOps = false;
if (useTransactions) {
TxHelper.begin();
}
switch (whichOp) {
case ADD_NEW_KEY:
doneWithOps = addNewKey();
break;
case INVALIDATE:
doneWithOps = invalidate();
break;
case DESTROY:
doneWithOps = destroy();
break;
case UPDATE_EXISTING_KEY:
doneWithOps = updateExistingKey();
break;
case GET:
doneWithOps = get();
break;
case LOCAL_INVALIDATE:
doneWithOps = localInvalidate();
break;
case LOCAL_DESTROY:
doneWithOps = localDestroy();
break;
default:
{
throw new TestException("Unknown operation " + whichOp);
}
}
if (useTransactions) {
try {
TxHelper.commit();
} catch (CommitConflictException e) {
// currently not expecting any conflicts ...
throw new TestException(
"Unexpected CommitConflictException " + TestHelper.getStackTrace(e));
}
}
if (doneWithOps) {
Log.getLogWriter().info("Done with operation " + whichOp);
availableOps.clear(whichOp);
}
if (sleepBetweenOps) {
Log.getLogWriter().info("Sleeping between ops for " + SLEEP_BETWEEN_OPS_MILLIS + " millis");
MasterController.sleepForMs(SLEEP_BETWEEN_OPS_MILLIS);
}
}
}
/**
* Do operations and pause when directed by the snapshotController Also, writes the snapshot to
* the blackboard and exports snapshot to disk
*/
protected static void snapshotResponder(RecoveryTest testInstance) throws Exception {
long pausing = RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.pausing);
if (pausing > 0) { // controller has signaled to pause
if ((Boolean)
(testInstance.threadIsPaused
.get())) { // this thread has already paused, so don't increment counter
Log.getLogWriter().info("Thread has paused");
} else {
Log.getLogWriter().info("This thread is pausing");
testInstance.threadIsPaused.set(new Boolean(true));
RecoveryBB.getBB().getSharedCounters().incrementAndRead(RecoveryBB.pausing);
}
long writeSnapshot = RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.writeSnapshot);
if (writeSnapshot > 0) {
long leader = RecoveryBB.getBB().getSharedCounters().incrementAndRead(RecoveryBB.leader);
if (leader == 1) { // this is the thread to write the snapshot
Log.getLogWriter().info("This thread is the leader; it will write the snapshot");
testInstance.writeSnapshot(true); // include non-persistent regions
long executionNumber =
RecoveryBB.getBB().getSharedCounters().read(RecoveryBB.executionNumber);
// Add numFilterObjects entries to each region (SnapshotFilter tests)
if (SnapshotPrms.useFilterOnExport() || SnapshotPrms.useFilterOnImport()) {
Map allRegionsSnapshot = new HashMap();
int numToCreate = SnapshotPrms.numFilterObjects();
for (Region aRegion : testInstance.allRegions) {
for (int i = 1; i <= numToCreate; i++) {
String key = "FilterObject_" + i;
String value =
"object to be filtered via snapshot.save() or snapshot.load(): this should never be a value in the cache once snapshot restored";
aRegion.put(key, value);
}
}
Log.getLogWriter()
.info("Wrote " + numToCreate + " FilterObject entries to each region");
}
CacheSnapshotService snapshot = CacheHelper.getCache().getSnapshotService();
SnapshotOptions options = snapshot.createOptions();
if (SnapshotPrms.useFilterOnExport()) {
options.setFilter(getSnapshotFilter());
}
String currDirName = System.getProperty("user.dir");
String snapshotDirName =
currDirName + File.separator + "cacheSnapshotDir_" + executionNumber;
Log.getLogWriter().info("Starting cacheSnapshot to " + snapshotDirName);
snapshot.save(new File(snapshotDirName), SnapshotFormat.GEMFIRE, options);
Log.getLogWriter().info("Completed cacheSnapshot to " + snapshotDirName);
RecoveryBB.getBB().getSharedCounters().increment(RecoveryBB.snapshotWritten);
}
}
MasterController.sleepForMs(5000);
} else { // not pausing
long minTaskGranularitySec = TestConfig.tab().longAt(TestHelperPrms.minTaskGranularitySec);
long minTaskGranularityMS = minTaskGranularitySec * TestHelper.SEC_MILLI_FACTOR;
testInstance.doOperations(minTaskGranularityMS);
}
}
/** Wait for the given vmid to recognize a severe alert. */
public static void waitForSevereAlert(int vmID) {
Log.getLogWriter().info("Waiting for vmID " + vmID + " to recognize a severe alert");
String key = ControllerBB.SevereAlertKey + vmID;
while (true) {
if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
Log.getLogWriter().info("Done waiting for vmID " + vmID + " to recognize a severe alert");
return;
} else {
MasterController.sleepForMs(2000);
}
checkForError();
}
}
/** Wait for the given vmid to become dead. */
public static void waitForIsDead(int vmID) {
Log.getLogWriter().info("Waiting for vmID " + vmID + " to become dead");
String key = ControllerBB.IsDeadKey + vmID;
while (true) {
if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
Log.getLogWriter().info("Done waiting for vmID " + vmID + " to become dead");
return;
} else {
MasterController.sleepForMs(2000);
}
checkForError();
}
}
/** Wait for the given vm to restart. */
public static void waitForRestartedVm(int vmID) {
Log.getLogWriter().info("Waiting for vmID " + vmID + " to restart");
String key = ControllerBB.InitIsCompleteKey + vmID;
while (true) {
if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
Log.getLogWriter().info("Done Waiting for vmID " + vmID + " to restart");
return;
} else {
MasterController.sleepForMs(2000);
}
checkForError();
}
}
/** Wait for membership failure to complete. */
public static void waitMembershipFailureComplete(int vmID) {
Log.getLogWriter().info("Waiting for vmID " + vmID + " to complete membership failure");
String key = ControllerBB.MembershipFailureCompleteKey + vmID;
while (true) {
if (ControllerBB.getBB().getSharedMap().containsKey(key)) {
Log.getLogWriter()
.info("Done waiting for vmID " + vmID + " to complete membership failure");
return;
} else {
MasterController.sleepForMs(2000);
}
checkForError();
}
}
protected void doAfterCreate(EntryEvent event) {
if (ControllerBB.isPlayDeadEnabled()) {
SBUtil.playDead();
}
if (ControllerBB.isSlowListenerEnabled()) { // slow
logCall("afterCreate", event);
Log.getLogWriter().info("Sleeping while slow listeners are enabled...");
do {
MasterController.sleepForMs(5000);
} while (ControllerBB.isSlowListenerEnabled());
Log.getLogWriter().info("Done being slow, returning from ForcedDiscListener.afterCreate");
} else if (ControllerBB.isSicknessEnabled()) { // sick
logCall("afterCreate", event);
SBUtil.beSick();
}
}
/**
* Verify that the system is waiting for member(s) to come up before recovering.
*
* @param expectedWaitingForMember This specifies the vmID of the member the system should be
* waiting for.
* @param membersWaiting This is a List of the members that should be waiting.
*/
protected static void verifyWaiting(
ClientVmInfo expectedWaitingForMember, List<ClientVmInfo> membersWaiting) {
AdminDistributedSystem adminDS = AdminHelper.getAdminDistributedSystem();
boolean conditionPreviouslyMet = false;
do {
try {
Set<PersistentID> waitingForMembers = adminDS.getMissingPersistentMembers();
Log.getLogWriter()
.info(
"System is waiting for "
+ waitingForMembers.size()
+ " member(s); "
+ waitingForMembers
+ ", test is expecting the system to be waiting for member with vmID "
+ expectedWaitingForMember.getVmid());
Set vmsWaitingFor = new TreeSet();
for (PersistentID id : waitingForMembers) {
String diskDirWaitingFor = id.getDirectory();
String searchStr = "vm_";
int index = diskDirWaitingFor.indexOf(searchStr) + searchStr.length();
int index2 = diskDirWaitingFor.indexOf("_", index);
String vmIdStr = diskDirWaitingFor.substring(index, index2);
vmsWaitingFor.add(vmIdStr);
}
if (vmsWaitingFor.size() == 1) { // waiting for correct number of members
String singleVmWaitingFor = (String) vmsWaitingFor.iterator().next();
if (singleVmWaitingFor.equals(
expectedWaitingForMember
.getVmid()
.toString())) { // waiting for 1 member only; it is the correct member
// now we found what we were waiting for, but to make sure the product sticks with this
// over a period of time; we will sleep and check again
Log.getLogWriter().info("System is waiting for expected member " + waitingForMembers);
if (conditionPreviouslyMet) { // this is the second time the condition has been met
verifyMissingDiskStoresCommandLineTool(waitingForMembers);
Log.getLogWriter()
.info(
"Verified that the system is waiting on vm ID "
+ expectedWaitingForMember.getVmid()
+ "; system is waiting for vm with ID "
+ singleVmWaitingFor);
return;
} else { // this if the first time the correct condition has been met
conditionPreviouslyMet = true;
Log.getLogWriter()
.info(
"Sleeping for 180 seconds to make sure the member the system is waiting for does not change");
MasterController.sleepForMs(180000);
}
} else {
Log.getLogWriter()
.info(
"System is not waiting for the expected member "
+ expectedWaitingForMember.getVmid());
conditionPreviouslyMet = false;
}
} else {
Log.getLogWriter()
.info(
"System is not waiting for the expected member "
+ expectedWaitingForMember.getVmid());
conditionPreviouslyMet = false;
}
} catch (AdminException e) {
throw new TestException(TestHelper.getStackTrace(e));
}
MasterController.sleepForMs(2000);
} while (true);
}