/**
* Forces recovery by revoking the members the system is waiting for. It is the responsibility of
* the caller to know that the correct members are being waited for since this will revoke all
* waiting members.
*
* @param doValidation If true, then validate the missing disk stores (used for serial tests or
* tests that are in a silent phase), if false then don't do validation (used for concurrent
* tests).
*/
public static void forceRecovery(boolean doValidation) {
AdminDistributedSystem adminDS = AdminHelper.getAdminDistributedSystem();
Set<PersistentID> waitingForMembers;
try {
waitingForMembers = adminDS.getMissingPersistentMembers();
if (doValidation) {
verifyMissingDiskStoresCommandLineTool(waitingForMembers);
}
for (PersistentID persistId : waitingForMembers) {
boolean revokeWithCommandLineTool = TestConfig.tab().getRandGen().nextBoolean();
// workaround for bug 42432; when this bug is fixed remove the following if
if (HostHelper.isWindows()) {
revokeWithCommandLineTool = false; // force usage of API
}
if (CliHelperPrms.getUseCli()) {
revokeWithCommandLineTool = true;
}
if (revokeWithCommandLineTool) { // revoke with command line tool
PersistenceUtil.runRevokeMissingDiskStore(persistId);
} else {
Log.getLogWriter().info("Revoking PersistentID " + persistId);
adminDS.revokePersistentMember(persistId.getUUID());
}
}
} catch (AdminException e) {
throw new TestException(TestHelper.getStackTrace(e));
}
}
/**
* Verify that the system is waiting for member(s) to come up before recovering.
*
* @param expectedWaitingForMember This specifies the vmID of the member the system should be
* waiting for.
* @param membersWaiting This is a List of the members that should be waiting.
*/
protected static void verifyWaiting(
ClientVmInfo expectedWaitingForMember, List<ClientVmInfo> membersWaiting) {
AdminDistributedSystem adminDS = AdminHelper.getAdminDistributedSystem();
boolean conditionPreviouslyMet = false;
do {
try {
Set<PersistentID> waitingForMembers = adminDS.getMissingPersistentMembers();
Log.getLogWriter()
.info(
"System is waiting for "
+ waitingForMembers.size()
+ " member(s); "
+ waitingForMembers
+ ", test is expecting the system to be waiting for member with vmID "
+ expectedWaitingForMember.getVmid());
Set vmsWaitingFor = new TreeSet();
for (PersistentID id : waitingForMembers) {
String diskDirWaitingFor = id.getDirectory();
String searchStr = "vm_";
int index = diskDirWaitingFor.indexOf(searchStr) + searchStr.length();
int index2 = diskDirWaitingFor.indexOf("_", index);
String vmIdStr = diskDirWaitingFor.substring(index, index2);
vmsWaitingFor.add(vmIdStr);
}
if (vmsWaitingFor.size() == 1) { // waiting for correct number of members
String singleVmWaitingFor = (String) vmsWaitingFor.iterator().next();
if (singleVmWaitingFor.equals(
expectedWaitingForMember
.getVmid()
.toString())) { // waiting for 1 member only; it is the correct member
// now we found what we were waiting for, but to make sure the product sticks with this
// over a period of time; we will sleep and check again
Log.getLogWriter().info("System is waiting for expected member " + waitingForMembers);
if (conditionPreviouslyMet) { // this is the second time the condition has been met
verifyMissingDiskStoresCommandLineTool(waitingForMembers);
Log.getLogWriter()
.info(
"Verified that the system is waiting on vm ID "
+ expectedWaitingForMember.getVmid()
+ "; system is waiting for vm with ID "
+ singleVmWaitingFor);
return;
} else { // this if the first time the correct condition has been met
conditionPreviouslyMet = true;
Log.getLogWriter()
.info(
"Sleeping for 180 seconds to make sure the member the system is waiting for does not change");
MasterController.sleepForMs(180000);
}
} else {
Log.getLogWriter()
.info(
"System is not waiting for the expected member "
+ expectedWaitingForMember.getVmid());
conditionPreviouslyMet = false;
}
} else {
Log.getLogWriter()
.info(
"System is not waiting for the expected member "
+ expectedWaitingForMember.getVmid());
conditionPreviouslyMet = false;
}
} catch (AdminException e) {
throw new TestException(TestHelper.getStackTrace(e));
}
MasterController.sleepForMs(2000);
} while (true);
}
/**
* Invokes AdminDistributedSystem.shutDownAllMembers() which disconnects all members but leaves
* the vms up (because hydra threads remain) including the possibility of this vm being
* disconnected, then this actually stops those vms (except this vm if it was targeted in the
* shutDownAllMembers...this vm will remain up but disconnect). Stopped vms are stopped with
* ON_DEMAND restart. This returns when the vms disconnected by shutDownAllMembers() (other than
* this one) are all stopped .
*
* @param adminDS The admin distributed system instance to use to call shutdownAllMembers.
* @param stopModes The stop modes to choose from.
* @return An Array [0] List of {@link ClientVmInfo} instances describing the VMs that were
* stopped. [1] Set, the return from shutdownAllMembers()
* @throws AdminException if the shutDownAllMembers call throws this exception.
*/
public static Object[] shutDownAllMembers(
AdminDistributedSystem adminDS, List<String> stopModes) {
if (adminDS == null) {
throw new HydraRuntimeException("AdminDistributedSystem cannot be null");
}
// Invoke shutDownAllMembers
Log.getLogWriter().info("AdminDS " + adminDS + " is shutting down all members...");
Set<DistributedMember> memberSet;
try {
long startTime = System.currentTimeMillis();
memberSet = adminDS.shutDownAllMembers();
long duration = System.currentTimeMillis() - startTime;
Log.getLogWriter()
.info(
"AdminDS "
+ adminDS
+ " shut down (disconnected) the following members "
+ "(vms remain up): "
+ memberSet
+ "; shutDownAll duration "
+ duration
+ "ms");
} catch (AdminException e1) {
throw new TestException(TestHelper.getStackTrace(e1));
}
// Now actually stop the vms.
// First get the ClientVmInfos for the members that shutDownAllMembers
// disconnected.
List<ClientVmInfo> allClientInfos = new ArrayList(); // all members that were shutdown
List<ClientVmInfo> allOtherClientInfos =
new ArrayList(); // all members that were shutdown except this member
ClientVmInfo thisClientInfo =
null; // this member, or will remain null if this member was not shutdown
List<String> stopModesToUse = new ArrayList();
for (DistributedMember aMember : memberSet) {
Integer vmId = null;
try {
vmId =
new Integer(RemoteTestModule.Master.getVmid(aMember.getHost(), aMember.getProcessId()));
} catch (java.rmi.RemoteException e) {
throw new HydraRuntimeException("Unable to get vmID for " + aMember + ": " + e);
}
ClientVmInfo infoFromBB =
(ClientVmInfo) StopStartBB.getBB().getSharedMap().get("StopStartVMInfo_for_vmid_" + vmId);
String clientName = null;
if (infoFromBB != null) {
clientName = infoFromBB.getClientName();
}
ClientVmInfo info = new ClientVmInfo(vmId, clientName, null);
allClientInfos.add(info);
if (vmId == RemoteTestModule.getMyVmid()) {
// shutdownAll disconnected this vm
thisClientInfo = info;
} else { // aMember is not the current vm
allOtherClientInfos.add(info);
}
stopModesToUse.add(
stopModes.get(TestConfig.tab().getRandGen().nextInt(0, stopModes.size() - 1)));
}
// now actually stop the vms; if this vm is included, do it last
Object[] returnArr = new Object[2];
if (thisClientInfo == null) { // shutDownAllMembers did not disconnect this vm
// we can just stop all of them now and this vm lives on
StopStartVMs.stopVMs(allClientInfos, stopModesToUse); // restart is ON_DEMAND
returnArr[0] = allClientInfos;
} else { // this vm was disconnected by shutDownAllMembers
// first shutdown all other members except this one
StopStartVMs.stopVMs(allOtherClientInfos, stopModesToUse.subList(0, stopModesToUse.size()));
returnArr[0] = allOtherClientInfos;
}
returnArr[1] = memberSet;
return returnArr;
}
