/**
* Waits for a List of threads concurrently stopping and starting Vms to finish, as returned by
* {@link #stopStartAsync(List,List)}. Resets the niceKill blackboard entry for each vm is set to
* false.
*
* @param targetVmList A list of hydra.ClientVmInfos to stop then restart.
* @param threadList A parallel list of threads on whom to wait.
*/
public static void joinStopStart(List targetVmList, List threadList) {
Log.getLogWriter().info("Joining stop/start threads...");
// wait for all threads to complete the stop and start
for (int i = 0; i < threadList.size(); i++) {
HydraSubthread aThread = (HydraSubthread) (threadList.get(i));
try {
aThread.join();
} catch (InterruptedException e) {
throw new TestException(TestHelper.getStackTrace(e));
}
Object err = StopStartBB.getBB().getSharedMap().get(StopStartBB.errKey);
if (err != null) {
throw new TestException(err.toString());
}
}
// reset the blackboard entries
for (int i = 0; i < targetVmList.size(); i++) {
ClientVmInfo targetVm = (ClientVmInfo) (targetVmList.get(i));
StopStartBB.getBB()
.getSharedMap()
.put(StopStartVMs.NiceKillInProgress + targetVm.getVmid(), new Boolean(false));
}
Log.getLogWriter().info("Done joining stop/start threads...");
}
/**
* Run method for this thread: if this thread is set to stop, then stop a VM and wait for it to
* stop. If this thread is set to start then start a VM and wait for it to start. Any errors are
* placed in the blackboard errStr.
*/
public void run() {
try {
Log.getLogWriter().info("Started " + getName());
try {
if (stop) {
ClientVmMgr.stop(
"Test is synchronously stopping "
+ targetVm
+ " with "
+ ClientVmMgr.toStopModeString(stopMode),
stopMode,
ClientVmMgr.ON_DEMAND,
targetVm);
}
VMotionUtil.doVMotion(targetVm);
if (start) {
ClientVmMgr.start("Test is synchronously starting " + targetVm, targetVm);
}
} catch (hydra.ClientVmNotFoundException e) {
Log.getLogWriter()
.info("Caught in thread " + getName() + ":" + TestHelper.getStackTrace(e));
StopStartBB.getBB().getSharedMap().put(StopStartBB.errKey, TestHelper.getStackTrace(e));
}
} catch (Throwable e) {
Log.getLogWriter().info("Caught in thread " + getName() + ":" + TestHelper.getStackTrace(e));
StopStartBB.getBB().getSharedMap().put(StopStartBB.errKey, TestHelper.getStackTrace(e));
}
Log.getLogWriter().info(getName() + " terminating");
}
/**
* Get the client name for the given vmID, by looking in the StopStartBB blackboard shared map.
* The map must have been created by calling StopStart_initTask. If this method was never invoked,
* then null is returned as the clientName cannot be deteremined.
*
* @param vmID The vmID to get the client name for.
* @returns The client name for vmID, or null if it cannot be determined.
*/
public static String getClientName(Integer vmID) {
Iterator it = StopStartBB.getBB().getSharedMap().getMap().values().iterator();
while (it.hasNext()) {
Object anObj = it.next();
if (anObj instanceof ClientVmInfo) {
ClientVmInfo info = (ClientVmInfo) (anObj);
if (info.getVmid().equals(vmID)) { // its a match
return info.getClientName();
}
}
}
return null;
}
/** Return a List of all ClientVmInfo instances in the StopStartVMs blackboard. */
public static List getAllVMs() {
List aList = new ArrayList();
Map aMap = StopStartBB.getBB().getSharedMap().getMap();
Iterator it = aMap.keySet().iterator();
while (it.hasNext()) {
Object key = it.next();
Object value = aMap.get(key);
if (value instanceof ClientVmInfo) {
aList.add(value);
}
}
return aList;
}
/**
* Concurrently stops a List of Vms, then restart them. Does not wait for the restart to complete
* before returning. This allows the caller to carry out operations concurrent with the stop and
* start activity, then wait for the restart to complete using {@link
* #joinStopStartThreads(List,List)}.
*
* <p>Side effects: For any VM which is performing a NICE_KILL because of this call, it is marked
* as such in the StopStartBB (blackboard). The blackboard will contain a key prefixed with
* util.StopStartVMs.NiceKillInProgress and appended to with its VmId. The value of this key is
* the true.
*
* <p>This blackboard entry is for the convenience of the caller, and it is up to the caller to
* look for it or not. When a VM undergoes a NICE_KILL, the VM closes the cache and disconnects
* from the distributed system. Any threads in that VM doing work can get exceptions during the
* close and/or disconnect step. This blackboard entry is useful for those threads to allow
* exceptions when they know an exception is possible.
*
* @param targetVmList A list of hydra.ClientVmInfos to stop then restart.
* @param stopModeList A parallel list of stop modes (Strings).
* @return threadList The threads created to stop the VMs.
*/
public static List stopStartAsync(List targetVmList, List stopModeList) {
if (targetVmList.size() != stopModeList.size()) {
throw new TestException(
"Expected targetVmList "
+ targetVmList
+ " and stopModeList "
+ stopModeList
+ " to be parallel lists of the same size, but they have different sizes");
}
Log.getLogWriter()
.info(
"In stopStartVMs, vms to stop: "
+ targetVmList
+ ", corresponding stop modes: "
+ stopModeList);
// mark in the blackboard any vms that will undergo a NICE_KILL
for (int i = 0; i < stopModeList.size(); i++) {
String stopMode = (String) (stopModeList.get(i));
if (ClientVmMgr.toStopMode(stopMode) == ClientVmMgr.NICE_KILL) {
// write nice kills to the blackboard so the killed vm may, if it chooses, handle any
// exceptions that are underway.
StopStartBB.getBB()
.getSharedMap()
.put(
NiceKillInProgress + ((ClientVmInfo) targetVmList.get(i)).getVmid(),
new Boolean(true));
}
}
// stop all vms in targetVms list
List threadList = new ArrayList();
for (int i = 0; i < targetVmList.size(); i++) {
ClientVmInfo targetVm = (ClientVmInfo) (targetVmList.get(i));
int stopMode = ClientVmMgr.toStopMode((String) (stopModeList.get(i)));
Thread aThread = new StopStartVMs(targetVm, stopMode);
aThread = new HydraSubthread(aThread);
threadList.add(aThread);
aThread.start();
}
return threadList;
}
/**
* Return a List of all other vms except for one along with a parallel stop mode list, and also
* return the one vm not in the list.
*
* @return [0] (List<ClientVmInfo>)A List of ClientVmInfo instances; this includes all vms except
* the current vm and except for one other vm. [1] (List<String>) A parallel List to [0]
* containing stop modes. [2] (ClientVMInfo) The ClientVmInfo instance of the one vm excluded
* from [0] (other than this vm) [3] (String) The stop mode for [2] [4] (List<ClientVmInfo>) A
* List of ClientVmInfo instances for all vms except this one. [5] (List<String>) A parallel
* List to [4] of stop modes.
*/
public static Object[] getOtherVMsDivided(String[] excludedClientNames) {
Vector<Integer> otherVmIDs = ClientVmMgr.getOtherClientVmids();
List<ClientVmInfo> allOtherVMs = new ArrayList();
List<String> stopModeList = new ArrayList();
for (int i = 0; i < otherVmIDs.size(); i++) {
ClientVmInfo info = new ClientVmInfo(otherVmIDs.get(i));
ClientVmInfo infoFromBB =
(ClientVmInfo) StopStartBB.getBB().getSharedMap().get(VmInfoKey + otherVmIDs.get(i));
if (infoFromBB != null) {
info = infoFromBB;
}
String clientName = info.getClientName();
if (clientName == null) {
allOtherVMs.add(info);
stopModeList.add(TestConfig.tab().stringAt(StopStartPrms.stopModes));
} else {
boolean inExcludedNames = false;
for (String excludeName : excludedClientNames) {
if (clientName.indexOf(excludeName) >= 0) inExcludedNames = true;
break;
}
if (!inExcludedNames) {
allOtherVMs.add(info);
stopModeList.add(TestConfig.tab().stringAt(StopStartPrms.stopModes));
}
}
}
List<ClientVmInfo> allOtherVMsExceptOne = allOtherVMs.subList(0, allOtherVMs.size() - 1);
List<String> stopModesExceptOne = stopModeList.subList(0, stopModeList.size() - 1);
ClientVmInfo remainingVM = allOtherVMs.get(allOtherVMs.size() - 1);
String remainingStopMode = stopModeList.get(stopModeList.size() - 1);
return new Object[] {
allOtherVMsExceptOne,
stopModesExceptOne,
remainingVM,
remainingStopMode,
allOtherVMs,
stopModeList
};
}
/**
* Invokes AdminDistributedSystem.shutDownAllMembers() which disconnects all members but leaves
* the vms up (because hydra threads remain) including the possibility of this vm being
* disconnected, then this actually stops those vms (except this vm if it was targeted in the
* shutDownAllMembers...this vm will remain up but disconnect). Stopped vms are stopped with
* ON_DEMAND restart. This returns when the vms disconnected by shutDownAllMembers() (other than
* this one) are all stopped .
*
* @param adminDS The admin distributed system instance to use to call shutdownAllMembers.
* @param stopModes The stop modes to choose from.
* @return An Array [0] List of {@link ClientVmInfo} instances describing the VMs that were
* stopped. [1] Set, the return from shutdownAllMembers()
* @throws AdminException if the shutDownAllMembers call throws this exception.
*/
public static Object[] shutDownAllMembers(
AdminDistributedSystem adminDS, List<String> stopModes) {
if (adminDS == null) {
throw new HydraRuntimeException("AdminDistributedSystem cannot be null");
}
// Invoke shutDownAllMembers
Log.getLogWriter().info("AdminDS " + adminDS + " is shutting down all members...");
Set<DistributedMember> memberSet;
try {
long startTime = System.currentTimeMillis();
memberSet = adminDS.shutDownAllMembers();
long duration = System.currentTimeMillis() - startTime;
Log.getLogWriter()
.info(
"AdminDS "
+ adminDS
+ " shut down (disconnected) the following members "
+ "(vms remain up): "
+ memberSet
+ "; shutDownAll duration "
+ duration
+ "ms");
} catch (AdminException e1) {
throw new TestException(TestHelper.getStackTrace(e1));
}
// Now actually stop the vms.
// First get the ClientVmInfos for the members that shutDownAllMembers
// disconnected.
List<ClientVmInfo> allClientInfos = new ArrayList(); // all members that were shutdown
List<ClientVmInfo> allOtherClientInfos =
new ArrayList(); // all members that were shutdown except this member
ClientVmInfo thisClientInfo =
null; // this member, or will remain null if this member was not shutdown
List<String> stopModesToUse = new ArrayList();
for (DistributedMember aMember : memberSet) {
Integer vmId = null;
try {
vmId =
new Integer(RemoteTestModule.Master.getVmid(aMember.getHost(), aMember.getProcessId()));
} catch (java.rmi.RemoteException e) {
throw new HydraRuntimeException("Unable to get vmID for " + aMember + ": " + e);
}
ClientVmInfo infoFromBB =
(ClientVmInfo) StopStartBB.getBB().getSharedMap().get("StopStartVMInfo_for_vmid_" + vmId);
String clientName = null;
if (infoFromBB != null) {
clientName = infoFromBB.getClientName();
}
ClientVmInfo info = new ClientVmInfo(vmId, clientName, null);
allClientInfos.add(info);
if (vmId == RemoteTestModule.getMyVmid()) {
// shutdownAll disconnected this vm
thisClientInfo = info;
} else { // aMember is not the current vm
allOtherClientInfos.add(info);
}
stopModesToUse.add(
stopModes.get(TestConfig.tab().getRandGen().nextInt(0, stopModes.size() - 1)));
}
// now actually stop the vms; if this vm is included, do it last
Object[] returnArr = new Object[2];
if (thisClientInfo == null) { // shutDownAllMembers did not disconnect this vm
// we can just stop all of them now and this vm lives on
StopStartVMs.stopVMs(allClientInfos, stopModesToUse); // restart is ON_DEMAND
returnArr[0] = allClientInfos;
} else { // this vm was disconnected by shutDownAllMembers
// first shutdown all other members except this one
StopStartVMs.stopVMs(allOtherClientInfos, stopModesToUse.subList(0, stopModesToUse.size()));
returnArr[0] = allOtherClientInfos;
}
returnArr[1] = memberSet;
return returnArr;
}
/**
* Concurrently stop and/or start a List of Vms. Wait for the operations to complete before
* returning.
*
* <p>Side effects: For any VM which is performing a NICE_KILL because of this call, it is marked
* as such in the StopStartBB (blackboard). The blackboard will contain a key prefixed with
* util.StopStartVMs.NiceKillInProgress and appended to with its VmId. The value of this key is
* the true.
*
* <p>This blackboard entry is for the convenience of the caller, and it is up to the caller to
* look for it or not. When a VM undergoes a NICE_KILL, the VM closes the cache and disconnects
* from the distributed system. Any threads in that VM doing work can get exceptions during the
* close and/or disconnect step. This blackboard entry is useful for those threads to allow
* exceptions when they know an exception is possible.
*
* @param targetVmList A list of hydra.ClientVmInfos to stop and/or start.
* @param stopModeList A parallel list of stop modes (Strings), if doStop is true.
* @param doStop True if we want to stop the targetVmList, false if not.
* @param doStart True if we want to start the targetVmList, false if not.
*/
private static void stopStartVMs(
List targetVmList, List stopModeList, boolean doStop, boolean doStart) {
Log.getLogWriter()
.info(
"In stopStartVMs, vms to "
+ getAction(doStop, doStart)
+ ": "
+ targetVmList
+ ", corresponding stop modes: "
+ stopModeList);
if (doStop) {
// mark in the blackboard any vms that will undergo a NICE_KILL
for (int i = 0; i < stopModeList.size(); i++) {
String stopMode = (String) (stopModeList.get(i));
if (ClientVmMgr.toStopMode(stopMode) == ClientVmMgr.NICE_KILL) {
// write nice kills to the blackboard so the killed vm may, if it chooses, handle any
// exceptions that are underway.
StopStartBB.getBB()
.getSharedMap()
.put(
NiceKillInProgress + ((ClientVmInfo) targetVmList.get(i)).getVmid(),
new Boolean(true));
}
}
}
// process the vms in targetVms list
List threadList = new ArrayList();
for (int i = 0; i < targetVmList.size(); i++) {
ClientVmInfo targetVm = (ClientVmInfo) (targetVmList.get(i));
int stopMode = 0;
if (doStop) {
stopMode = ClientVmMgr.toStopMode((String) (stopModeList.get(i)));
}
StringBuffer aStr = new StringBuffer();
Thread aThread = new StopStartVMs(targetVm, stopMode, doStop, doStart);
aThread = new HydraSubthread(aThread);
threadList.add(aThread);
aThread.start();
}
// wait for all threads to complete their actions
for (int i = 0; i < threadList.size(); i++) {
Thread aThread = (Thread) (threadList.get(i));
try {
aThread.join();
} catch (InterruptedException e) {
throw new TestException(TestHelper.getStackTrace(e));
}
}
String err = (String) (StopStartBB.getBB().getSharedMap().get(StopStartBB.errKey));
if (err != null) {
throw new TestException(err);
}
// reset the blackboard entries
if (doStop) {
for (int i = 0; i < targetVmList.size(); i++) {
ClientVmInfo targetVm = (ClientVmInfo) (targetVmList.get(i));
StopStartBB.getBB()
.getSharedMap()
.put(StopStartVMs.NiceKillInProgress + targetVm.getVmid(), new Boolean(false));
}
}
Log.getLogWriter()
.info("In stopStartVMs, done with " + getAction(doStop, doStart) + ": " + targetVmList);
}
/**
* An init task to write ClientVmInfo instances to the blackboard, for use later in choosing vms
* to stop/start. This is only necessary if getOtherVMs(numToTarget, clientMatchStr) is used.
*/
public static void StopStart_initTask() {
int myVMid = RemoteTestModule.getMyVmid();
ClientVmInfo myInfo =
new ClientVmInfo(new Integer(myVMid), RemoteTestModule.getMyClientName(), null);
StopStartBB.getBB().getSharedMap().put(VmInfoKey + myVMid, myInfo);
}
/** Returns true if a NICE_KILL is in progress in this VM, false otherwise */
public static boolean niceKillInProgress() {
Object niceKill =
StopStartBB.getBB().getSharedMap().get(NiceKillInProgress + RemoteTestModule.getMyVmid());
boolean niceKillInProgress = (niceKill != null) && (((Boolean) niceKill).booleanValue());
return niceKillInProgress;
}
