public void terminateInstance(@NotNull final VmwareCloudInstance instance) {
LOG.info("Stopping instance " + instance.getName());
instance.setStatus(InstanceStatus.SCHEDULED_TO_STOP);
myAsyncTaskExecutor.executeAsync(
new VmwareTaskWrapper(
new Callable<Task>() {
public Task call() throws Exception {
return myApiConnector.stopInstance(instance);
}
},
"Stop " + instance.getName()),
new ImageStatusTaskWrapper(instance) {
@Override
public void onComplete() {
instance.setStatus(InstanceStatus.STOPPED);
if (myImageDetails
.getBehaviour()
.isDeleteAfterStop()) { // we only destroy proper instances.
deleteInstance(instance);
}
}
});
}
public boolean canStartNewInstance() {
if (getErrorInfo() != null) {
LOG.debug("Can't start new instance, if image is erroneous");
return false;
}
if (myImageDetails.getBehaviour().isUseOriginal()) {
final VmwareCloudInstance myInstance = myInstances.get(myImageDetails.getSourceName());
if (myInstance == null) {
return false;
}
return myInstance.getStatus() == InstanceStatus.STOPPED;
}
final boolean countStoppedVmsInLimit =
TeamCityProperties.getBooleanOrTrue(VmwareConstants.CONSIDER_STOPPED_VMS_LIMIT)
&& myImageDetails.getBehaviour().isDeleteAfterStop();
final List<String> consideredInstances = new ArrayList<String>();
for (Map.Entry<String, VmwareCloudInstance> entry : myInstances.entrySet()) {
if (entry.getValue().getStatus() != InstanceStatus.STOPPED || countStoppedVmsInLimit)
consideredInstances.add(entry.getKey());
}
final boolean canStartMore = consideredInstances.size() < myImageDetails.getMaxInstances();
LOG.debug(
String.format(
"Instances count: %d %s, can start more: %s",
consideredInstances.size(),
Arrays.toString(consideredInstances.toArray()),
String.valueOf(canStartMore)));
return canStartMore;
}
@Override
public void detectNewInstances(final Map<String, AbstractInstance> realInstances) {
for (String instanceName : realInstances.keySet()) {
if (myInstances.get(instanceName) == null) {
final VmwareInstance realInstance = (VmwareInstance) realInstances.get(instanceName);
final VmwareCloudInstance newInstance =
new VmwareCloudInstance(this, instanceName, realInstance.getSnapshotName());
newInstance.setStatus(realInstance.getInstanceStatus());
myInstances.put(instanceName, newInstance);
}
}
}
private void deleteInstance(@NotNull final VmwareCloudInstance instance) {
if (instance.getErrorInfo() == null) {
LOG.info("Will delete instance " + instance.getName());
final VmwareInstance vmInstance;
try {
vmInstance = myApiConnector.getInstanceDetails(instance.getName());
myAsyncTaskExecutor.executeAsync(
vmInstance.deleteInstance(),
new ImageStatusTaskWrapper(instance) {
@Override
public void onSuccess() {
removeInstance(instance.getName());
}
});
} catch (VmwareCheckedCloudException e) {
LOG.warn("An exception during deleting instance " + instance.getName(), e);
instance.updateErrors(TypedCloudErrorInfo.fromException(e));
}
} else {
LOG.warn(
String.format(
"Won't delete instance %s with error: %s (%s)",
instance.getName(),
instance.getErrorInfo().getMessage(),
instance.getErrorInfo().getDetailedMessage()));
}
}
private synchronized void startVM(
@NotNull final VmwareCloudInstance instance,
@NotNull final CloudInstanceUserData cloudInstanceUserData) {
instance.setStatus(InstanceStatus.STARTING);
myAsyncTaskExecutor.executeAsync(
new VmwareTaskWrapper(
new Callable<Task>() {
public Task call() throws Exception {
return myApiConnector.startInstance(
instance, instance.getName(), cloudInstanceUserData);
}
},
"Start instance " + instance.getName()),
new ImageStatusTaskWrapper(instance) {
@Override
public void onSuccess() {
reconfigureVmTask(instance, cloudInstanceUserData);
}
});
}
public void check_can_start_new_instance_limits() throws RemoteException, InterruptedException {
final CloudInstanceUserData data =
new CloudInstanceUserData(
"aaa",
"bbbb",
"localhost",
10000l,
"profileDescr",
Collections.<String, String>emptyMap());
assertTrue(myImage.canStartNewInstance());
myImage.startNewInstance(data);
assertTrue(myImage.canStartNewInstance());
myImage.startNewInstance(data);
assertTrue(myImage.canStartNewInstance());
myImage.startNewInstance(data);
assertTrue(myImage.canStartNewInstance());
myImage.startNewInstance(data);
assertTrue(myImage.canStartNewInstance());
final VmwareCloudInstance instance2Stop = myImage.startNewInstance(data);
assertFalse(myImage.canStartNewInstance());
new WaitFor(5 * 1000) {
@Override
protected boolean condition() {
return instance2Stop.getStatus() == InstanceStatus.RUNNING;
}
};
final FakeVirtualMachine vm2Stop =
FakeModel.instance().getVirtualMachine(instance2Stop.getName());
final String result = vm2Stop.powerOffVM_Task().waitForTask();
assertEquals(Task.SUCCESS, result);
instance2Stop.setStatus(InstanceStatus.STOPPED);
assertTrue(myImage.canStartNewInstance());
System.setProperty(VmwareConstants.CONSIDER_STOPPED_VMS_LIMIT, "true");
assertFalse(myImage.canStartNewInstance());
System.getProperties().remove(VmwareConstants.CONSIDER_STOPPED_VMS_LIMIT);
assertTrue(myImage.canStartNewInstance());
}
@Override
public synchronized VmwareCloudInstance startNewInstance(
@NotNull final CloudInstanceUserData cloudInstanceUserData) throws QuotaException {
try {
final VmwareCloudInstance instance = getOrCreateInstance();
if (instance == null) {
return null;
}
boolean willClone = !myApiConnector.checkVirtualMachineExists(instance.getName());
LOG.info("Will clone for " + instance.getName() + ": " + willClone);
if (willClone && myImageDetails.getMaxInstances() <= myInstances.size()) {
throw new QuotaException(
String.format(
"Cannot clone '%s' into '%s' - limit exceeded",
myImageDetails.getSourceName(), instance.getName()));
}
instance.setStatus(InstanceStatus.SCHEDULED_TO_START);
if (!myInstances.containsKey(instance.getName())) {
addInstance(instance);
}
if (willClone) {
myAsyncTaskExecutor.executeAsync(
new VmwareTaskWrapper(
new Callable<Task>() {
public Task call() throws Exception {
return myApiConnector.cloneAndStartVm(
instance, myImageDetails.getResourcePoolId(), myImageDetails.getFolderId());
}
},
"Clone and start instance " + instance.getName()),
new ImageStatusTaskWrapper(instance) {
@Override
public void onSuccess() {
reconfigureVmTask(instance, cloudInstanceUserData);
}
@Override
public void onError(final Throwable th) {
super.onError(th);
removeInstance(instance.getName());
}
});
} else {
startVM(instance, cloudInstanceUserData);
}
return instance;
} catch (QuotaException e) {
throw e;
} catch (VmwareCheckedCloudException e) {
throw new CloudException("Unable to start new instance: " + e.toString());
}
}
public VmwareCloudImage(
@NotNull final VMWareApiConnector apiConnector,
@NotNull final VmwareCloudImageDetails imageDetails,
@NotNull final CloudAsyncTaskExecutor asyncTaskExecutor,
@NotNull final File idxStorage) {
super(imageDetails.getNickname(), imageDetails.getNickname());
myImageDetails = imageDetails;
myApiConnector = apiConnector;
myAsyncTaskExecutor = asyncTaskExecutor;
myInstances.clear();
myIdxFile = new File(idxStorage, imageDetails.getNickname() + ".idx");
if (!myIdxFile.exists()) {
try {
FileUtil.writeFileAndReportErrors(myIdxFile, "1");
} catch (IOException e) {
LOG.warn(
String.format(
"Unable to write idx file '%s': %s", myIdxFile.getAbsolutePath(), e.toString()));
}
}
Map<String, VmwareInstance> realInstances = null;
try {
realInstances = myApiConnector.listImageInstances(this);
} catch (VmwareCheckedCloudException e) {
updateErrors(TypedCloudErrorInfo.fromException(e));
return;
}
if (imageDetails.getBehaviour().isUseOriginal()) {
final VmwareCloudInstance imageInstance =
new VmwareCloudInstance(
this, imageDetails.getSourceName(), VmwareConstants.CURRENT_STATE);
myInstances.put(myImageDetails.getSourceName(), imageInstance);
final VmwareInstance vmwareInstance = realInstances.get(imageDetails.getSourceName());
if (vmwareInstance != null) {
imageInstance.setStatus(vmwareInstance.getInstanceStatus());
} else {
imageInstance.setStatus(InstanceStatus.UNKNOWN);
imageInstance.updateErrors(
new TypedCloudErrorInfo("NoVM", "VM doesn't exist: " + imageDetails.getSourceName()));
}
} else {
for (String instanceName : realInstances.keySet()) {
final VmwareInstance instance = realInstances.get(instanceName);
final String snapshotName = instance.getSnapshotName();
VmwareCloudInstance cloudInstance =
new VmwareCloudInstance(this, instanceName, snapshotName);
cloudInstance.setStatus(instance.getInstanceStatus());
myInstances.put(instanceName, cloudInstance);
}
}
}
private synchronized void reconfigureVmTask(
@NotNull final VmwareCloudInstance instance,
@NotNull final CloudInstanceUserData cloudInstanceUserData) {
myAsyncTaskExecutor.executeAsync(
new VmwareTaskWrapper(
new Callable<Task>() {
public Task call() throws Exception {
return myApiConnector.reconfigureInstance(
instance, instance.getName(), cloudInstanceUserData);
}
},
"Reconfigure " + instance.getName()),
new ImageStatusTaskWrapper(instance) {
@Override
public void onSuccess() {
instance.setStatus(InstanceStatus.RUNNING);
instance.updateErrors();
LOG.info("Instance started successfully");
}
});
}
@Override
public void onError(final Throwable th) {
myInstance.setStatus(InstanceStatus.ERROR);
if (th != null) {
myInstance.updateErrors(TypedCloudErrorInfo.fromException(th));
LOG.warn(
"An error occurred: "
+ th.getLocalizedMessage()
+ " during processing "
+ myInstance.getName());
} else {
myInstance.updateErrors(
new TypedCloudErrorInfo(
"Unknown error during processing instance " + myInstance.getName()));
LOG.warn("Unknown error during processing " + myInstance.getName());
}
}
public void addInstance(@NotNull final VmwareCloudInstance instance) {
LOG.info(
String.format(
"Image %s, put instance %s", myImageDetails.getSourceName(), instance.getName()));
myInstances.put(instance.getName(), instance);
}
@NotNull
protected synchronized VmwareCloudInstance getOrCreateInstance()
throws VmwareCheckedCloudException {
if (!canStartNewInstance()) {
throw new QuotaException("Unable to start more instances of image " + getName());
}
if (myImageDetails.getBehaviour().isUseOriginal()) {
LOG.info("Won't create a new instance - using original");
return myInstances.get(myImageDetails.getSourceName());
}
final String latestSnapshotName =
myApiConnector.getLatestSnapshot(
myImageDetails.getSourceName(), myImageDetails.getSnapshotName());
if (!myImageDetails.useCurrentVersion() && latestSnapshotName == null) {
updateErrors(new TypedCloudErrorInfo("No such snapshot: " + getSnapshotName()));
throw new VmwareCheckedCloudException(
"Unable to find snapshot: " + myImageDetails.getSnapshotName());
}
if (!myImageDetails.getBehaviour().isDeleteAfterStop()) {
// on demand clone
final Map<String, VmwareInstance> vmClones = myApiConnector.listImageInstances(this);
// start an existsing one.
final VmwareInstance imageVm =
myApiConnector.getInstanceDetails(myImageDetails.getSourceName());
for (VmwareInstance vmInstance : vmClones.values()) {
if (vmInstance.getInstanceStatus() == InstanceStatus.STOPPED) {
final String vmName = vmInstance.getName();
final VmwareCloudInstance instance = myInstances.get(vmName);
if (instance == null) {
LOG.warn("Unable to find instance " + vmName + " in myInstances.");
continue;
}
// checking if this instance is already starting.
if (instance.getStatus() != InstanceStatus.STOPPED) continue;
if (myImageDetails.useCurrentVersion()) {
if (imageVm.getChangeVersion() == null
|| !imageVm.getChangeVersion().equals(vmInstance.getChangeVersion())) {
LOG.info(
String.format(
"Change version for %s is outdated: '%s' vs '%s'",
vmName, vmInstance.getChangeVersion(), imageVm.getChangeVersion()));
deleteInstance(instance);
continue;
}
} else {
final String snapshotName = vmInstance.getSnapshotName();
if (latestSnapshotName != null && !latestSnapshotName.equals(snapshotName)) {
LOG.info(
String.format(
"VM %s Snapshot is not the latest one: '%s' vs '%s'",
vmName, snapshotName, latestSnapshotName));
deleteInstance(instance);
continue;
}
}
LOG.info("Will use existing VM with name " + vmName);
return instance;
}
}
}
// wasn't able to find an existing candidate, so will clone into a new VM
final String newVmName = generateNewVmName();
LOG.info("Will create a new VM with name " + newVmName);
return new VmwareCloudInstance(this, newVmName, latestSnapshotName);
}