// @Test
public void testCopyBaseImage() {
DataStore primaryStore = createPrimaryDataStore();
primaryStoreId = primaryStore.getId();
primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId);
VolumeVO volume = createVolume(image.getId(), primaryStore.getId());
VolumeInfo volInfo = this.volFactory.getVolume(volume.getId());
AsyncCallFuture<VolumeApiResult> future =
this.volumeService.createVolumeFromTemplateAsync(
volInfo,
this.primaryStoreId,
this.templateFactory.getTemplate(this.image.getId(), DataStoreRole.Image));
try {
VolumeApiResult result = future.get();
AssertJUnit.assertTrue(result.isSuccess());
VolumeInfo newVol = result.getVolume();
this.volumeService.destroyVolume(newVol.getId());
VolumeInfo vol = this.volFactory.getVolume(volume.getId());
this.volumeService.expungeVolumeAsync(vol);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ConcurrentOperationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// @Test(priority=3)
public void createAndDeleteDataDisk() {
DataStore primaryStore = this.primaryStore;
VolumeVO volume = createVolume(null, primaryStore.getId());
VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore);
AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeAsync(vol, primaryStore);
try {
future.get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// delete the volume
vol = volumeFactory.getVolume(volume.getId(), primaryStore);
future = volumeService.expungeVolumeAsync(vol);
try {
future.get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@DB
public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) {
if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED)
|| (dnldStatus == Status.ABANDONED)) {
VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId());
synchronized (_listenerVolumeMap) {
_listenerVolumeMap.remove(volumeHost);
}
}
VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId());
Transaction txn = Transaction.currentTxn();
txn.start();
if (dnldStatus == Status.DOWNLOADED) {
// Create usage event
long size = -1;
if (volumeHost != null) {
size = volumeHost.getSize();
volume.setSize(size);
this._volumeDao.update(volume.getId(), volume);
} else {
s_logger.warn("Failed to get size for volume" + volume.getName());
}
String eventType = EventTypes.EVENT_VOLUME_UPLOAD;
if (volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
UsageEventUtils.publishUsageEvent(
eventType,
volume.getAccountId(),
host.getDataCenterId(),
volume.getId(),
volume.getName(),
null,
0l,
size,
volume.getClass().getName(),
volume.getUuid());
}
} else if (dnldStatus == Status.DOWNLOAD_ERROR
|| dnldStatus == Status.ABANDONED
|| dnldStatus == Status.UNKNOWN) {
// Decrement the volume and secondary storage space count
_resourceLimitMgr.decrementResourceCount(
volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume);
_resourceLimitMgr.recalculateResourceCount(
volume.getAccountId(),
volume.getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
}
txn.commit();
}
@Test
public void testCreateTemplateFromVolume() {
DataStore primaryStore = createPrimaryDataStore();
primaryStoreId = primaryStore.getId();
primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId);
VolumeVO volume = createVolume(null, primaryStore.getId());
VolumeInfo volInfo = this.volFactory.getVolume(volume.getId());
AsyncCallFuture<VolumeApiResult> future =
this.volumeService.createVolumeAsync(volInfo, primaryStore);
try {
VolumeApiResult result = future.get();
AssertJUnit.assertTrue(result.isSuccess());
volInfo = result.getVolume();
VMTemplateVO templateVO = createTemplateInDb();
TemplateInfo tmpl = this.templateFactory.getTemplate(templateVO.getId(), DataStoreRole.Image);
DataStore imageStore = this.dataStoreMgr.getImageStore(this.dcId);
this.imageService.createTemplateFromVolumeAsync(volInfo, tmpl, imageStore);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Test
public void testDeleteDisk() {
DataStore primaryStore = createPrimaryDataStore();
primaryStoreId = primaryStore.getId();
primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId);
VolumeVO volume = createVolume(null, primaryStore.getId());
VolumeInfo volInfo = this.volFactory.getVolume(volume.getId());
AsyncCallFuture<VolumeApiResult> future =
this.volumeService.createVolumeAsync(volInfo, primaryStore);
try {
VolumeApiResult result = future.get();
VolumeInfo vol = result.getVolume();
this.volumeService.destroyVolume(volInfo.getId());
volInfo = this.volFactory.getVolume(vol.getId());
this.volumeService.expungeVolumeAsync(volInfo);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ConcurrentOperationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public void createAsync(
DataStore dataStore,
DataObject dataObject,
AsyncCompletionCallback<CreateCmdResult> callback) {
String iqn = null;
String errMsg = null;
if (dataObject.getType() == DataObjectType.VOLUME) {
VolumeInfo volumeInfo = (VolumeInfo) dataObject;
AccountVO account = _accountDao.findById(volumeInfo.getAccountId());
String sfAccountName = getSfAccountName(account.getUuid(), account.getAccountId());
long storagePoolId = dataStore.getId();
SolidFireConnection sfConnection = getSolidFireConnection(storagePoolId);
if (!sfAccountExists(sfAccountName, sfConnection)) {
SolidFireUtil.SolidFireAccount sfAccount =
createSolidFireAccount(sfAccountName, sfConnection);
updateCsDbWithAccountInfo(account.getId(), sfAccount);
}
SolidFireUtil.SolidFireVolume sfVolume = createSolidFireVolume(volumeInfo, sfConnection);
iqn = sfVolume.getIqn();
VolumeVO volume = this._volumeDao.findById(volumeInfo.getId());
volume.set_iScsiName(iqn);
volume.setFolder(String.valueOf(sfVolume.getId()));
volume.setPoolType(StoragePoolType.IscsiLUN);
volume.setPoolId(storagePoolId);
_volumeDao.update(volume.getId(), volume);
StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId());
long capacityBytes = storagePool.getCapacityBytes();
long usedBytes = storagePool.getUsedBytes();
usedBytes += volumeInfo.getSize();
storagePool.setUsedBytes(usedBytes > capacityBytes ? capacityBytes : usedBytes);
_storagePoolDao.update(storagePoolId, storagePool);
} else {
errMsg = "Invalid DataObjectType (" + dataObject.getType() + ") passed to createAsync";
}
// path = iqn
// size is pulled from DataObject instance, if errMsg is null
CreateCmdResult result = new CreateCmdResult(iqn, new Answer(null, errMsg == null, errMsg));
result.setResult(errMsg);
callback.complete(result);
}
@Test
public void testCreateDataDisk() {
DataStore primaryStore = createPrimaryDataStore();
primaryStoreId = primaryStore.getId();
primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId);
VolumeVO volume = createVolume(null, primaryStore.getId());
VolumeInfo volInfo = this.volFactory.getVolume(volume.getId());
this.volumeService.createVolumeAsync(volInfo, primaryStore);
}
@Override
public void run() {
try {
List<VolumeVO> volumes = _volsDao.listAll();
Map<Long, List<VolumeCommand>> commandsByPool = new HashMap<Long, List<VolumeCommand>>();
for (VolumeVO volume : volumes) {
List<VolumeCommand> commands = commandsByPool.get(volume.getPoolId());
if (commands == null) {
commands = new ArrayList<VolumeCommand>();
commandsByPool.put(volume.getPoolId(), commands);
}
VolumeCommand vCommand = new VolumeCommand();
vCommand.volumeId = volume.getId();
vCommand.command = new GetFileStatsCommand(volume);
commands.add(vCommand);
}
ConcurrentHashMap<Long, VolumeStats> volumeStats =
new ConcurrentHashMap<Long, VolumeStats>();
for (Iterator<Long> iter = commandsByPool.keySet().iterator(); iter.hasNext(); ) {
Long poolId = iter.next();
List<VolumeCommand> commandsList = commandsByPool.get(poolId);
long[] volumeIdArray = new long[commandsList.size()];
Commands commands = new Commands(OnError.Continue);
for (int i = 0; i < commandsList.size(); i++) {
VolumeCommand vCommand = commandsList.get(i);
volumeIdArray[i] = vCommand.volumeId;
commands.addCommand(vCommand.command);
}
List<StoragePoolHostVO> poolhosts = _storagePoolHostDao.listByPoolId(poolId);
for (StoragePoolHostVO poolhost : poolhosts) {
Answer[] answers = _agentMgr.send(poolhost.getHostId(), commands);
if (answers != null) {
long totalBytes = 0L;
for (int i = 0; i < answers.length; i++) {
if (answers[i].getResult()) {
VolumeStats vStats = (VolumeStats) answers[i];
volumeStats.put(volumeIdArray[i], vStats);
totalBytes += vStats.getBytesUsed();
}
}
break;
}
}
}
// We replace the existing volumeStats so that it does not grow with no bounds
_volumeStats = volumeStats;
} catch (AgentUnavailableException e) {
s_logger.debug(e.getMessage());
} catch (Throwable t) {
s_logger.error("Error trying to retrieve volume stats", t);
}
}
@Test(priority = 2)
public void createVolumeFromTemplate() {
DataStore primaryStore = this.primaryStore;
TemplateInfo te = createTemplate();
VolumeVO volume = createVolume(te.getId(), primaryStore.getId());
VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore);
// ve.createVolumeFromTemplate(primaryStore.getId(), new VHD(), te);
AsyncCallFuture<VolumeApiResult> future =
volumeService.createVolumeFromTemplateAsync(vol, primaryStore.getId(), te);
try {
future.get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
protected void createDb() {
DataCenterVO dc =
new DataCenterVO(
UUID.randomUUID().toString(),
"test",
"8.8.8.8",
null,
"10.0.0.1",
null,
"10.0.0.1/24",
null,
null,
NetworkType.Basic,
null,
null,
true,
true,
null,
null);
dc = dcDao.persist(dc);
dcId = dc.getId();
HostPodVO pod =
new HostPodVO(UUID.randomUUID().toString(), dc.getId(), "255.255.255.255", "", 8, "test");
pod = podDao.persist(pod);
podId = pod.getId();
ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
cluster.setHypervisorType(HypervisorType.XenServer.toString());
cluster.setClusterType(ClusterType.CloudManaged);
cluster.setManagedState(ManagedState.Managed);
cluster = clusterDao.persist(cluster);
clusterId = cluster.getId();
DataStoreProvider provider =
providerMgr.getDataStoreProvider("ancient primary data store provider");
storage = new StoragePoolVO();
storage.setDataCenterId(dcId);
storage.setPodId(podId);
storage.setPoolType(StoragePoolType.NetworkFilesystem);
storage.setClusterId(clusterId);
storage.setStatus(StoragePoolStatus.Up);
storage.setScope(ScopeType.CLUSTER);
storage.setAvailableBytes(1000);
storage.setCapacityBytes(20000);
storage.setHostAddress(UUID.randomUUID().toString());
storage.setPath(UUID.randomUUID().toString());
storage.setStorageProviderName(provider.getName());
storage = storagePoolDao.persist(storage);
storagePoolId = storage.getId();
storageMgr.createCapacityEntry(storage.getId());
diskOffering = new DiskOfferingVO();
diskOffering.setDiskSize(500);
diskOffering.setName("test-disk");
diskOffering.setSystemUse(false);
diskOffering.setUseLocalStorage(false);
diskOffering.setCustomized(false);
diskOffering.setRecreatable(false);
diskOffering = diskOfferingDao.persist(diskOffering);
diskOfferingId = diskOffering.getId();
volume =
new VolumeVO(
Volume.Type.ROOT,
"volume",
dcId,
1,
1,
diskOffering.getId(),
diskOffering.getDiskSize());
volume = volumeDao.persist(volume);
volumeId = volume.getId();
}
@Override
public void handleVolumeSync(HostVO ssHost) {
if (ssHost == null) {
s_logger.warn("Huh? ssHost is null");
return;
}
long sserverId = ssHost.getId();
if (!(ssHost.getType() == Host.Type.SecondaryStorage
|| ssHost.getType() == Host.Type.LocalSecondaryStorage)) {
s_logger.warn("Huh? Agent id " + sserverId + " is not secondary storage host");
return;
}
Map<Long, TemplateInfo> volumeInfos = listVolume(ssHost);
if (volumeInfos == null) {
return;
}
List<VolumeHostVO> dbVolumes = _volumeHostDao.listBySecStorage(sserverId);
List<VolumeHostVO> toBeDownloaded = new ArrayList<VolumeHostVO>(dbVolumes);
for (VolumeHostVO volumeHost : dbVolumes) {
VolumeVO volume = _volumeDao.findById(volumeHost.getVolumeId());
// Exists then don't download
if (volumeInfos.containsKey(volume.getId())) {
TemplateInfo volInfo = volumeInfos.remove(volume.getId());
toBeDownloaded.remove(volumeHost);
s_logger.info(
"Volume Sync found " + volume.getUuid() + " already in the volume host table");
if (volumeHost.getDownloadState() != Status.DOWNLOADED) {
volumeHost.setErrorString("");
}
if (volInfo.isCorrupted()) {
volumeHost.setDownloadState(Status.DOWNLOAD_ERROR);
String msg = "Volume " + volume.getUuid() + " is corrupted on secondary storage ";
volumeHost.setErrorString(msg);
s_logger.info("msg");
if (volumeHost.getDownloadUrl() == null) {
msg =
"Volume ("
+ volume.getUuid()
+ ") with install path "
+ volInfo.getInstallPath()
+ "is corrupted, please check in secondary storage: "
+ volumeHost.getHostId();
s_logger.warn(msg);
} else {
toBeDownloaded.add(volumeHost);
}
} else { // Put them in right status
volumeHost.setDownloadPercent(100);
volumeHost.setDownloadState(Status.DOWNLOADED);
volumeHost.setInstallPath(volInfo.getInstallPath());
volumeHost.setSize(volInfo.getSize());
volumeHost.setPhysicalSize(volInfo.getPhysicalSize());
volumeHost.setLastUpdated(new Date());
_volumeHostDao.update(volumeHost.getId(), volumeHost);
if (volume.getSize() == 0) {
// Set volume size in volumes table
volume.setSize(volInfo.getSize());
_volumeDao.update(volumeHost.getVolumeId(), volume);
}
if (volInfo.getSize() > 0) {
try {
String url = _volumeHostDao.findByVolumeId(volume.getId()).getDownloadUrl();
_resourceLimitMgr.checkResourceLimit(
_accountMgr.getAccount(volume.getAccountId()),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
volInfo.getSize() - UriUtils.getRemoteSize(url));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(
_alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
volume.getDataCenterId(),
volume.getPodId(),
e.getMessage(),
e.getMessage());
} finally {
_resourceLimitMgr.recalculateResourceCount(
volume.getAccountId(),
volume.getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
}
}
}
continue;
}
// Volume is not on secondary but we should download.
if (volumeHost.getDownloadState() != Status.DOWNLOADED) {
s_logger.info(
"Volume Sync did not find "
+ volume.getName()
+ " ready on server "
+ sserverId
+ ", will request download to start/resume shortly");
toBeDownloaded.add(volumeHost);
}
}
// Download volumes which haven't been downloaded yet.
if (toBeDownloaded.size() > 0) {
for (VolumeHostVO volumeHost : toBeDownloaded) {
if (volumeHost.getDownloadUrl() == null) { // If url is null we can't initiate the download
continue;
}
s_logger.debug(
"Volume "
+ volumeHost.getVolumeId()
+ " needs to be downloaded to "
+ ssHost.getName());
downloadVolumeToStorage(
_volumeDao.findById(volumeHost.getVolumeId()),
ssHost,
volumeHost.getDownloadUrl(),
volumeHost.getChecksum(),
volumeHost.getFormat());
}
}
// Delete volumes which are not present on DB.
for (Long uniqueName : volumeInfos.keySet()) {
TemplateInfo vInfo = volumeInfos.get(uniqueName);
DeleteVolumeCommand dtCommand =
new DeleteVolumeCommand(ssHost.getStorageUrl(), vInfo.getInstallPath());
try {
_agentMgr.sendToSecStorage(ssHost, dtCommand, null);
} catch (AgentUnavailableException e) {
String err =
"Failed to delete "
+ vInfo.getTemplateName()
+ " on secondary storage "
+ sserverId
+ " which isn't in the database";
s_logger.error(err);
return;
}
String description =
"Deleted volume "
+ vInfo.getTemplateName()
+ " on secondary storage "
+ sserverId
+ " since it isn't in the database";
s_logger.info(description);
}
}
private void downloadVolumeToStorage(
VolumeVO volume, HostVO sserver, String url, String checkSum, ImageFormat format) {
boolean downloadJobExists = false;
VolumeHostVO volumeHost = null;
volumeHost = _volumeHostDao.findByHostVolume(sserver.getId(), volume.getId());
if (volumeHost == null) {
volumeHost =
new VolumeHostVO(
sserver.getId(),
volume.getId(),
sserver.getDataCenterId(),
new Date(),
0,
VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED,
null,
null,
"jobid0000",
null,
url,
checkSum,
format);
_volumeHostDao.persist(volumeHost);
} else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) {
downloadJobExists = true;
}
Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes();
String secUrl = sserver.getStorageUrl();
if (volumeHost != null) {
start();
DownloadCommand dcmd =
new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format);
if (downloadJobExists) {
dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART);
dcmd.setResourceType(ResourceType.VOLUME);
}
dcmd.setProxy(getHttpProxy());
HostVO ssvm = _ssvmMgr.pickSsvmHost(sserver);
if (ssvm == null) {
s_logger.warn(
"There is no secondary storage VM for secondary storage host " + sserver.getName());
return;
}
DownloadListener dl =
new DownloadListener(
ssvm,
sserver,
volume,
_timer,
_volumeHostDao,
volumeHost.getId(),
this,
dcmd,
_volumeDao,
_storageMgr,
_resourceLimitMgr,
_alertMgr,
_accountMgr);
if (downloadJobExists) {
dl.setCurrState(volumeHost.getDownloadState());
}
DownloadListener old = null;
synchronized (_listenerVolumeMap) {
old = _listenerVolumeMap.put(volumeHost, dl);
}
if (old != null) {
old.abandon();
}
try {
send(ssvm.getId(), dcmd, dl);
} catch (AgentUnavailableException e) {
s_logger.warn(
"Unable to start /resume download of volume "
+ volume.getName()
+ " to "
+ sserver.getName(),
e);
dl.setDisconnected();
dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
}
}
}