// @Test public void testCopyBaseImage() { DataStore primaryStore = createPrimaryDataStore(); primaryStoreId = primaryStore.getId(); primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId); VolumeVO volume = createVolume(image.getId(), primaryStore.getId()); VolumeInfo volInfo = this.volFactory.getVolume(volume.getId()); AsyncCallFuture<VolumeApiResult> future = this.volumeService.createVolumeFromTemplateAsync( volInfo, this.primaryStoreId, this.templateFactory.getTemplate(this.image.getId(), DataStoreRole.Image)); try { VolumeApiResult result = future.get(); AssertJUnit.assertTrue(result.isSuccess()); VolumeInfo newVol = result.getVolume(); this.volumeService.destroyVolume(newVol.getId()); VolumeInfo vol = this.volFactory.getVolume(volume.getId()); this.volumeService.expungeVolumeAsync(vol); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ConcurrentOperationException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
// @Test(priority=3) public void createAndDeleteDataDisk() { DataStore primaryStore = this.primaryStore; VolumeVO volume = createVolume(null, primaryStore.getId()); VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore); AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeAsync(vol, primaryStore); try { future.get(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } // delete the volume vol = volumeFactory.getVolume(volume.getId(), primaryStore); future = volumeService.expungeVolumeAsync(vol); try { future.get(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@DB public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) { if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus == Status.ABANDONED)) { VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId()); synchronized (_listenerVolumeMap) { _listenerVolumeMap.remove(volumeHost); } } VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId()); Transaction txn = Transaction.currentTxn(); txn.start(); if (dnldStatus == Status.DOWNLOADED) { // Create usage event long size = -1; if (volumeHost != null) { size = volumeHost.getSize(); volume.setSize(size); this._volumeDao.update(volume.getId(), volume); } else { s_logger.warn("Failed to get size for volume" + volume.getName()); } String eventType = EventTypes.EVENT_VOLUME_UPLOAD; if (volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { UsageEventUtils.publishUsageEvent( eventType, volume.getAccountId(), host.getDataCenterId(), volume.getId(), volume.getName(), null, 0l, size, volume.getClass().getName(), volume.getUuid()); } } else if (dnldStatus == Status.DOWNLOAD_ERROR || dnldStatus == Status.ABANDONED || dnldStatus == Status.UNKNOWN) { // Decrement the volume and secondary storage space count _resourceLimitMgr.decrementResourceCount( volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume); _resourceLimitMgr.recalculateResourceCount( volume.getAccountId(), volume.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); } txn.commit(); }
@Test public void testCreateTemplateFromVolume() { DataStore primaryStore = createPrimaryDataStore(); primaryStoreId = primaryStore.getId(); primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId); VolumeVO volume = createVolume(null, primaryStore.getId()); VolumeInfo volInfo = this.volFactory.getVolume(volume.getId()); AsyncCallFuture<VolumeApiResult> future = this.volumeService.createVolumeAsync(volInfo, primaryStore); try { VolumeApiResult result = future.get(); AssertJUnit.assertTrue(result.isSuccess()); volInfo = result.getVolume(); VMTemplateVO templateVO = createTemplateInDb(); TemplateInfo tmpl = this.templateFactory.getTemplate(templateVO.getId(), DataStoreRole.Image); DataStore imageStore = this.dataStoreMgr.getImageStore(this.dcId); this.imageService.createTemplateFromVolumeAsync(volInfo, tmpl, imageStore); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@Test public void testDeleteDisk() { DataStore primaryStore = createPrimaryDataStore(); primaryStoreId = primaryStore.getId(); primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId); VolumeVO volume = createVolume(null, primaryStore.getId()); VolumeInfo volInfo = this.volFactory.getVolume(volume.getId()); AsyncCallFuture<VolumeApiResult> future = this.volumeService.createVolumeAsync(volInfo, primaryStore); try { VolumeApiResult result = future.get(); VolumeInfo vol = result.getVolume(); this.volumeService.destroyVolume(volInfo.getId()); volInfo = this.volFactory.getVolume(vol.getId()); this.volumeService.expungeVolumeAsync(volInfo); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ConcurrentOperationException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@Override public void createAsync( DataStore dataStore, DataObject dataObject, AsyncCompletionCallback<CreateCmdResult> callback) { String iqn = null; String errMsg = null; if (dataObject.getType() == DataObjectType.VOLUME) { VolumeInfo volumeInfo = (VolumeInfo) dataObject; AccountVO account = _accountDao.findById(volumeInfo.getAccountId()); String sfAccountName = getSfAccountName(account.getUuid(), account.getAccountId()); long storagePoolId = dataStore.getId(); SolidFireConnection sfConnection = getSolidFireConnection(storagePoolId); if (!sfAccountExists(sfAccountName, sfConnection)) { SolidFireUtil.SolidFireAccount sfAccount = createSolidFireAccount(sfAccountName, sfConnection); updateCsDbWithAccountInfo(account.getId(), sfAccount); } SolidFireUtil.SolidFireVolume sfVolume = createSolidFireVolume(volumeInfo, sfConnection); iqn = sfVolume.getIqn(); VolumeVO volume = this._volumeDao.findById(volumeInfo.getId()); volume.set_iScsiName(iqn); volume.setFolder(String.valueOf(sfVolume.getId())); volume.setPoolType(StoragePoolType.IscsiLUN); volume.setPoolId(storagePoolId); _volumeDao.update(volume.getId(), volume); StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId()); long capacityBytes = storagePool.getCapacityBytes(); long usedBytes = storagePool.getUsedBytes(); usedBytes += volumeInfo.getSize(); storagePool.setUsedBytes(usedBytes > capacityBytes ? capacityBytes : usedBytes); _storagePoolDao.update(storagePoolId, storagePool); } else { errMsg = "Invalid DataObjectType (" + dataObject.getType() + ") passed to createAsync"; } // path = iqn // size is pulled from DataObject instance, if errMsg is null CreateCmdResult result = new CreateCmdResult(iqn, new Answer(null, errMsg == null, errMsg)); result.setResult(errMsg); callback.complete(result); }
@Test public void testCreateDataDisk() { DataStore primaryStore = createPrimaryDataStore(); primaryStoreId = primaryStore.getId(); primaryStore = this.dataStoreMgr.getPrimaryDataStore(primaryStoreId); VolumeVO volume = createVolume(null, primaryStore.getId()); VolumeInfo volInfo = this.volFactory.getVolume(volume.getId()); this.volumeService.createVolumeAsync(volInfo, primaryStore); }
@Override public void run() { try { List<VolumeVO> volumes = _volsDao.listAll(); Map<Long, List<VolumeCommand>> commandsByPool = new HashMap<Long, List<VolumeCommand>>(); for (VolumeVO volume : volumes) { List<VolumeCommand> commands = commandsByPool.get(volume.getPoolId()); if (commands == null) { commands = new ArrayList<VolumeCommand>(); commandsByPool.put(volume.getPoolId(), commands); } VolumeCommand vCommand = new VolumeCommand(); vCommand.volumeId = volume.getId(); vCommand.command = new GetFileStatsCommand(volume); commands.add(vCommand); } ConcurrentHashMap<Long, VolumeStats> volumeStats = new ConcurrentHashMap<Long, VolumeStats>(); for (Iterator<Long> iter = commandsByPool.keySet().iterator(); iter.hasNext(); ) { Long poolId = iter.next(); List<VolumeCommand> commandsList = commandsByPool.get(poolId); long[] volumeIdArray = new long[commandsList.size()]; Commands commands = new Commands(OnError.Continue); for (int i = 0; i < commandsList.size(); i++) { VolumeCommand vCommand = commandsList.get(i); volumeIdArray[i] = vCommand.volumeId; commands.addCommand(vCommand.command); } List<StoragePoolHostVO> poolhosts = _storagePoolHostDao.listByPoolId(poolId); for (StoragePoolHostVO poolhost : poolhosts) { Answer[] answers = _agentMgr.send(poolhost.getHostId(), commands); if (answers != null) { long totalBytes = 0L; for (int i = 0; i < answers.length; i++) { if (answers[i].getResult()) { VolumeStats vStats = (VolumeStats) answers[i]; volumeStats.put(volumeIdArray[i], vStats); totalBytes += vStats.getBytesUsed(); } } break; } } } // We replace the existing volumeStats so that it does not grow with no bounds _volumeStats = volumeStats; } catch (AgentUnavailableException e) { s_logger.debug(e.getMessage()); } catch (Throwable t) { s_logger.error("Error trying to retrieve volume stats", t); } }
@Test(priority = 2) public void createVolumeFromTemplate() { DataStore primaryStore = this.primaryStore; TemplateInfo te = createTemplate(); VolumeVO volume = createVolume(te.getId(), primaryStore.getId()); VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore); // ve.createVolumeFromTemplate(primaryStore.getId(), new VHD(), te); AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeFromTemplateAsync(vol, primaryStore.getId(), te); try { future.get(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
protected void createDb() { DataCenterVO dc = new DataCenterVO( UUID.randomUUID().toString(), "test", "8.8.8.8", null, "10.0.0.1", null, "10.0.0.1/24", null, null, NetworkType.Basic, null, null, true, true, null, null); dc = dcDao.persist(dc); dcId = dc.getId(); HostPodVO pod = new HostPodVO(UUID.randomUUID().toString(), dc.getId(), "255.255.255.255", "", 8, "test"); pod = podDao.persist(pod); podId = pod.getId(); ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster"); cluster.setHypervisorType(HypervisorType.XenServer.toString()); cluster.setClusterType(ClusterType.CloudManaged); cluster.setManagedState(ManagedState.Managed); cluster = clusterDao.persist(cluster); clusterId = cluster.getId(); DataStoreProvider provider = providerMgr.getDataStoreProvider("ancient primary data store provider"); storage = new StoragePoolVO(); storage.setDataCenterId(dcId); storage.setPodId(podId); storage.setPoolType(StoragePoolType.NetworkFilesystem); storage.setClusterId(clusterId); storage.setStatus(StoragePoolStatus.Up); storage.setScope(ScopeType.CLUSTER); storage.setAvailableBytes(1000); storage.setCapacityBytes(20000); storage.setHostAddress(UUID.randomUUID().toString()); storage.setPath(UUID.randomUUID().toString()); storage.setStorageProviderName(provider.getName()); storage = storagePoolDao.persist(storage); storagePoolId = storage.getId(); storageMgr.createCapacityEntry(storage.getId()); diskOffering = new DiskOfferingVO(); diskOffering.setDiskSize(500); diskOffering.setName("test-disk"); diskOffering.setSystemUse(false); diskOffering.setUseLocalStorage(false); diskOffering.setCustomized(false); diskOffering.setRecreatable(false); diskOffering = diskOfferingDao.persist(diskOffering); diskOfferingId = diskOffering.getId(); volume = new VolumeVO( Volume.Type.ROOT, "volume", dcId, 1, 1, diskOffering.getId(), diskOffering.getDiskSize()); volume = volumeDao.persist(volume); volumeId = volume.getId(); }
@Override public void handleVolumeSync(HostVO ssHost) { if (ssHost == null) { s_logger.warn("Huh? ssHost is null"); return; } long sserverId = ssHost.getId(); if (!(ssHost.getType() == Host.Type.SecondaryStorage || ssHost.getType() == Host.Type.LocalSecondaryStorage)) { s_logger.warn("Huh? Agent id " + sserverId + " is not secondary storage host"); return; } Map<Long, TemplateInfo> volumeInfos = listVolume(ssHost); if (volumeInfos == null) { return; } List<VolumeHostVO> dbVolumes = _volumeHostDao.listBySecStorage(sserverId); List<VolumeHostVO> toBeDownloaded = new ArrayList<VolumeHostVO>(dbVolumes); for (VolumeHostVO volumeHost : dbVolumes) { VolumeVO volume = _volumeDao.findById(volumeHost.getVolumeId()); // Exists then don't download if (volumeInfos.containsKey(volume.getId())) { TemplateInfo volInfo = volumeInfos.remove(volume.getId()); toBeDownloaded.remove(volumeHost); s_logger.info( "Volume Sync found " + volume.getUuid() + " already in the volume host table"); if (volumeHost.getDownloadState() != Status.DOWNLOADED) { volumeHost.setErrorString(""); } if (volInfo.isCorrupted()) { volumeHost.setDownloadState(Status.DOWNLOAD_ERROR); String msg = "Volume " + volume.getUuid() + " is corrupted on secondary storage "; volumeHost.setErrorString(msg); s_logger.info("msg"); if (volumeHost.getDownloadUrl() == null) { msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath() + "is corrupted, please check in secondary storage: " + volumeHost.getHostId(); s_logger.warn(msg); } else { toBeDownloaded.add(volumeHost); } } else { // Put them in right status volumeHost.setDownloadPercent(100); volumeHost.setDownloadState(Status.DOWNLOADED); volumeHost.setInstallPath(volInfo.getInstallPath()); volumeHost.setSize(volInfo.getSize()); volumeHost.setPhysicalSize(volInfo.getPhysicalSize()); volumeHost.setLastUpdated(new Date()); _volumeHostDao.update(volumeHost.getId(), volumeHost); if (volume.getSize() == 0) { // Set volume size in volumes table volume.setSize(volInfo.getSize()); _volumeDao.update(volumeHost.getVolumeId(), volume); } if (volInfo.getSize() > 0) { try { String url = _volumeHostDao.findByVolumeId(volume.getId()).getDownloadUrl(); _resourceLimitMgr.checkResourceLimit( _accountMgr.getAccount(volume.getAccountId()), com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize() - UriUtils.getRemoteSize(url)); } catch (ResourceAllocationException e) { s_logger.warn(e.getMessage()); _alertMgr.sendAlert( _alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage()); } finally { _resourceLimitMgr.recalculateResourceCount( volume.getAccountId(), volume.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); } } } continue; } // Volume is not on secondary but we should download. if (volumeHost.getDownloadState() != Status.DOWNLOADED) { s_logger.info( "Volume Sync did not find " + volume.getName() + " ready on server " + sserverId + ", will request download to start/resume shortly"); toBeDownloaded.add(volumeHost); } } // Download volumes which haven't been downloaded yet. if (toBeDownloaded.size() > 0) { for (VolumeHostVO volumeHost : toBeDownloaded) { if (volumeHost.getDownloadUrl() == null) { // If url is null we can't initiate the download continue; } s_logger.debug( "Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + ssHost.getName()); downloadVolumeToStorage( _volumeDao.findById(volumeHost.getVolumeId()), ssHost, volumeHost.getDownloadUrl(), volumeHost.getChecksum(), volumeHost.getFormat()); } } // Delete volumes which are not present on DB. for (Long uniqueName : volumeInfos.keySet()) { TemplateInfo vInfo = volumeInfos.get(uniqueName); DeleteVolumeCommand dtCommand = new DeleteVolumeCommand(ssHost.getStorageUrl(), vInfo.getInstallPath()); try { _agentMgr.sendToSecStorage(ssHost, dtCommand, null); } catch (AgentUnavailableException e) { String err = "Failed to delete " + vInfo.getTemplateName() + " on secondary storage " + sserverId + " which isn't in the database"; s_logger.error(err); return; } String description = "Deleted volume " + vInfo.getTemplateName() + " on secondary storage " + sserverId + " since it isn't in the database"; s_logger.info(description); } }
private void downloadVolumeToStorage( VolumeVO volume, HostVO sserver, String url, String checkSum, ImageFormat format) { boolean downloadJobExists = false; VolumeHostVO volumeHost = null; volumeHost = _volumeHostDao.findByHostVolume(sserver.getId(), volume.getId()); if (volumeHost == null) { volumeHost = new VolumeHostVO( sserver.getId(), volume.getId(), sserver.getDataCenterId(), new Date(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, url, checkSum, format); _volumeHostDao.persist(volumeHost); } else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) { downloadJobExists = true; } Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes(); String secUrl = sserver.getStorageUrl(); if (volumeHost != null) { start(); DownloadCommand dcmd = new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format); if (downloadJobExists) { dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART); dcmd.setResourceType(ResourceType.VOLUME); } dcmd.setProxy(getHttpProxy()); HostVO ssvm = _ssvmMgr.pickSsvmHost(sserver); if (ssvm == null) { s_logger.warn( "There is no secondary storage VM for secondary storage host " + sserver.getName()); return; } DownloadListener dl = new DownloadListener( ssvm, sserver, volume, _timer, _volumeHostDao, volumeHost.getId(), this, dcmd, _volumeDao, _storageMgr, _resourceLimitMgr, _alertMgr, _accountMgr); if (downloadJobExists) { dl.setCurrState(volumeHost.getDownloadState()); } DownloadListener old = null; synchronized (_listenerVolumeMap) { old = _listenerVolumeMap.put(volumeHost, dl); } if (old != null) { old.abandon(); } try { send(ssvm.getId(), dcmd, dl); } catch (AgentUnavailableException e) { s_logger.warn( "Unable to start /resume download of volume " + volume.getName() + " to " + sserver.getName(), e); dl.setDisconnected(); dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); } } }