public void testDownload() {
s_logger.info("Testing Download answer");
VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
Mockito.when(template.getId()).thenReturn(1L);
Mockito.when(template.getFormat()).thenReturn(ImageFormat.QCOW2);
Mockito.when(template.getName()).thenReturn("templatename");
Mockito.when(template.getTemplateType()).thenReturn(TemplateType.USER);
Mockito.when(template.getDisplayText()).thenReturn("displayText");
Mockito.when(template.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(template.getUrl()).thenReturn("url");
NfsTO nfs = new NfsTO("secUrl", DataStoreRole.Image);
TemplateObjectTO to = new TemplateObjectTO(template);
to.setImageDataStore(nfs);
DownloadCommand cmd = new DownloadCommand(to, 30000000l);
Request req = new Request(1, 1, cmd, true);
req.logD("Debug for Download");
DownloadAnswer answer =
new DownloadAnswer(
"jobId",
50,
"errorString",
Status.ABANDONED,
"filesystempath",
"installpath",
10000000,
20000000,
"chksum");
Response resp = new Response(req, answer);
resp.logD("Debug for Download");
}
public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
super(template.getName(), template.getOrigUrl(), template.getFormat(), template.getAccountId());
_store = template.getDataStore();
installPath = template.getPath();
hvm = template.isRequiresHvm();
checksum = template.getChecksum();
id = template.getId();
description = template.getDescription();
if (_store instanceof NfsTO) {
setSecUrl(((NfsTO) _store).getUrl());
}
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
}
@Override
protected Answer execute(CopyCommand cmd) {
DataTO srcData = cmd.getSrcTO();
DataTO destData = cmd.getDestTO();
DataStoreTO srcDataStore = srcData.getDataStore();
DataStoreTO destDataStore = destData.getDataStore();
// if copied between s3 and nfs cache, go to resource
boolean needDelegation = false;
if (destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache) {
if (srcDataStore instanceof S3TO || srcDataStore instanceof SwiftTO) {
needDelegation = true;
}
}
if (srcDataStore.getRole() == DataStoreRole.ImageCache
&& destDataStore.getRole() == DataStoreRole.Image) {
// need to take extra processing for vmware, such as packing to ova, before sending to S3
if (srcData.getObjectType() == DataObjectType.VOLUME) {
NfsTO cacheStore = (NfsTO) srcDataStore;
String parentPath = storageResource.getRootDir(cacheStore.getUrl(), _nfsVersion);
VolumeObjectTO vol = (VolumeObjectTO) srcData;
String path = vol.getPath();
int index = path.lastIndexOf(File.separator);
String name = path.substring(index + 1);
storageManager.createOva(parentPath + File.separator + path, name);
vol.setPath(path + File.separator + name + ".ova");
} else if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
// sync template from NFS cache to S3 in NFS migration to S3 case
storageManager.createOvaForTemplate((TemplateObjectTO) srcData);
} else if (srcData.getObjectType() == DataObjectType.SNAPSHOT) {
// pack ova first
// sync snapshot from NFS cache to S3 in NFS migration to S3 case
String parentPath = storageResource.getRootDir(srcDataStore.getUrl(), _nfsVersion);
SnapshotObjectTO snap = (SnapshotObjectTO) srcData;
String path = snap.getPath();
int index = path.lastIndexOf(File.separator);
String name = path.substring(index + 1);
String snapDir = path.substring(0, index);
storageManager.createOva(parentPath + File.separator + snapDir, name);
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
// create template from snapshot on src at first, then copy it to s3
TemplateObjectTO cacheTemplate = (TemplateObjectTO) destData;
cacheTemplate.setDataStore(srcDataStore);
CopyCmdAnswer answer = (CopyCmdAnswer) processor.createTemplateFromSnapshot(cmd);
if (!answer.getResult()) {
return answer;
}
cacheTemplate.setDataStore(destDataStore);
TemplateObjectTO template = (TemplateObjectTO) answer.getNewData();
template.setDataStore(srcDataStore);
CopyCommand newCmd =
new CopyCommand(template, destData, cmd.getWait(), cmd.executeInSequence());
Answer result = storageResource.defaultAction(newCmd);
// clean up template data on staging area
try {
DeleteCommand deleteCommand = new DeleteCommand(template);
storageResource.defaultAction(deleteCommand);
} catch (Exception e) {
s_logger.debug("Failed to clean up staging area:", e);
}
return result;
}
}
needDelegation = true;
}
if (srcData.getObjectType() == DataObjectType.SNAPSHOT
&& srcData.getDataStore().getRole() == DataStoreRole.Primary) {
// for back up snapshot, we need to do backup to cache, then to object store if object store
// is used.
if (cmd.getCacheTO() != null) {
cmd.setDestTO(cmd.getCacheTO());
CopyCmdAnswer answer = (CopyCmdAnswer) processor.backupSnapshot(cmd);
if (!answer.getResult()) {
return answer;
}
NfsTO cacheStore = (NfsTO) cmd.getCacheTO().getDataStore();
String parentPath = storageResource.getRootDir(cacheStore.getUrl(), _nfsVersion);
SnapshotObjectTO newSnapshot = (SnapshotObjectTO) answer.getNewData();
String path = newSnapshot.getPath();
int index = path.lastIndexOf(File.separator);
String name = path.substring(index + 1);
String dir = path.substring(0, index);
storageManager.createOva(parentPath + File.separator + dir, name);
newSnapshot.setPath(newSnapshot.getPath() + ".ova");
newSnapshot.setDataStore(cmd.getCacheTO().getDataStore());
CopyCommand newCmd =
new CopyCommand(newSnapshot, destData, cmd.getWait(), cmd.executeInSequence());
Answer result = storageResource.defaultAction(newCmd);
// clean up data on staging area
try {
newSnapshot.setPath(path);
DeleteCommand deleteCommand = new DeleteCommand(newSnapshot);
storageResource.defaultAction(deleteCommand);
} catch (Exception e) {
s_logger.debug("Failed to clean up staging area:", e);
}
return result;
}
}
if (needDelegation) {
return storageResource.defaultAction(cmd);
} else {
return super.execute(cmd);
}
}
@Override
public void handleTemplateSync(DataStore store) {
if (store == null) {
s_logger.warn("Huh? image store is null");
return;
}
long storeId = store.getId();
// add lock to make template sync for a data store only be done once
String lockString = "templatesync.storeId:" + storeId;
GlobalLock syncLock = GlobalLock.getInternLock(lockString);
try {
if (syncLock.lock(3)) {
try {
Long zoneId = store.getScope().getScopeId();
Map<String, TemplateProp> templateInfos = listTemplate(store);
if (templateInfos == null) {
return;
}
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
List<VMTemplateVO> allTemplates = null;
if (zoneId == null) {
// region wide store
allTemplates = _templateDao.listAllActive();
} else {
// zone wide store
allTemplates = _templateDao.listAllInZone(zoneId);
}
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
if (rtngTmplts != null) {
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (!allTemplates.contains(rtngTmplt)) {
allTemplates.add(rtngTmplt);
}
}
}
if (defaultBuiltin != null) {
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (!allTemplates.contains(builtinTmplt)) {
allTemplates.add(builtinTmplt);
}
}
}
toBeDownloaded.addAll(allTemplates);
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore =
_vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
if (templateInfos.containsKey(uniqueName)) {
TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
toBeDownloaded.remove(tmplt);
if (tmpltStore != null) {
s_logger.info("Template Sync found " + uniqueName + " already in the image store");
if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
tmpltStore.setErrorString("");
}
if (tmpltInfo.isCorrupted()) {
tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
String msg =
"Template "
+ tmplt.getName()
+ ":"
+ tmplt.getId()
+ " is corrupted on secondary storage "
+ tmpltStore.getId();
tmpltStore.setErrorString(msg);
s_logger.info("msg");
if (tmplt.getUrl() == null) {
msg =
"Private Template ("
+ tmplt
+ ") with install path "
+ tmpltInfo.getInstallPath()
+ "is corrupted, please check in image store: "
+ tmpltStore.getDataStoreId();
s_logger.warn(msg);
} else {
s_logger.info(
"Removing template_store_ref entry for corrupted template "
+ tmplt.getName());
_vmTemplateStoreDao.remove(tmpltStore.getId());
toBeDownloaded.add(tmplt);
}
} else {
tmpltStore.setDownloadPercent(100);
tmpltStore.setDownloadState(Status.DOWNLOADED);
tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
tmpltStore.setSize(tmpltInfo.getSize());
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
tmpltStore.setLastUpdated(new Date());
// update size in vm_template table
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
tmlpt.setSize(tmpltInfo.getSize());
_templateDao.update(tmplt.getId(), tmlpt);
// Skipping limit checks for SYSTEM Account and for the templates created from
// volumes or snapshots
// which already got checked and incremented during createTemplate API call.
if (tmpltInfo.getSize() > 0
&& tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
&& tmplt.getUrl() != null) {
long accountId = tmplt.getAccountId();
try {
_resourceLimitMgr.checkResourceLimit(
_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(
AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
zoneId,
null,
e.getMessage(),
e.getMessage());
} finally {
_resourceLimitMgr.recalculateResourceCount(
accountId,
_accountMgr.getAccount(accountId).getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage
.getOrdinal());
}
}
}
_vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
} else {
tmpltStore =
new TemplateDataStoreVO(
storeId,
tmplt.getId(),
new Date(),
100,
Status.DOWNLOADED,
null,
null,
null,
tmpltInfo.getInstallPath(),
tmplt.getUrl());
tmpltStore.setSize(tmpltInfo.getSize());
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
tmpltStore.setDataStoreRole(store.getRole());
_vmTemplateStoreDao.persist(tmpltStore);
// update size in vm_template table
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
tmlpt.setSize(tmpltInfo.getSize());
_templateDao.update(tmplt.getId(), tmlpt);
associateTemplateToZone(tmplt.getId(), zoneId);
}
} else {
s_logger.info(
"Template Sync did not find "
+ uniqueName
+ " on image store "
+ storeId
+ ", may request download based on available hypervisor types");
if (tmpltStore != null) {
if (isRegionStore(store)
&& tmpltStore.getDownloadState()
== VMTemplateStorageResourceAssoc.Status.DOWNLOADED
&& tmpltStore.getState() == State.Ready
&& tmpltStore.getInstallPath() == null) {
s_logger.info(
"Keep fake entry in template store table for migration of previous NFS to object store");
} else {
s_logger.info(
"Removing leftover template "
+ uniqueName
+ " entry from template store table");
// remove those leftover entries
_vmTemplateStoreDao.remove(tmpltStore.getId());
}
}
}
}
if (toBeDownloaded.size() > 0) {
/* Only download templates whose hypervirsor type is in the zone */
List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
if (availHypers.isEmpty()) {
/*
* This is for cloudzone, local secondary storage resource
* started before cluster created
*/
availHypers.add(HypervisorType.KVM);
}
/* Baremetal need not to download any template */
availHypers.remove(HypervisorType.BareMetal);
availHypers.add(HypervisorType.None); // bug 9809: resume ISO
// download.
for (VMTemplateVO tmplt : toBeDownloaded) {
if (tmplt.getUrl() == null) { // If url is null we can't
s_logger.info(
"Skip downloading template "
+ tmplt.getUniqueName()
+ " since no url is specified.");
continue;
}
// if this is private template, skip sync to a new image store
if (!tmplt.isPublicTemplate()
&& !tmplt.isFeatured()
&& tmplt.getTemplateType() != TemplateType.SYSTEM) {
s_logger.info(
"Skip sync downloading private template "
+ tmplt.getUniqueName()
+ " to a new image store");
continue;
}
// if this is a region store, and there is already an DOWNLOADED entry there without
// install_path information, which
// means that this is a duplicate entry from migration of previous NFS to staging.
if (isRegionStore(store)) {
TemplateDataStoreVO tmpltStore =
_vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
if (tmpltStore != null
&& tmpltStore.getDownloadState()
== VMTemplateStorageResourceAssoc.Status.DOWNLOADED
&& tmpltStore.getState() == State.Ready
&& tmpltStore.getInstallPath() == null) {
s_logger.info("Skip sync template for migration of previous NFS to object store");
continue;
}
}
if (availHypers.contains(tmplt.getHypervisorType())) {
s_logger.info(
"Downloading template "
+ tmplt.getUniqueName()
+ " to image store "
+ store.getName());
associateTemplateToZone(tmplt.getId(), zoneId);
TemplateInfo tmpl =
_templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
createTemplateAsync(tmpl, store, null);
} else {
s_logger.info(
"Skip downloading template "
+ tmplt.getUniqueName()
+ " since current data center does not have hypervisor "
+ tmplt.getHypervisorType().toString());
}
}
}
for (String uniqueName : templateInfos.keySet()) {
TemplateProp tInfo = templateInfos.get(uniqueName);
if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
// we cannot directly call deleteTemplateSync here to
// reuse delete logic since in this case, our db does not have
// this template at all.
TemplateObjectTO tmplTO = new TemplateObjectTO();
tmplTO.setDataStore(store.getTO());
tmplTO.setPath(tInfo.getInstallPath());
tmplTO.setId(tInfo.getId());
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
EndPoint ep = _epSelector.select(store);
Answer answer = null;
if (ep == null) {
String errMsg =
"No remote endpoint to send command, check if host or ssvm is down?";
s_logger.error(errMsg);
answer = new Answer(dtCommand, false, errMsg);
} else {
answer = ep.sendMessage(dtCommand);
}
if (answer == null || !answer.getResult()) {
s_logger.info("Failed to deleted template at store: " + store.getName());
} else {
String description =
"Deleted template "
+ tInfo.getTemplateName()
+ " on secondary storage "
+ storeId;
s_logger.info(description);
}
}
}
} finally {
syncLock.unlock();
}
} else {
s_logger.info(
"Couldn't get global lock on "
+ lockString
+ ", another thread may be doing template sync on data store "
+ storeId
+ " now.");
}
} finally {
syncLock.releaseRef();
}
}
@Test(priority = -1)
public void setUp() {
ComponentContext.initComponentsLifeCycle();
host = hostDao.findByGuid(this.getHostGuid());
if (host != null) {
dcId = host.getDataCenterId();
clusterId = host.getClusterId();
podId = host.getPodId();
imageStore = this.imageStoreDao.findByName(imageStoreName);
} else {
// create data center
DataCenterVO dc =
new DataCenterVO(
UUID.randomUUID().toString(),
"test",
"8.8.8.8",
null,
"10.0.0.1",
null,
"10.0.0.1/24",
null,
null,
NetworkType.Basic,
null,
null,
true,
true,
null,
null);
dc = dcDao.persist(dc);
dcId = dc.getId();
// create pod
HostPodVO pod =
new HostPodVO(
UUID.randomUUID().toString(),
dc.getId(),
this.getHostGateway(),
this.getHostCidr(),
8,
"test");
pod = podDao.persist(pod);
podId = pod.getId();
// create xen cluster
ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
cluster.setHypervisorType(HypervisorType.VMware.toString());
cluster.setClusterType(ClusterType.ExternalManaged);
cluster.setManagedState(ManagedState.Managed);
cluster = clusterDao.persist(cluster);
clusterId = cluster.getId();
// setup vcenter
ClusterDetailsVO clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "url", null);
this.clusterDetailsDao.persist(clusterDetailVO);
clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "username", null);
this.clusterDetailsDao.persist(clusterDetailVO);
clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "password", null);
this.clusterDetailsDao.persist(clusterDetailVO);
// create xen host
host = new HostVO(this.getHostGuid());
host.setName("devcloud vmware host");
host.setType(Host.Type.Routing);
host.setPrivateIpAddress(this.getHostIp());
host.setDataCenterId(dc.getId());
host.setVersion("6.0.1");
host.setAvailable(true);
host.setSetup(true);
host.setPodId(podId);
host.setLastPinged(0);
host.setResourceState(ResourceState.Enabled);
host.setHypervisorType(HypervisorType.VMware);
host.setClusterId(cluster.getId());
host = hostDao.persist(host);
imageStore = new ImageStoreVO();
imageStore.setName(imageStoreName);
imageStore.setDataCenterId(dcId);
imageStore.setProviderName("CloudStack ImageStore Provider");
imageStore.setRole(DataStoreRole.Image);
imageStore.setUrl(this.getSecondaryStorage());
imageStore.setUuid(UUID.randomUUID().toString());
imageStore.setProtocol("nfs");
imageStore = imageStoreDao.persist(imageStore);
}
image = new VMTemplateVO();
image.setTemplateType(TemplateType.USER);
image.setUrl(this.getTemplateUrl());
image.setUniqueName(UUID.randomUUID().toString());
image.setName(UUID.randomUUID().toString());
image.setPublicTemplate(true);
image.setFeatured(true);
image.setRequiresHvm(true);
image.setBits(64);
image.setFormat(Storage.ImageFormat.VHD);
image.setEnablePassword(true);
image.setEnableSshKey(true);
image.setGuestOSId(1);
image.setBootable(true);
image.setPrepopulate(true);
image.setCrossZones(true);
image.setExtractable(true);
image = imageDataDao.persist(image);
/*
* TemplateDataStoreVO templateStore = new TemplateDataStoreVO();
*
* templateStore.setDataStoreId(imageStore.getId());
* templateStore.setDownloadPercent(100);
* templateStore.setDownloadState(Status.DOWNLOADED);
* templateStore.setDownloadUrl(imageStore.getUrl());
* templateStore.setInstallPath(this.getImageInstallPath());
* templateStore.setTemplateId(image.getId());
* templateStoreDao.persist(templateStore);
*/
DataStore store = this.dataStoreMgr.getDataStore(imageStore.getId(), DataStoreRole.Image);
TemplateInfo template = templateFactory.getTemplate(image.getId(), DataStoreRole.Image);
DataObject templateOnStore = store.create(template);
TemplateObjectTO to = new TemplateObjectTO();
to.setPath(this.getImageInstallPath());
CopyCmdAnswer answer = new CopyCmdAnswer(to);
templateOnStore.processEvent(Event.CreateOnlyRequested);
templateOnStore.processEvent(Event.OperationSuccessed, answer);
}