@Override
public String getSecondaryStorageStoreUrl(long dcId) {
String secUrl = null;
DataStore secStore = _dataStoreMgr.getImageStore(dcId);
if (secStore != null) {
secUrl = secStore.getUri();
}
if (secUrl == null) {
// we are using non-NFS image store, then use cache storage instead
s_logger.info("Secondary storage is not NFS, we need to use staging storage");
DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId);
if (cacheStore != null) {
secUrl = cacheStore.getUri();
} else {
s_logger.warn("No staging storage is found when non-NFS secondary storage is used");
}
}
return secUrl;
}
// This routine is used to push templates currently on cache store, but not in region store to
// region store.
// used in migrating existing NFS secondary storage to S3.
@Override
public void syncTemplateToRegionStore(long templateId, DataStore store) {
if (_storeMgr.isRegionStore(store)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Sync template " + templateId + " from cache to object store...");
}
// if template is on region wide object store, check if it is really downloaded there (by
// checking install_path). Sync template to region
// wide store if it is not there physically.
TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);
if (tmplOnStore == null) {
throw new CloudRuntimeException(
"Cannot find an entry in template_store_ref for template "
+ templateId
+ " on region store: "
+ store.getName());
}
if (tmplOnStore.getInstallPath() == null || tmplOnStore.getInstallPath().length() == 0) {
// template is not on region store yet, sync to region store
TemplateInfo srcTemplate = _templateFactory.getReadyTemplateOnCache(templateId);
if (srcTemplate == null) {
throw new CloudRuntimeException(
"Cannot find template " + templateId + " on cache store");
}
AsyncCallFuture<TemplateApiResult> future = syncToRegionStoreAsync(srcTemplate, store);
try {
TemplateApiResult result = future.get();
if (result.isFailed()) {
throw new CloudRuntimeException(
"sync template from cache to region wide store failed for image store "
+ store.getName()
+ ":"
+ result.getResult());
}
_cacheMgr.releaseCacheObject(
srcTemplate); // reduce reference count for template on cache, so it can recycled by
// schedule
} catch (Exception ex) {
throw new CloudRuntimeException(
"sync template from cache to region wide store failed for image store "
+ store.getName());
}
}
}
}
@Override
public void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId) {
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
List<DataStore> stores = _storeMgr.getImageStoresByScope(new ZoneScope(dcId));
if (stores == null || stores.isEmpty()) {
return;
}
/* Download all the templates in zone with the same hypervisortype */
for (DataStore store : stores) {
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (rtngTmplt.getHypervisorType() == hostHyper) {
toBeDownloaded.add(rtngTmplt);
}
}
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (builtinTmplt.getHypervisorType() == hostHyper) {
toBeDownloaded.add(builtinTmplt);
}
}
for (VMTemplateVO template : toBeDownloaded) {
TemplateDataStoreVO tmpltHost =
_vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId());
if (tmpltHost == null
|| tmpltHost.getState() != ObjectInDataStoreStateMachine.State.Ready) {
associateTemplateToZone(template.getId(), dcId);
s_logger.info(
"Downloading builtin template "
+ template.getUniqueName()
+ " to data center: "
+ dcId);
TemplateInfo tmplt = _templateFactory.getTemplate(template.getId(), DataStoreRole.Image);
createTemplateAsync(tmplt, store, null);
}
}
}
}
@Override
public void handleTemplateSync(DataStore store) {
if (store == null) {
s_logger.warn("Huh? image store is null");
return;
}
long storeId = store.getId();
// add lock to make template sync for a data store only be done once
String lockString = "templatesync.storeId:" + storeId;
GlobalLock syncLock = GlobalLock.getInternLock(lockString);
try {
if (syncLock.lock(3)) {
try {
Long zoneId = store.getScope().getScopeId();
Map<String, TemplateProp> templateInfos = listTemplate(store);
if (templateInfos == null) {
return;
}
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
List<VMTemplateVO> allTemplates = null;
if (zoneId == null) {
// region wide store
allTemplates =
_templateDao.listByState(
VirtualMachineTemplate.State.Active,
VirtualMachineTemplate.State.NotUploaded,
VirtualMachineTemplate.State.UploadInProgress);
} else {
// zone wide store
allTemplates =
_templateDao.listInZoneByState(
zoneId,
VirtualMachineTemplate.State.Active,
VirtualMachineTemplate.State.NotUploaded,
VirtualMachineTemplate.State.UploadInProgress);
}
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
if (rtngTmplts != null) {
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (!allTemplates.contains(rtngTmplt)) {
allTemplates.add(rtngTmplt);
}
}
}
if (defaultBuiltin != null) {
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (!allTemplates.contains(builtinTmplt)) {
allTemplates.add(builtinTmplt);
}
}
}
toBeDownloaded.addAll(allTemplates);
final StateMachine2<
VirtualMachineTemplate.State,
VirtualMachineTemplate.Event,
VirtualMachineTemplate>
stateMachine = VirtualMachineTemplate.State.getStateMachine();
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore =
_vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
if (templateInfos.containsKey(uniqueName)) {
TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
toBeDownloaded.remove(tmplt);
if (tmpltStore != null) {
s_logger.info("Template Sync found " + uniqueName + " already in the image store");
if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
tmpltStore.setErrorString("");
}
if (tmpltInfo.isCorrupted()) {
tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
String msg =
"Template "
+ tmplt.getName()
+ ":"
+ tmplt.getId()
+ " is corrupted on secondary storage "
+ tmpltStore.getId();
tmpltStore.setErrorString(msg);
s_logger.info(msg);
_alertMgr.sendAlert(
AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, zoneId, null, msg, msg);
if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
|| tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
s_logger.info(
"Template Sync found "
+ uniqueName
+ " on image store "
+ storeId
+ " uploaded using SSVM as corrupted, marking it as failed");
tmpltStore.setState(State.Failed);
try {
stateMachine.transitTo(
tmplt, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
} catch (NoTransitionException e) {
s_logger.error(
"Unexpected state transition exception for template "
+ tmplt.getName()
+ ". Details: "
+ e.getMessage());
}
} else if (tmplt.getUrl() == null) {
msg =
"Private template ("
+ tmplt
+ ") with install path "
+ tmpltInfo.getInstallPath()
+ " is corrupted, please check in image store: "
+ tmpltStore.getDataStoreId();
s_logger.warn(msg);
} else {
s_logger.info(
"Removing template_store_ref entry for corrupted template "
+ tmplt.getName());
_vmTemplateStoreDao.remove(tmpltStore.getId());
toBeDownloaded.add(tmplt);
}
} else {
tmpltStore.setDownloadPercent(100);
tmpltStore.setDownloadState(Status.DOWNLOADED);
tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
tmpltStore.setSize(tmpltInfo.getSize());
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
tmpltStore.setLastUpdated(new Date());
// update size in vm_template table
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
tmlpt.setSize(tmpltInfo.getSize());
_templateDao.update(tmplt.getId(), tmlpt);
if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
|| tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
try {
stateMachine.transitTo(
tmplt,
VirtualMachineTemplate.Event.OperationSucceeded,
null,
_templateDao);
} catch (NoTransitionException e) {
s_logger.error(
"Unexpected state transition exception for template "
+ tmplt.getName()
+ ". Details: "
+ e.getMessage());
}
}
// Skipping limit checks for SYSTEM Account and for the templates created from
// volumes or snapshots
// which already got checked and incremented during createTemplate API call.
if (tmpltInfo.getSize() > 0
&& tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
&& tmplt.getUrl() != null) {
long accountId = tmplt.getAccountId();
try {
_resourceLimitMgr.checkResourceLimit(
_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(
AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
zoneId,
null,
e.getMessage(),
e.getMessage());
} finally {
_resourceLimitMgr.recalculateResourceCount(
accountId,
_accountMgr.getAccount(accountId).getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage
.getOrdinal());
}
}
}
_vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
} else {
tmpltStore =
new TemplateDataStoreVO(
storeId,
tmplt.getId(),
new Date(),
100,
Status.DOWNLOADED,
null,
null,
null,
tmpltInfo.getInstallPath(),
tmplt.getUrl());
tmpltStore.setSize(tmpltInfo.getSize());
tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
tmpltStore.setDataStoreRole(store.getRole());
_vmTemplateStoreDao.persist(tmpltStore);
// update size in vm_template table
VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
tmlpt.setSize(tmpltInfo.getSize());
_templateDao.update(tmplt.getId(), tmlpt);
associateTemplateToZone(tmplt.getId(), zoneId);
}
} else if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
|| tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
s_logger.info(
"Template Sync did not find "
+ uniqueName
+ " on image store "
+ storeId
+ " uploaded using SSVM, marking it as failed");
toBeDownloaded.remove(tmplt);
tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
String msg =
"Template "
+ tmplt.getName()
+ ":"
+ tmplt.getId()
+ " is corrupted on secondary storage "
+ tmpltStore.getId();
tmpltStore.setErrorString(msg);
tmpltStore.setState(State.Failed);
_vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
try {
stateMachine.transitTo(
tmplt, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
} catch (NoTransitionException e) {
s_logger.error(
"Unexpected state transition exception for template "
+ tmplt.getName()
+ ". Details: "
+ e.getMessage());
}
} else {
s_logger.info(
"Template Sync did not find "
+ uniqueName
+ " on image store "
+ storeId
+ ", may request download based on available hypervisor types");
if (tmpltStore != null) {
if (_storeMgr.isRegionStore(store)
&& tmpltStore.getDownloadState()
== VMTemplateStorageResourceAssoc.Status.DOWNLOADED
&& tmpltStore.getState() == State.Ready
&& tmpltStore.getInstallPath() == null) {
s_logger.info(
"Keep fake entry in template store table for migration of previous NFS to object store");
} else {
s_logger.info(
"Removing leftover template "
+ uniqueName
+ " entry from template store table");
// remove those leftover entries
_vmTemplateStoreDao.remove(tmpltStore.getId());
}
}
}
}
if (toBeDownloaded.size() > 0) {
/* Only download templates whose hypervirsor type is in the zone */
List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
if (availHypers.isEmpty()) {
/*
* This is for cloudzone, local secondary storage resource
* started before cluster created
*/
availHypers.add(HypervisorType.KVM);
}
/* Baremetal need not to download any template */
availHypers.remove(HypervisorType.BareMetal);
availHypers.add(HypervisorType.None); // bug 9809: resume ISO
// download.
for (VMTemplateVO tmplt : toBeDownloaded) {
if (tmplt.getUrl() == null) { // If url is null, skip downloading
s_logger.info(
"Skip downloading template "
+ tmplt.getUniqueName()
+ " since no url is specified.");
continue;
}
// if this is private template, skip sync to a new image store
if (!tmplt.isPublicTemplate()
&& !tmplt.isFeatured()
&& tmplt.getTemplateType() != TemplateType.SYSTEM) {
s_logger.info(
"Skip sync downloading private template "
+ tmplt.getUniqueName()
+ " to a new image store");
continue;
}
// if this is a region store, and there is already an DOWNLOADED entry there without
// install_path information, which
// means that this is a duplicate entry from migration of previous NFS to staging.
if (_storeMgr.isRegionStore(store)) {
TemplateDataStoreVO tmpltStore =
_vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
if (tmpltStore != null
&& tmpltStore.getDownloadState()
== VMTemplateStorageResourceAssoc.Status.DOWNLOADED
&& tmpltStore.getState() == State.Ready
&& tmpltStore.getInstallPath() == null) {
s_logger.info("Skip sync template for migration of previous NFS to object store");
continue;
}
}
if (availHypers.contains(tmplt.getHypervisorType())) {
s_logger.info(
"Downloading template "
+ tmplt.getUniqueName()
+ " to image store "
+ store.getName());
associateTemplateToZone(tmplt.getId(), zoneId);
TemplateInfo tmpl =
_templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
createTemplateAsync(tmpl, store, null);
} else {
s_logger.info(
"Skip downloading template "
+ tmplt.getUniqueName()
+ " since current data center does not have hypervisor "
+ tmplt.getHypervisorType().toString());
}
}
}
for (String uniqueName : templateInfos.keySet()) {
TemplateProp tInfo = templateInfos.get(uniqueName);
if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
// we cannot directly call deleteTemplateSync here to reuse delete logic since in this
// case db does not have this template at all.
TemplateObjectTO tmplTO = new TemplateObjectTO();
tmplTO.setDataStore(store.getTO());
tmplTO.setPath(tInfo.getInstallPath());
tmplTO.setId(tInfo.getId());
DeleteCommand dtCommand = new DeleteCommand(tmplTO);
EndPoint ep = _epSelector.select(store);
Answer answer = null;
if (ep == null) {
String errMsg =
"No remote endpoint to send command, check if host or ssvm is down?";
s_logger.error(errMsg);
answer = new Answer(dtCommand, false, errMsg);
} else {
answer = ep.sendMessage(dtCommand);
}
if (answer == null || !answer.getResult()) {
s_logger.info("Failed to deleted template at store: " + store.getName());
} else {
String description =
"Deleted template "
+ tInfo.getTemplateName()
+ " on secondary storage "
+ storeId;
s_logger.info(description);
}
}
}
} finally {
syncLock.unlock();
}
} else {
s_logger.info(
"Couldn't get global lock on "
+ lockString
+ ", another thread may be doing template sync on data store "
+ storeId
+ " now.");
}
} finally {
syncLock.releaseRef();
}
}
