private void updateNetworkLabels(HostVO host) {
// check if networkLabels need to be updated in details
// we send only private and storage network label to the resource.
String privateNetworkLabel =
_networkMgr.getDefaultManagementTrafficLabel(
host.getDataCenterId(), host.getHypervisorType());
String storageNetworkLabel =
_networkMgr.getDefaultStorageTrafficLabel(host.getDataCenterId(), host.getHypervisorType());
String privateDevice = host.getDetail("private.network.device");
String storageDevice = host.getDetail("storage.network.device1");
boolean update = false;
if (privateNetworkLabel != null && !privateNetworkLabel.equalsIgnoreCase(privateDevice)) {
host.setDetail("private.network.device", privateNetworkLabel);
update = true;
}
if (storageNetworkLabel != null && !storageNetworkLabel.equalsIgnoreCase(storageDevice)) {
host.setDetail("storage.network.device1", storageNetworkLabel);
update = true;
}
if (update) {
_hostDao.saveDetails(host);
}
}
@Override
public void scheduleRestartForVmsOnHost(final HostVO host) {
if (host.getType() != Host.Type.Routing) {
return;
}
s_logger.warn("Scheduling restart for VMs on host " + host.getId());
final List<VMInstanceVO> vms = _instanceDao.listByHostId(host.getId());
final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
// send an email alert that the host is down
StringBuilder sb = null;
if ((vms != null) && !vms.isEmpty()) {
sb = new StringBuilder();
sb.append(" Starting HA on the following VMs: ");
// collect list of vm names for the alert email
VMInstanceVO vm = vms.get(0);
if (vm.isHaEnabled()) {
sb.append(" " + vm.getName());
}
for (int i = 1; i < vms.size(); i++) {
vm = vms.get(i);
if (vm.isHaEnabled()) {
sb.append(" " + vm.getName());
}
}
}
// send an email alert that the host is down, include VMs
HostPodVO podVO = _podDao.findById(host.getPodId());
String hostDesc =
"name: "
+ host.getName()
+ " (id:"
+ host.getId()
+ "), availability zone: "
+ dcVO.getName()
+ ", pod: "
+ podVO.getName();
_alertMgr.sendAlert(
AlertManager.ALERT_TYPE_HOST,
host.getDataCenterId(),
host.getPodId(),
"Host is down, " + hostDesc,
"Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : ""));
for (final VMInstanceVO vm : vms) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Notifying HA Mgr of to investigate vm " + vm.getId() + "-" + vm.getName());
}
scheduleRestart(vm, true);
}
}
// TODO: add test for method
@Override
public final HostVO createHostVOForDirectConnectAgent(
final HostVO host,
final StartupCommand[] startup,
final ServerResource resource,
final Map<String, String> details,
final List<String> hostTags) {
StartupCommand firstCmd = startup[0];
if (!(firstCmd instanceof StartupRoutingCommand)) {
return null;
}
StartupRoutingCommand ssCmd = ((StartupRoutingCommand) firstCmd);
if (ssCmd.getHypervisorType() != HypervisorType.Hyperv) {
return null;
}
s_logger.info(
"Host: "
+ host.getName()
+ " connected with hypervisor type: "
+ HypervisorType.Hyperv
+ ". Checking CIDR...");
HostPodVO pod = _podDao.findById(host.getPodId());
DataCenterVO dc = _dcDao.findById(host.getDataCenterId());
_resourceMgr.checkCIDR(pod, dc, ssCmd.getPrivateIpAddress(), ssCmd.getPrivateNetmask());
return _resourceMgr.fillRoutingHostVO(host, ssCmd, HypervisorType.Hyperv, details, hostTags);
}
protected HashMap<String, Object> buildConfigParams(HostVO host) {
HashMap<String, Object> params = new HashMap<String, Object>(host.getDetails().size() + 5);
params.putAll(host.getDetails());
params.put("guid", host.getGuid());
params.put("zone", Long.toString(host.getDataCenterId()));
if (host.getPodId() != null) {
params.put("pod", Long.toString(host.getPodId()));
}
if (host.getClusterId() != null) {
params.put("cluster", Long.toString(host.getClusterId()));
String guid = null;
ClusterVO cluster = _clusterDao.findById(host.getClusterId());
if (cluster.getGuid() == null) {
guid = host.getDetail("pool");
} else {
guid = cluster.getGuid();
}
if (guid != null && !guid.isEmpty()) {
params.put("pool", guid);
}
}
params.put("ipaddress", host.getPrivateIpAddress());
params.put("secondary.storage.vm", "false");
params.put(
"max.template.iso.size", _configDao.getValue(Config.MaxTemplateAndIsoSize.toString()));
params.put("migratewait", _configDao.getValue(Config.MigrateWait.toString()));
return params;
}
@Override
public boolean prepareCreateTemplate(Long pxeServerId, UserVm vm, String templateUrl) {
List<NicVO> nics = _nicDao.listByVmId(vm.getId());
if (nics.size() != 1) {
throw new CloudRuntimeException("Wrong nic number " + nics.size() + " of vm " + vm.getId());
}
/* use last host id when VM stopped */
Long hostId = (vm.getHostId() == null ? vm.getLastHostId() : vm.getHostId());
HostVO host = _hostDao.findById(hostId);
DataCenterVO dc = _dcDao.findById(host.getDataCenterId());
NicVO nic = nics.get(0);
String mask = nic.getNetmask();
String mac = nic.getMacAddress();
String ip = nic.getIp4Address();
String gateway = nic.getGateway();
String dns = dc.getDns1();
if (dns == null) {
dns = dc.getDns2();
}
try {
prepareCreateTemplateCommand cmd =
new prepareCreateTemplateCommand(ip, mac, mask, gateway, dns, templateUrl);
Answer ans = _agentMgr.send(pxeServerId, cmd);
return ans.getResult();
} catch (Exception e) {
s_logger.debug("Prepare for creating baremetal template failed", e);
return false;
}
}
@Override
public void handleSysTemplateDownload(HostVO host) {
List<HypervisorType> hypers =
_resourceMgr.listAvailHypervisorInZone(host.getId(), host.getDataCenterId());
HypervisorType hostHyper = host.getHypervisorType();
if (hypers.contains(hostHyper)) {
return;
}
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
List<HostVO> ssHosts =
_resourceMgr.listAllUpAndEnabledHostsInOneZoneByType(
Host.Type.SecondaryStorage, host.getDataCenterId());
if (ssHosts == null || ssHosts.isEmpty()) {
return;
}
/*Download all the templates in zone with the same hypervisortype*/
for (HostVO ssHost : ssHosts) {
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (rtngTmplt.getHypervisorType() == hostHyper) {
toBeDownloaded.add(rtngTmplt);
}
}
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (builtinTmplt.getHypervisorType() == hostHyper) {
toBeDownloaded.add(builtinTmplt);
}
}
for (VMTemplateVO template : toBeDownloaded) {
VMTemplateHostVO tmpltHost =
_vmTemplateHostDao.findByHostTemplate(ssHost.getId(), template.getId());
if (tmpltHost == null || tmpltHost.getDownloadState() != Status.DOWNLOADED) {
downloadTemplateToStorage(template, ssHost);
}
}
}
}
@DB
public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) {
if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED)
|| (dnldStatus == Status.ABANDONED)) {
VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId());
synchronized (_listenerVolumeMap) {
_listenerVolumeMap.remove(volumeHost);
}
}
VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId());
Transaction txn = Transaction.currentTxn();
txn.start();
if (dnldStatus == Status.DOWNLOADED) {
// Create usage event
long size = -1;
if (volumeHost != null) {
size = volumeHost.getSize();
volume.setSize(size);
this._volumeDao.update(volume.getId(), volume);
} else {
s_logger.warn("Failed to get size for volume" + volume.getName());
}
String eventType = EventTypes.EVENT_VOLUME_UPLOAD;
if (volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
UsageEventUtils.publishUsageEvent(
eventType,
volume.getAccountId(),
host.getDataCenterId(),
volume.getId(),
volume.getName(),
null,
0l,
size,
volume.getClass().getName(),
volume.getUuid());
}
} else if (dnldStatus == Status.DOWNLOAD_ERROR
|| dnldStatus == Status.ABANDONED
|| dnldStatus == Status.UNKNOWN) {
// Decrement the volume and secondary storage space count
_resourceLimitMgr.decrementResourceCount(
volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume);
_resourceLimitMgr.recalculateResourceCount(
volume.getAccountId(),
volume.getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
}
txn.commit();
}
@Override
protected HashMap<String, Object> buildConfigParams(HostVO host) {
HashMap<String, Object> params = super.buildConfigParams(host);
Map<String, String> clusterDetails = _clusterDetailsDao.findDetails(host.getClusterId());
// Get zone wide traffic labels from guest traffic and public traffic
String guestTrafficLabel =
_netmgr.getDefaultGuestTrafficLabel(host.getDataCenterId(), HypervisorType.VMware);
String publicTrafficLabel =
_netmgr.getDefaultPublicTrafficLabel(host.getDataCenterId(), HypervisorType.VMware);
_readGlobalConfigParameters();
VirtualSwitchType defaultVirtualSwitchType = getDefaultVirtualSwitchType();
params.put(
"guestTrafficInfo",
getTrafficInfo(
TrafficType.Guest, guestTrafficLabel, clusterDetails, defaultVirtualSwitchType));
params.put(
"publicTrafficInfo",
getTrafficInfo(
TrafficType.Public, publicTrafficLabel, clusterDetails, defaultVirtualSwitchType));
return params;
}
@DB
public void handleDownloadEvent(HostVO host, VMTemplateVO template, Status dnldStatus) {
if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED)
|| (dnldStatus == Status.ABANDONED)) {
VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId());
synchronized (_listenerMap) {
_listenerMap.remove(vmTemplateHost);
}
}
VMTemplateHostVO vmTemplateHost =
_vmTemplateHostDao.findByHostTemplate(host.getId(), template.getId());
Transaction txn = Transaction.currentTxn();
txn.start();
if (dnldStatus == Status.DOWNLOADED) {
long size = -1;
if (vmTemplateHost != null) {
size = vmTemplateHost.getPhysicalSize();
template.setSize(size);
this._templateDao.update(template.getId(), template);
} else {
s_logger.warn("Failed to get size for template" + template.getName());
}
String eventType = EventTypes.EVENT_TEMPLATE_CREATE;
if ((template.getFormat()).equals(ImageFormat.ISO)) {
eventType = EventTypes.EVENT_ISO_CREATE;
}
if (template.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
UsageEventUtils.publishUsageEvent(
eventType,
template.getAccountId(),
host.getDataCenterId(),
template.getId(),
template.getName(),
null,
template.getSourceTemplateId(),
size,
template.getClass().getName(),
template.getUuid());
}
}
txn.commit();
}
@Override
public void processConnect(HostVO agent, StartupCommand cmd, boolean forRebalance) {
if ((cmd instanceof StartupStorageCommand)) {
StartupStorageCommand scmd = (StartupStorageCommand) cmd;
if (scmd.getResourceType() == Storage.StorageResourceType.SECONDARY_STORAGE) {
_ssVmMgr.generateSetupCommand(agent.getId());
return;
}
} else if (cmd instanceof StartupSecondaryStorageCommand) {
if (s_logger.isInfoEnabled()) {
s_logger.info("Received a host startup notification " + cmd);
}
_ssVmMgr.onAgentConnect(agent.getDataCenterId(), cmd);
_ssVmMgr.generateSetupCommand(agent.getId());
_ssVmMgr.generateFirewallConfiguration(agent.getId());
_ssVmMgr.generateVMSetupCommand(agent.getId());
return;
}
return;
}
@Override
@DB
public boolean processAnswers(long agentId, long seq, Answer[] answers) {
/*
* Do not collect Direct Network usage stats if the Traffic Monitor is not owned by this mgmt server
*/
HostVO host = _hostDao.findById(agentId);
if (host != null) {
if ((host.getManagementServerId() == null)
|| (mgmtSrvrId != host.getManagementServerId())) {
s_logger.warn(
"Not the owner. Not collecting Direct Network usage from TrafficMonitor : "
+ agentId);
return false;
}
} else {
s_logger.warn(
"Agent not found. Not collecting Direct Network usage from TrafficMonitor : "
+ agentId);
return false;
}
GlobalLock scanLock =
GlobalLock.getInternLock("direct.network.usage.collect" + host.getDataCenterId());
try {
if (scanLock.lock(10)) {
try {
return collectDirectNetworkUsage(host);
} finally {
scanLock.unlock();
}
}
} finally {
scanLock.releaseRef();
}
return false;
}
private String generateCopyUrl(HostVO sourceServer, VMTemplateHostVO srcTmpltHost) {
List<SecondaryStorageVmVO> ssVms =
_secStorageVmDao.getSecStorageVmListInStates(
SecondaryStorageVm.Role.templateProcessor,
sourceServer.getDataCenterId(),
State.Running);
if (ssVms.size() > 0) {
SecondaryStorageVmVO ssVm = ssVms.get(0);
if (ssVm.getPublicIpAddress() == null) {
s_logger.warn("A running secondary storage vm has a null public ip?");
return null;
}
return generateCopyUrl(
ssVm.getPublicIpAddress(), sourceServer.getParent(), srcTmpltHost.getInstallPath());
}
VMTemplateVO tmplt = _templateDao.findById(srcTmpltHost.getTemplateId());
HypervisorType hyperType = tmplt.getHypervisorType();
/*No secondary storage vm yet*/
if (hyperType != null && hyperType == HypervisorType.KVM) {
return "file://" + sourceServer.getParent() + "/" + srcTmpltHost.getInstallPath();
}
return null;
}
@Override
@DB
public boolean delete(TemplateProfile profile) {
boolean success = true;
VMTemplateVO template = profile.getTemplate();
Long zoneId = profile.getZoneId();
Long templateId = template.getId();
String zoneName;
List<HostVO> secondaryStorageHosts;
if (!template.isCrossZones() && zoneId != null) {
DataCenterVO zone = _dcDao.findById(zoneId);
zoneName = zone.getName();
secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(zoneId);
} else {
zoneName = "(all zones)";
secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInAllZones();
}
s_logger.debug(
"Attempting to mark template host refs for template: "
+ template.getName()
+ " as destroyed in zone: "
+ zoneName);
// Make sure the template is downloaded to all the necessary secondary storage hosts
for (HostVO secondaryStorageHost : secondaryStorageHosts) {
long hostId = secondaryStorageHost.getId();
List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
for (VMTemplateHostVO templateHostVO : templateHostVOs) {
if (templateHostVO.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
String errorMsg = "Please specify a template that is not currently being downloaded.";
s_logger.debug(
"Template: "
+ template.getName()
+ " is currently being downloaded to secondary storage host: "
+ secondaryStorageHost.getName()
+ "; cant' delete it.");
throw new CloudRuntimeException(errorMsg);
}
}
}
Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId());
String eventType = "";
if (template.getFormat().equals(ImageFormat.ISO)) {
eventType = EventTypes.EVENT_ISO_DELETE;
} else {
eventType = EventTypes.EVENT_TEMPLATE_DELETE;
}
// Iterate through all necessary secondary storage hosts and mark the template on each host as
// destroyed
for (HostVO secondaryStorageHost : secondaryStorageHosts) {
long hostId = secondaryStorageHost.getId();
long sZoneId = secondaryStorageHost.getDataCenterId();
List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
for (VMTemplateHostVO templateHostVO : templateHostVOs) {
VMTemplateHostVO lock = _tmpltHostDao.acquireInLockTable(templateHostVO.getId());
try {
if (lock == null) {
s_logger.debug(
"Failed to acquire lock when deleting templateHostVO with ID: "
+ templateHostVO.getId());
success = false;
break;
}
UsageEventVO usageEvent =
new UsageEventVO(eventType, account.getId(), sZoneId, templateId, null);
_usageEventDao.persist(usageEvent);
templateHostVO.setDestroyed(true);
_tmpltHostDao.update(templateHostVO.getId(), templateHostVO);
String installPath = templateHostVO.getInstallPath();
if (installPath != null) {
Answer answer =
_agentMgr.sendToSecStorage(
secondaryStorageHost,
new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(), installPath));
if (answer == null || !answer.getResult()) {
s_logger.debug(
"Failed to delete "
+ templateHostVO
+ " due to "
+ ((answer == null) ? "answer is null" : answer.getDetails()));
} else {
_tmpltHostDao.remove(templateHostVO.getId());
s_logger.debug("Deleted template at: " + installPath);
}
} else {
_tmpltHostDao.remove(templateHostVO.getId());
}
VMTemplateZoneVO templateZone = _tmpltZoneDao.findByZoneTemplate(sZoneId, templateId);
if (templateZone != null) {
_tmpltZoneDao.remove(templateZone.getId());
}
} finally {
if (lock != null) {
_tmpltHostDao.releaseFromLockTable(lock.getId());
}
}
}
if (!success) {
break;
}
}
s_logger.debug(
"Successfully marked template host refs for template: "
+ template.getName()
+ " as destroyed in zone: "
+ zoneName);
// If there are no more non-destroyed template host entries for this template, delete it
if (success && (_tmpltHostDao.listByTemplateId(templateId).size() == 0)) {
long accountId = template.getAccountId();
VMTemplateVO lock = _tmpltDao.acquireInLockTable(templateId);
try {
if (lock == null) {
s_logger.debug("Failed to acquire lock when deleting template with ID: " + templateId);
success = false;
} else if (_tmpltDao.remove(templateId)) {
// Decrement the number of templates
_resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template);
}
} finally {
if (lock != null) {
_tmpltDao.releaseFromLockTable(lock.getId());
}
}
s_logger.debug(
"Removed template: "
+ template.getName()
+ " because all of its template host refs were marked as destroyed.");
}
return success;
}
@Override
public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer)
throws StorageUnavailableException {
boolean downloadJobExists = false;
VMTemplateHostVO destTmpltHost = null;
VMTemplateHostVO srcTmpltHost = null;
srcTmpltHost = _vmTemplateHostDao.findByHostTemplate(sourceServer.getId(), template.getId());
if (srcTmpltHost == null) {
throw new InvalidParameterValueException(
"Template " + template.getName() + " not associated with " + sourceServer.getName());
}
String url = generateCopyUrl(sourceServer, srcTmpltHost);
if (url == null) {
s_logger.warn(
"Unable to start/resume copy of template "
+ template.getUniqueName()
+ " to "
+ destServer.getName()
+ ", no secondary storage vm in running state in source zone");
throw new CloudRuntimeException(
"No secondary VM in running state in zone " + sourceServer.getDataCenterId());
}
destTmpltHost = _vmTemplateHostDao.findByHostTemplate(destServer.getId(), template.getId());
if (destTmpltHost == null) {
destTmpltHost =
new VMTemplateHostVO(
destServer.getId(),
template.getId(),
new Date(),
0,
VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED,
null,
null,
"jobid0000",
null,
url);
destTmpltHost.setCopy(true);
destTmpltHost.setPhysicalSize(srcTmpltHost.getPhysicalSize());
_vmTemplateHostDao.persist(destTmpltHost);
} else if ((destTmpltHost.getJobId() != null) && (destTmpltHost.getJobId().length() > 2)) {
downloadJobExists = true;
}
Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
if (srcTmpltHost.getSize() > maxTemplateSizeInBytes) {
throw new CloudRuntimeException(
"Cant copy the template as the template's size "
+ srcTmpltHost.getSize()
+ " is greater than max.template.iso.size "
+ maxTemplateSizeInBytes);
}
if (destTmpltHost != null) {
start();
String sourceChecksum =
this.templateMgr.getChecksum(srcTmpltHost.getHostId(), srcTmpltHost.getInstallPath());
DownloadCommand dcmd =
new DownloadCommand(
destServer.getStorageUrl(),
url,
template,
TemplateConstants.DEFAULT_HTTP_AUTH_USER,
_copyAuthPasswd,
maxTemplateSizeInBytes);
if (downloadJobExists) {
dcmd =
new DownloadProgressCommand(dcmd, destTmpltHost.getJobId(), RequestType.GET_OR_RESTART);
}
dcmd.setProxy(getHttpProxy());
dcmd.setChecksum(
sourceChecksum); // We need to set the checksum as the source template might be a
// compressed url and have cksum for compressed image. Bug #10775
HostVO ssAhost = _ssvmMgr.pickSsvmHost(destServer);
if (ssAhost == null) {
s_logger.warn(
"There is no secondary storage VM for secondary storage host " + destServer.getName());
return false;
}
DownloadListener dl =
new DownloadListener(
ssAhost,
destServer,
template,
_timer,
_vmTemplateHostDao,
destTmpltHost.getId(),
this,
dcmd,
_templateDao,
_resourceLimitMgr,
_alertMgr,
_accountMgr);
if (downloadJobExists) {
dl.setCurrState(destTmpltHost.getDownloadState());
}
DownloadListener old = null;
synchronized (_listenerMap) {
old = _listenerMap.put(destTmpltHost, dl);
}
if (old != null) {
old.abandon();
}
try {
send(ssAhost.getId(), dcmd, dl);
return true;
} catch (AgentUnavailableException e) {
s_logger.warn(
"Unable to start /resume COPY of template "
+ template.getUniqueName()
+ " to "
+ destServer.getName(),
e);
dl.setDisconnected();
dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
e.printStackTrace();
}
}
return false;
}
@Override
public void run() {
try {
SearchCriteria<HostVO> sc = _hostDao.createSearchCriteria();
sc.addAnd("status", SearchCriteria.Op.EQ, Status.Up.toString());
sc.addAnd("type", SearchCriteria.Op.EQ, Host.Type.Storage.toString());
ConcurrentHashMap<Long, StorageStats> storageStats =
new ConcurrentHashMap<Long, StorageStats>();
List<HostVO> hosts = _hostDao.search(sc, null);
for (HostVO host : hosts) {
GetStorageStatsCommand command = new GetStorageStatsCommand(host.getGuid());
Answer answer = _agentMgr.easySend(host.getId(), command);
if (answer != null && answer.getResult()) {
storageStats.put(host.getId(), (StorageStats) answer);
}
}
sc = _hostDao.createSearchCriteria();
sc.addAnd("status", SearchCriteria.Op.EQ, Status.Up.toString());
sc.addAnd("type", SearchCriteria.Op.EQ, Host.Type.SecondaryStorage.toString());
hosts = _hostDao.search(sc, null);
for (HostVO host : hosts) {
GetStorageStatsCommand command = new GetStorageStatsCommand(host.getGuid());
Answer answer = _agentMgr.easySend(host.getId(), command);
if (answer != null && answer.getResult()) {
storageStats.put(host.getId(), (StorageStats) answer);
}
}
_storageStats = storageStats;
ConcurrentHashMap<Long, StorageStats> storagePoolStats =
new ConcurrentHashMap<Long, StorageStats>();
List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
for (StoragePoolVO pool : storagePools) {
GetStorageStatsCommand command =
new GetStorageStatsCommand(pool.getUuid(), pool.getPoolType(), pool.getPath());
Answer answer = _storageManager.sendToPool(pool, command);
if (answer != null && answer.getResult()) {
storagePoolStats.put(pool.getId(), (StorageStats) answer);
}
}
_storagePoolStats = storagePoolStats;
// a list to store the new capacity entries that will be committed once everything is
// calculated
List<CapacityVO> newCapacities = new ArrayList<CapacityVO>();
// Updating the storage entries and creating new ones if they dont exist.
Transaction txn = Transaction.open(Transaction.CLOUD_DB);
try {
if (s_logger.isTraceEnabled()) {
s_logger.trace("recalculating system storage capacity");
}
txn.start();
for (Long hostId : storageStats.keySet()) {
StorageStats stats = storageStats.get(hostId);
short capacityType = -1;
HostVO host = _hostDao.findById(hostId);
host.setTotalSize(stats.getCapacityBytes());
_hostDao.update(host.getId(), host);
SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
capacitySC.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, hostId);
capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, host.getDataCenterId());
if (Host.Type.SecondaryStorage.equals(host.getType())) {
capacityType = CapacityVO.CAPACITY_TYPE_SECONDARY_STORAGE;
} else if (Host.Type.Storage.equals(host.getType())) {
capacityType = CapacityVO.CAPACITY_TYPE_STORAGE;
}
if (-1 != capacityType) {
capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);
List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);
if (capacities.size() == 0) { // Create a new one
CapacityVO capacity =
new CapacityVO(
host.getId(),
host.getDataCenterId(),
host.getPodId(),
stats.getByteUsed(),
stats.getCapacityBytes(),
capacityType);
_capacityDao.persist(capacity);
} else { // Update if it already exists.
CapacityVO capacity = capacities.get(0);
capacity.setUsedCapacity(stats.getByteUsed());
capacity.setTotalCapacity(stats.getCapacityBytes());
_capacityDao.update(capacity.getId(), capacity);
}
}
} // End of for
txn.commit();
} catch (Exception ex) {
txn.rollback();
s_logger.error("Unable to start transaction for storage capacity update");
} finally {
txn.close();
}
for (Long poolId : storagePoolStats.keySet()) {
StorageStats stats = storagePoolStats.get(poolId);
StoragePoolVO pool = _storagePoolDao.findById(poolId);
if (pool == null) {
continue;
}
pool.setCapacityBytes(stats.getCapacityBytes());
long available = stats.getCapacityBytes() - stats.getByteUsed();
if (available < 0) {
available = 0;
}
pool.setAvailableBytes(available);
_storagePoolDao.update(pool.getId(), pool);
_storageManager.createCapacityEntry(pool, 0L);
}
} catch (Throwable t) {
s_logger.error("Error trying to retrieve storage stats", t);
}
}
private void downloadVolumeToStorage(
VolumeVO volume, HostVO sserver, String url, String checkSum, ImageFormat format) {
boolean downloadJobExists = false;
VolumeHostVO volumeHost = null;
volumeHost = _volumeHostDao.findByHostVolume(sserver.getId(), volume.getId());
if (volumeHost == null) {
volumeHost =
new VolumeHostVO(
sserver.getId(),
volume.getId(),
sserver.getDataCenterId(),
new Date(),
0,
VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED,
null,
null,
"jobid0000",
null,
url,
checkSum,
format);
_volumeHostDao.persist(volumeHost);
} else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) {
downloadJobExists = true;
}
Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes();
String secUrl = sserver.getStorageUrl();
if (volumeHost != null) {
start();
DownloadCommand dcmd =
new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format);
if (downloadJobExists) {
dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART);
dcmd.setResourceType(ResourceType.VOLUME);
}
dcmd.setProxy(getHttpProxy());
HostVO ssvm = _ssvmMgr.pickSsvmHost(sserver);
if (ssvm == null) {
s_logger.warn(
"There is no secondary storage VM for secondary storage host " + sserver.getName());
return;
}
DownloadListener dl =
new DownloadListener(
ssvm,
sserver,
volume,
_timer,
_volumeHostDao,
volumeHost.getId(),
this,
dcmd,
_volumeDao,
_storageMgr,
_resourceLimitMgr,
_alertMgr,
_accountMgr);
if (downloadJobExists) {
dl.setCurrState(volumeHost.getDownloadState());
}
DownloadListener old = null;
synchronized (_listenerVolumeMap) {
old = _listenerVolumeMap.put(volumeHost, dl);
}
if (old != null) {
old.abandon();
}
try {
send(ssvm.getId(), dcmd, dl);
} catch (AgentUnavailableException e) {
s_logger.warn(
"Unable to start /resume download of volume "
+ volume.getName()
+ " to "
+ sserver.getName(),
e);
dl.setDisconnected();
dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
}
}
}
protected Long restart(HaWorkVO work) {
List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
if (items.size() > 0) {
StringBuilder str =
new StringBuilder(
"Cancelling this work item because newer ones have been scheduled. Work Ids = [");
for (HaWorkVO item : items) {
str.append(item.getId()).append(", ");
}
str.delete(str.length() - 2, str.length()).append("]");
s_logger.info(str.toString());
return null;
}
items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
if (items.size() > 0) {
StringBuilder str =
new StringBuilder(
"Waiting because there's HA work being executed on an item currently. Work Ids =[");
for (HaWorkVO item : items) {
str.append(item.getId()).append(", ");
}
str.delete(str.length() - 2, str.length()).append("]");
s_logger.info(str.toString());
return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
}
long vmId = work.getInstanceId();
VMInstanceVO vm = _itMgr.findByIdAndType(work.getType(), work.getInstanceId());
if (vm == null) {
s_logger.info("Unable to find vm: " + vmId);
return null;
}
s_logger.info("HA on " + vm);
if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
s_logger.info(
"VM "
+ vm
+ " has been changed. Current State = "
+ vm.getState()
+ " Previous State = "
+ work.getPreviousState()
+ " last updated = "
+ vm.getUpdated()
+ " previous updated = "
+ work.getUpdateTime());
return null;
}
short alertType = AlertManager.ALERT_TYPE_USERVM;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
} else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_SSVM;
}
HostVO host = _hostDao.findById(work.getHostId());
boolean isHostRemoved = false;
if (host == null) {
host = _hostDao.findByIdIncludingRemoved(work.getHostId());
if (host != null) {
s_logger.debug(
"VM "
+ vm.toString()
+ " is now no longer on host "
+ work.getHostId()
+ " as the host is removed");
isHostRemoved = true;
}
}
DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
HostPodVO podVO = _podDao.findById(host.getPodId());
String hostDesc =
"name: "
+ host.getName()
+ "(id:"
+ host.getId()
+ "), availability zone: "
+ dcVO.getName()
+ ", pod: "
+ podVO.getName();
Boolean alive = null;
if (work.getStep() == Step.Investigating) {
if (!isHostRemoved) {
if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
return null;
}
Enumeration<Investigator> en = _investigators.enumeration();
Investigator investigator = null;
while (en.hasMoreElements()) {
investigator = en.nextElement();
alive = investigator.isVmAlive(vm, host);
s_logger.info(investigator.getName() + " found " + vm + "to be alive? " + alive);
if (alive != null) {
break;
}
}
boolean fenced = false;
if (alive == null) {
s_logger.debug("Fencing off VM that we don't know the state of");
Enumeration<FenceBuilder> enfb = _fenceBuilders.enumeration();
while (enfb.hasMoreElements()) {
FenceBuilder fb = enfb.nextElement();
Boolean result = fb.fenceOff(vm, host);
s_logger.info("Fencer " + fb.getName() + " returned " + result);
if (result != null && result) {
fenced = true;
break;
}
}
} else if (!alive) {
fenced = true;
} else {
s_logger.debug(
"VM " + vm.getHostName() + " is found to be alive by " + investigator.getName());
if (host.getStatus() == Status.Up) {
s_logger.info(vm + " is alive and host is up. No need to restart it.");
return null;
} else {
s_logger.debug("Rescheduling because the host is not up but the vm is alive");
return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
}
}
if (!fenced) {
s_logger.debug("We were unable to fence off the VM " + vm);
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}
try {
_itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
} catch (ResourceUnavailableException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (OperationTimedoutException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (ConcurrentOperationException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
}
work.setStep(Step.Scheduled);
_haDao.update(work.getId(), work);
} else {
s_logger.debug(
"How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
try {
_itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
} catch (ResourceUnavailableException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (OperationTimedoutException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (ConcurrentOperationException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
}
}
}
vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());
if (!_forceHA && !vm.isHaEnabled()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM is not HA enabled so we're done.");
}
return null; // VM doesn't require HA
}
if (!_storageMgr.canVmRestartOnAnotherServer(vm.getId())) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM can not restart on another server.");
}
return null;
}
if (work.getTimesTried() > _maxRetries) {
s_logger.warn("Retried to max times so deleting: " + vmId);
return null;
}
try {
VMInstanceVO started =
_itMgr.advanceStart(
vm,
new HashMap<VirtualMachineProfile.Param, Object>(),
_accountMgr.getSystemUser(),
_accountMgr.getSystemAccount());
if (started != null) {
s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
return null;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
}
} catch (final InsufficientCapacityException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (final ResourceUnavailableException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (ConcurrentOperationException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (OperationTimedoutException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
}
vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());
work.setUpdateTime(vm.getUpdated());
work.setPreviousState(vm.getState());
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}
@Override
public void handleTemplateSync(HostVO ssHost) {
if (ssHost == null) {
s_logger.warn("Huh? ssHost is null");
return;
}
long sserverId = ssHost.getId();
long zoneId = ssHost.getDataCenterId();
if (!(ssHost.getType() == Host.Type.SecondaryStorage
|| ssHost.getType() == Host.Type.LocalSecondaryStorage)) {
s_logger.warn("Huh? Agent id " + sserverId + " is not secondary storage host");
return;
}
Map<String, TemplateInfo> templateInfos = listTemplate(ssHost);
if (templateInfos == null) {
return;
}
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
List<VMTemplateVO> allTemplates = _templateDao.listAllInZone(zoneId);
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
if (rtngTmplts != null) {
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (!allTemplates.contains(rtngTmplt)) {
allTemplates.add(rtngTmplt);
}
}
}
if (defaultBuiltin != null) {
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (!allTemplates.contains(builtinTmplt)) {
allTemplates.add(builtinTmplt);
}
}
}
toBeDownloaded.addAll(allTemplates);
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(sserverId, tmplt.getId());
if (templateInfos.containsKey(uniqueName)) {
TemplateInfo tmpltInfo = templateInfos.remove(uniqueName);
toBeDownloaded.remove(tmplt);
if (tmpltHost != null) {
s_logger.info(
"Template Sync found " + tmplt.getName() + " already in the template host table");
if (tmpltHost.getDownloadState() != Status.DOWNLOADED) {
tmpltHost.setErrorString("");
}
if (tmpltInfo.isCorrupted()) {
tmpltHost.setDownloadState(Status.DOWNLOAD_ERROR);
String msg =
"Template "
+ tmplt.getName()
+ ":"
+ tmplt.getId()
+ " is corrupted on secondary storage "
+ tmpltHost.getId();
tmpltHost.setErrorString(msg);
s_logger.info("msg");
if (tmplt.getUrl() == null) {
msg =
"Private Template ("
+ tmplt
+ ") with install path "
+ tmpltInfo.getInstallPath()
+ "is corrupted, please check in secondary storage: "
+ tmpltHost.getHostId();
s_logger.warn(msg);
} else {
toBeDownloaded.add(tmplt);
}
} else {
tmpltHost.setDownloadPercent(100);
tmpltHost.setDownloadState(Status.DOWNLOADED);
tmpltHost.setInstallPath(tmpltInfo.getInstallPath());
tmpltHost.setSize(tmpltInfo.getSize());
tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize());
tmpltHost.setLastUpdated(new Date());
// Skipping limit checks for SYSTEM Account and for the templates created from volumes
// or snapshots
// which already got checked and incremented during createTemplate API call.
if (tmpltInfo.getSize() > 0
&& tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
&& tmplt.getUrl() != null) {
long accountId = tmplt.getAccountId();
try {
_resourceLimitMgr.checkResourceLimit(
_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(
_alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
ssHost.getDataCenterId(),
null,
e.getMessage(),
e.getMessage());
} finally {
_resourceLimitMgr.recalculateResourceCount(
accountId,
_accountMgr.getAccount(accountId).getDomainId(),
com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
}
}
}
_vmTemplateHostDao.update(tmpltHost.getId(), tmpltHost);
} else {
tmpltHost =
new VMTemplateHostVO(
sserverId,
tmplt.getId(),
new Date(),
100,
Status.DOWNLOADED,
null,
null,
null,
tmpltInfo.getInstallPath(),
tmplt.getUrl());
tmpltHost.setSize(tmpltInfo.getSize());
tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize());
_vmTemplateHostDao.persist(tmpltHost);
VMTemplateZoneVO tmpltZoneVO =
_vmTemplateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
if (tmpltZoneVO == null) {
tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date());
_vmTemplateZoneDao.persist(tmpltZoneVO);
} else {
tmpltZoneVO.setLastUpdated(new Date());
_vmTemplateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
}
}
continue;
}
if (tmpltHost != null && tmpltHost.getDownloadState() != Status.DOWNLOADED) {
s_logger.info(
"Template Sync did not find "
+ tmplt.getName()
+ " ready on server "
+ sserverId
+ ", will request download to start/resume shortly");
} else if (tmpltHost == null) {
s_logger.info(
"Template Sync did not find "
+ tmplt.getName()
+ " on the server "
+ sserverId
+ ", will request download shortly");
VMTemplateHostVO templtHost =
new VMTemplateHostVO(
sserverId,
tmplt.getId(),
new Date(),
0,
Status.NOT_DOWNLOADED,
null,
null,
null,
null,
tmplt.getUrl());
_vmTemplateHostDao.persist(templtHost);
VMTemplateZoneVO tmpltZoneVO = _vmTemplateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
if (tmpltZoneVO == null) {
tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date());
_vmTemplateZoneDao.persist(tmpltZoneVO);
} else {
tmpltZoneVO.setLastUpdated(new Date());
_vmTemplateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
}
}
}
if (toBeDownloaded.size() > 0) {
/* Only download templates whose hypervirsor type is in the zone */
List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
if (availHypers.isEmpty()) {
/*
* This is for cloudzone, local secondary storage resource
* started before cluster created
*/
availHypers.add(HypervisorType.KVM);
}
/* Baremetal need not to download any template */
availHypers.remove(HypervisorType.BareMetal);
availHypers.add(HypervisorType.None); // bug 9809: resume ISO
// download.
for (VMTemplateVO tmplt : toBeDownloaded) {
if (tmplt.getUrl() == null) { // If url is null we can't
// initiate the download
continue;
}
// if this is private template, and there is no record for this
// template in this sHost, skip
if (!tmplt.isPublicTemplate() && !tmplt.isFeatured()) {
VMTemplateHostVO tmpltHost =
_vmTemplateHostDao.findByHostTemplate(sserverId, tmplt.getId());
if (tmpltHost == null) {
continue;
}
}
if (availHypers.contains(tmplt.getHypervisorType())) {
if (_swiftMgr.isSwiftEnabled()) {
if (_swiftMgr.isTemplateInstalled(tmplt.getId())) {
continue;
}
}
s_logger.debug(
"Template " + tmplt.getName() + " needs to be downloaded to " + ssHost.getName());
downloadTemplateToStorage(tmplt, ssHost);
} else {
s_logger.info(
"Skipping download of template "
+ tmplt.getName()
+ " since we don't have any "
+ tmplt.getHypervisorType()
+ " hypervisors");
}
}
}
for (String uniqueName : templateInfos.keySet()) {
TemplateInfo tInfo = templateInfos.get(uniqueName);
List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(tInfo.getId());
// check if there is any Vm using this ISO.
if (userVmUsingIso == null || userVmUsingIso.isEmpty()) {
DeleteTemplateCommand dtCommand =
new DeleteTemplateCommand(ssHost.getStorageUrl(), tInfo.getInstallPath());
try {
_agentMgr.sendToSecStorage(ssHost, dtCommand, null);
} catch (AgentUnavailableException e) {
String err =
"Failed to delete "
+ tInfo.getTemplateName()
+ " on secondary storage "
+ sserverId
+ " which isn't in the database";
s_logger.error(err);
return;
}
String description =
"Deleted template "
+ tInfo.getTemplateName()
+ " on secondary storage "
+ sserverId
+ " since it isn't in the database";
s_logger.info(description);
}
}
}
private boolean collectDirectNetworkUsage(HostVO host) {
s_logger.debug("Direct Network Usage stats collector is running...");
long zoneId = host.getDataCenterId();
DetailVO lastCollectDetail = _detailsDao.findDetail(host.getId(), "last_collection");
if (lastCollectDetail == null) {
s_logger.warn(
"Last collection time not available. Skipping direct usage collection for Traffic Monitor: "
+ host.getId());
return false;
}
Date lastCollection = new Date(new Long(lastCollectDetail.getValue()));
// Get list of IPs currently allocated
List<IPAddressVO> allocatedIps = listAllocatedDirectIps(zoneId);
Calendar rightNow = Calendar.getInstance();
// Allow 2 hours for traffic sentinel to populate historical traffic
// This coule be made configurable
rightNow.add(Calendar.HOUR_OF_DAY, -2);
Date now = rightNow.getTime();
if (lastCollection.after(now)) {
s_logger.debug(
"Current time is less than 2 hours after last collection time : "
+ lastCollection.toString()
+ ". Skipping direct network usage collection");
return false;
}
// Get IP Assign/Release events from lastCollection time till now
List<UsageEventVO> IpEvents = _eventDao.listDirectIpEvents(lastCollection, now, zoneId);
Map<String, Date> ipAssigment = new HashMap<String, Date>();
List<UsageIPAddressVO> IpPartialUsage =
new ArrayList<
UsageIPAddressVO>(); // Ips which were allocated only for the part of collection
// duration
List<UsageIPAddressVO> fullDurationIpUsage =
new ArrayList<
UsageIPAddressVO>(); // Ips which were allocated only for the entire collection
// duration
// Use UsageEvents to track the IP assignment
// Add them to IpUsage list with account_id , ip_address, alloc_date, release_date
for (UsageEventVO IpEvent : IpEvents) {
String address = IpEvent.getResourceName();
if (EventTypes.EVENT_NET_IP_ASSIGN.equals(IpEvent.getType())) {
ipAssigment.put(address, IpEvent.getCreateDate());
} else if (EventTypes.EVENT_NET_IP_RELEASE.equals(IpEvent.getType())) {
if (ipAssigment.containsKey(address)) {
Date assigned = ipAssigment.get(address);
ipAssigment.remove(address);
IpPartialUsage.add(
new UsageIPAddressVO(
IpEvent.getAccountId(), address, assigned, IpEvent.getCreateDate()));
} else {
// Ip was assigned prior to lastCollection Date
IpPartialUsage.add(
new UsageIPAddressVO(
IpEvent.getAccountId(), address, lastCollection, IpEvent.getCreateDate()));
}
}
}
List<String> IpList = new ArrayList<String>();
for (IPAddressVO ip : allocatedIps) {
if (ip.getAllocatedToAccountId() == AccountVO.ACCOUNT_ID_SYSTEM) {
// Ignore usage for system account
continue;
}
String address = (ip.getAddress()).toString();
if (ipAssigment.containsKey(address)) {
// Ip was assigned during the current period but not release till Date now
IpPartialUsage.add(
new UsageIPAddressVO(
ip.getAllocatedToAccountId(), address, ipAssigment.get(address), now));
} else {
// Ip was not assigned or released during current period. Consider entire duration for
// usage calculation (lastCollection to now)
fullDurationIpUsage.add(
new UsageIPAddressVO(ip.getAllocatedToAccountId(), address, lastCollection, now));
// Store just the Ips to send the list as part of DirectNetworkUsageCommand
IpList.add(address);
}
}
List<UserStatisticsVO> collectedStats = new ArrayList<UserStatisticsVO>();
// Get usage for Ips which were assigned for the entire duration
if (fullDurationIpUsage.size() > 0) {
DirectNetworkUsageCommand cmd =
new DirectNetworkUsageCommand(IpList, lastCollection, now, _TSinclZones, _TSexclZones);
DirectNetworkUsageAnswer answer =
(DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
String details = (answer != null) ? answer.getDetails() : "details unavailable";
String msg =
"Unable to get network usage stats from "
+ host.getId()
+ " due to: "
+ details
+ ".";
s_logger.error(msg);
return false;
} else {
for (UsageIPAddressVO usageIp : fullDurationIpUsage) {
String publicIp = usageIp.getAddress();
long[] bytesSentRcvd = answer.get(publicIp);
Long bytesSent = bytesSentRcvd[0];
Long bytesRcvd = bytesSentRcvd[1];
if (bytesSent == null || bytesRcvd == null) {
s_logger.debug("Incorrect bytes for IP: " + publicIp);
continue;
}
if (bytesSent == 0L && bytesRcvd == 0L) {
s_logger.trace("Ignore zero bytes for IP: " + publicIp);
continue;
}
UserStatisticsVO stats =
new UserStatisticsVO(usageIp.getAccountId(), zoneId, null, null, null, null);
stats.setCurrentBytesSent(bytesSent);
stats.setCurrentBytesReceived(bytesRcvd);
collectedStats.add(stats);
}
}
}
// Get usage for Ips which were assigned for part of the duration period
for (UsageIPAddressVO usageIp : IpPartialUsage) {
IpList = new ArrayList<String>();
IpList.add(usageIp.getAddress());
DirectNetworkUsageCommand cmd =
new DirectNetworkUsageCommand(
IpList, usageIp.getAssigned(), usageIp.getReleased(), _TSinclZones, _TSexclZones);
DirectNetworkUsageAnswer answer =
(DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
String details = (answer != null) ? answer.getDetails() : "details unavailable";
String msg =
"Unable to get network usage stats from "
+ host.getId()
+ " due to: "
+ details
+ ".";
s_logger.error(msg);
return false;
} else {
String publicIp = usageIp.getAddress();
long[] bytesSentRcvd = answer.get(publicIp);
Long bytesSent = bytesSentRcvd[0];
Long bytesRcvd = bytesSentRcvd[1];
if (bytesSent == null || bytesRcvd == null) {
s_logger.debug("Incorrect bytes for IP: " + publicIp);
continue;
}
if (bytesSent == 0L && bytesRcvd == 0L) {
s_logger.trace("Ignore zero bytes for IP: " + publicIp);
continue;
}
UserStatisticsVO stats =
new UserStatisticsVO(usageIp.getAccountId(), zoneId, null, null, null, null);
stats.setCurrentBytesSent(bytesSent);
stats.setCurrentBytesReceived(bytesRcvd);
collectedStats.add(stats);
}
}
if (collectedStats.size() == 0) {
s_logger.debug("No new direct network stats. No need to persist");
return false;
}
// Persist all the stats and last_collection time in a single transaction
Transaction txn = Transaction.open(Transaction.CLOUD_DB);
try {
txn.start();
for (UserStatisticsVO stat : collectedStats) {
UserStatisticsVO stats =
_statsDao.lock(
stat.getAccountId(),
stat.getDataCenterId(),
0L,
null,
host.getId(),
"DirectNetwork");
if (stats == null) {
stats =
new UserStatisticsVO(
stat.getAccountId(), zoneId, null, host.getId(), "DirectNetwork", 0L);
stats.setCurrentBytesSent(stat.getCurrentBytesSent());
stats.setCurrentBytesReceived(stat.getCurrentBytesReceived());
_statsDao.persist(stats);
} else {
stats.setCurrentBytesSent(stats.getCurrentBytesSent() + stat.getCurrentBytesSent());
stats.setCurrentBytesReceived(
stats.getCurrentBytesReceived() + stat.getCurrentBytesReceived());
_statsDao.update(stats.getId(), stats);
}
}
lastCollectDetail.setValue("" + now.getTime());
_detailsDao.update(lastCollectDetail.getId(), lastCollectDetail);
txn.commit();
} finally {
txn.close();
}
return true;
}
public Long migrate(final HaWorkVO work) {
final long vmId = work.getInstanceId();
final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
VMInstanceVO vm = mgr.get(vmId);
if (vm == null || vm.getRemoved() != null) {
s_logger.debug("Unable to find the vm " + vmId);
return null;
}
s_logger.info("Migrating vm: " + vm.toString());
if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
s_logger.info("VM is not longer running on the current hostId");
return null;
}
short alertType = AlertManager.ALERT_TYPE_USERVM_MIGRATE;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
}
HostVO fromHost = _hostDao.findById(vm.getHostId());
String fromHostName = ((fromHost == null) ? "unknown" : fromHost.getName());
HostVO toHost = null;
if (work.getStep() == Step.Scheduled) {
if (vm.getState() != State.Running) {
s_logger.info(
"VM's state is not ready for migration. "
+ vm.toString()
+ " State is "
+ vm.getState().toString());
return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
}
DataCenterVO dcVO = _dcDao.findById(fromHost.getDataCenterId());
HostPodVO podVO = _podDao.findById(fromHost.getPodId());
try {
toHost = mgr.prepareForMigration(vm);
if (toHost == null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Unable to find a host for migrating vm " + vmId);
}
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Unable to find a suitable host");
}
} catch (final InsufficientCapacityException e) {
s_logger.warn("Unable to mgirate due to insufficient capacity " + vm.toString());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Insufficient capacity");
} catch (final StorageUnavailableException e) {
s_logger.warn("Storage is unavailable: " + vm.toString());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Storage is gone.");
}
if (toHost == null) {
_agentMgr.maintenanceFailed(vm.getHostId());
return null;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Migrating from " + work.getHostId() + " to " + toHost.getId());
}
work.setStep(Step.Migrating);
work.setHostId(toHost.getId());
_haDao.update(work.getId(), work);
}
if (work.getStep() == Step.Migrating) {
vm = mgr.get(vmId); // let's see if anything has changed.
boolean migrated = false;
if (vm == null
|| vm.getRemoved() != null
|| vm.getHostId() == null
|| !_itMgr.stateTransitTo(vm, Event.MigrationRequested, vm.getHostId())) {
s_logger.info("Migration cancelled because state has changed: " + vm.toString());
} else {
try {
boolean isWindows =
_guestOSCategoryDao
.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId())
.getName()
.equalsIgnoreCase("Windows");
MigrateCommand cmd =
new MigrateCommand(vm.getInstanceName(), toHost.getPrivateIpAddress(), isWindows);
Answer answer = _agentMgr.send(fromHost.getId(), cmd);
if (answer != null && answer.getResult()) {
migrated = true;
_storageMgr.unshare(vm, fromHost);
work.setStep(Step.Investigating);
_haDao.update(work.getId(), work);
}
} catch (final AgentUnavailableException e) {
s_logger.debug("host became unavailable");
} catch (final OperationTimedoutException e) {
s_logger.debug("operation timed out");
if (e.isActive()) {
scheduleRestart(vm, true);
}
}
}
if (!migrated) {
s_logger.info("Migration was unsuccessful. Cleaning up: " + vm.toString());
DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
HostPodVO podVO = _podDao.findById(vm.getPodId());
_alertMgr.sendAlert(
alertType,
fromHost.getDataCenterId(),
fromHost.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHost.getName()
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Migrate Command failed. Please check logs.");
_itMgr.stateTransitTo(vm, Event.MigrationFailedOnSource, toHost.getId());
_agentMgr.maintenanceFailed(vm.getHostId());
Command cleanup = mgr.cleanup(vm, null);
_agentMgr.easySend(toHost.getId(), cleanup);
_storageMgr.unshare(vm, toHost);
return null;
}
}
if (toHost == null) {
toHost = _hostDao.findById(work.getHostId());
}
DataCenterVO dcVO = _dcDao.findById(toHost.getDataCenterId());
HostPodVO podVO = _podDao.findById(toHost.getPodId());
try {
if (!mgr.completeMigration(vm, toHost)) {
_alertMgr.sendAlert(
alertType,
toHost.getDataCenterId(),
toHost.getPodId(),
"Unable to migrate "
+ vmId
+ " to host "
+ toHost.getName()
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Migration not completed");
s_logger.warn("Unable to complete migration: " + vm.toString());
} else {
s_logger.info("Migration is complete: " + vm.toString());
}
return null;
} catch (final AgentUnavailableException e) {
s_logger.warn("Agent is unavailable for " + vm.toString());
} catch (final OperationTimedoutException e) {
s_logger.warn("Operation timed outfor " + vm.toString());
}
_itMgr.stateTransitTo(vm, Event.MigrationFailedOnDest, toHost.getId());
return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
}
@Override
public boolean generateSetupCommand(Long ssHostId) {
HostVO cssHost = _hostDao.findById(ssHostId);
Long zoneId = cssHost.getDataCenterId();
if (cssHost.getType() == Host.Type.SecondaryStorageVM) {
SecondaryStorageVmVO secStorageVm = _secStorageVmDao.findByInstanceName(cssHost.getName());
if (secStorageVm == null) {
s_logger.warn("secondary storage VM " + cssHost.getName() + " doesn't exist");
return false;
}
List<HostVO> ssHosts = _hostDao.listSecondaryStorageHosts(zoneId);
for (HostVO ssHost : ssHosts) {
String secUrl = ssHost.getStorageUrl();
SecStorageSetupCommand setupCmd = new SecStorageSetupCommand(secUrl);
Answer answer = _agentMgr.easySend(ssHostId, setupCmd);
if (answer != null && answer.getResult()) {
SecStorageSetupAnswer an = (SecStorageSetupAnswer) answer;
ssHost.setParent(an.get_dir());
_hostDao.update(ssHost.getId(), ssHost);
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Successfully programmed secondary storage "
+ ssHost.getName()
+ " in secondary storage VM "
+ secStorageVm.getInstanceName());
}
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Successfully programmed secondary storage "
+ ssHost.getName()
+ " in secondary storage VM "
+ secStorageVm.getInstanceName());
}
return false;
}
}
} else if (cssHost.getType() == Host.Type.SecondaryStorage) {
List<SecondaryStorageVmVO> alreadyRunning =
_secStorageVmDao.getSecStorageVmListInStates(
SecondaryStorageVm.Role.templateProcessor, zoneId, State.Running);
String secUrl = cssHost.getStorageUrl();
SecStorageSetupCommand setupCmd = new SecStorageSetupCommand(secUrl);
for (SecondaryStorageVmVO ssVm : alreadyRunning) {
HostVO host = _hostDao.findByName(ssVm.getInstanceName());
Answer answer = _agentMgr.easySend(host.getId(), setupCmd);
if (answer != null && answer.getResult()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Successfully programmed secondary storage "
+ host.getName()
+ " in secondary storage VM "
+ ssVm.getInstanceName());
}
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Successfully programmed secondary storage "
+ host.getName()
+ " in secondary storage VM "
+ ssVm.getInstanceName());
}
return false;
}
}
}
return true;
}
protected void loadResource(Long hostId) {
HostVO host = hostDao.findById(hostId);
Map<String, Object> params = new HashMap<String, Object>();
params.put("guid", host.getGuid());
params.put("ipaddress", host.getPrivateIpAddress());
params.put("username", "root");
params.put("password", "password");
params.put("zone", String.valueOf(host.getDataCenterId()));
params.put("pod", String.valueOf(host.getPodId()));
ServerResource resource = null;
if (host.getHypervisorType() == HypervisorType.XenServer) {
resource = new XcpOssResource();
try {
resource.configure(host.getName(), params);
} catch (ConfigurationException e) {
logger.debug("Failed to load resource:" + e.toString());
}
} else if (host.getHypervisorType() == HypervisorType.KVM) {
resource = new LibvirtComputingResource();
try {
params.put("public.network.device", "cloudbr0");
params.put("private.network.device", "cloudbr0");
resource.configure(host.getName(), params);
} catch (ConfigurationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else if (host.getHypervisorType() == HypervisorType.VMware) {
ClusterVO cluster = clusterDao.findById(host.getClusterId());
String url = clusterDetailsDao.findDetail(cluster.getId(), "url").getValue();
URI uri;
try {
uri = new URI(url);
String userName = clusterDetailsDao.findDetail(cluster.getId(), "username").getValue();
String password = clusterDetailsDao.findDetail(cluster.getId(), "password").getValue();
VmwareServerDiscoverer discover = new VmwareServerDiscoverer();
Map<? extends ServerResource, Map<String, String>> resources =
discover.find(
host.getDataCenterId(),
host.getPodId(),
host.getClusterId(),
uri,
userName,
password,
null);
for (Map.Entry<? extends ServerResource, Map<String, String>> entry :
resources.entrySet()) {
resource = entry.getKey();
}
if (resource == null) {
throw new CloudRuntimeException("can't find resource");
}
} catch (DiscoveryException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
hostResourcesMap.put(hostId, resource);
HostEnvironment env = new HostEnvironment();
SetupCommand cmd = new SetupCommand(env);
cmd.setNeedSetup(true);
resource.executeRequest(cmd);
}
@DB
@Override
public void updateCapacityForHost(HostVO host) {
// prepare the service offerings
List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
for (ServiceOfferingVO offering : offerings) {
offeringsMap.put(offering.getId(), offering);
}
long usedCpu = 0;
long usedMemory = 0;
long reservedMemory = 0;
long reservedCpu = 0;
List<VMInstanceVO> vms = _vmDao.listUpByHostId(host.getId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Found " + vms.size() + " VMs on host " + host.getId());
}
for (VMInstanceVO vm : vms) {
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
usedMemory += so.getRamSize() * 1024L * 1024L;
usedCpu += so.getCpu() * so.getSpeed();
}
List<VMInstanceVO> vmsByLastHostId = _vmDao.listByLastHostId(host.getId());
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Found " + vmsByLastHostId.size() + " VM, not running on host " + host.getId());
}
for (VMInstanceVO vm : vmsByLastHostId) {
long secondsSinceLastUpdate =
(DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
reservedMemory += so.getRamSize() * 1024L * 1024L;
reservedCpu += so.getCpu() * so.getSpeed();
}
}
CapacityVO cpuCap = _capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_CPU);
CapacityVO memCap =
_capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_MEMORY);
if (cpuCap != null && memCap != null) {
if (cpuCap.getUsedCapacity() == usedCpu && cpuCap.getReservedCapacity() == reservedCpu) {
s_logger.debug(
"No need to calibrate cpu capacity, host:"
+ host.getId()
+ " usedCpu: "
+ cpuCap.getUsedCapacity()
+ " reservedCpu: "
+ cpuCap.getReservedCapacity());
} else if (cpuCap.getReservedCapacity() != reservedCpu) {
s_logger.debug(
"Calibrate reserved cpu for host: "
+ host.getId()
+ " old reservedCpu:"
+ cpuCap.getReservedCapacity()
+ " new reservedCpu:"
+ reservedCpu);
cpuCap.setReservedCapacity(reservedCpu);
} else if (cpuCap.getUsedCapacity() != usedCpu) {
s_logger.debug(
"Calibrate used cpu for host: "
+ host.getId()
+ " old usedCpu:"
+ cpuCap.getUsedCapacity()
+ " new usedCpu:"
+ usedCpu);
cpuCap.setUsedCapacity(usedCpu);
}
if (memCap.getUsedCapacity() == usedMemory
&& memCap.getReservedCapacity() == reservedMemory) {
s_logger.debug(
"No need to calibrate memory capacity, host:"
+ host.getId()
+ " usedMem: "
+ memCap.getUsedCapacity()
+ " reservedMem: "
+ memCap.getReservedCapacity());
} else if (memCap.getReservedCapacity() != reservedMemory) {
s_logger.debug(
"Calibrate reserved memory for host: "
+ host.getId()
+ " old reservedMem:"
+ memCap.getReservedCapacity()
+ " new reservedMem:"
+ reservedMemory);
memCap.setReservedCapacity(reservedMemory);
} else if (memCap.getUsedCapacity() != usedMemory) {
/*
* Didn't calibrate for used memory, because VMs can be in state(starting/migrating) that I don't know on which host they are
* allocated
*/
s_logger.debug(
"Calibrate used memory for host: "
+ host.getId()
+ " old usedMem: "
+ memCap.getUsedCapacity()
+ " new usedMem: "
+ usedMemory);
memCap.setUsedCapacity(usedMemory);
}
try {
_capacityDao.update(cpuCap.getId(), cpuCap);
_capacityDao.update(memCap.getId(), memCap);
} catch (Exception e) {
s_logger.error(
"Caught exception while updating cpu/memory capacity for the host " + host.getId(), e);
}
} else {
Transaction txn = Transaction.currentTxn();
CapacityState capacityState =
_configMgr.findClusterAllocationState(ApiDBUtils.findClusterById(host.getClusterId()))
== AllocationState.Disabled
? CapacityState.Disabled
: CapacityState.Enabled;
txn.start();
CapacityVO capacity =
new CapacityVO(
host.getId(),
host.getDataCenterId(),
host.getPodId(),
host.getClusterId(),
usedMemory,
host.getTotalMemory(),
CapacityVO.CAPACITY_TYPE_MEMORY);
capacity.setReservedCapacity(reservedMemory);
capacity.setCapacityState(capacityState);
_capacityDao.persist(capacity);
capacity =
new CapacityVO(
host.getId(),
host.getDataCenterId(),
host.getPodId(),
host.getClusterId(),
usedCpu,
(long) (host.getCpus().longValue() * host.getSpeed().longValue()),
CapacityVO.CAPACITY_TYPE_CPU);
capacity.setReservedCapacity(reservedCpu);
capacity.setCapacityState(capacityState);
_capacityDao.persist(capacity);
txn.commit();
}
}
@Test(priority = -1)
public void setUp() {
ComponentContext.initComponentsLifeCycle();
host = hostDao.findByGuid(this.getHostGuid());
if (host != null) {
dcId = host.getDataCenterId();
clusterId = host.getClusterId();
podId = host.getPodId();
return;
}
// create data center
DataCenterVO dc =
new DataCenterVO(
UUID.randomUUID().toString(),
"test",
"8.8.8.8",
null,
"10.0.0.1",
null,
"10.0.0.1/24",
null,
null,
NetworkType.Basic,
null,
null,
true,
true,
null,
null);
dc = dcDao.persist(dc);
dcId = dc.getId();
// create pod
HostPodVO pod =
new HostPodVO(
UUID.randomUUID().toString(),
dc.getId(),
this.getHostGateway(),
this.getHostCidr(),
8,
"test");
pod = podDao.persist(pod);
podId = pod.getId();
// create xen cluster
ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
cluster.setHypervisorType(HypervisorType.XenServer.toString());
cluster.setClusterType(ClusterType.CloudManaged);
cluster.setManagedState(ManagedState.Managed);
cluster = clusterDao.persist(cluster);
clusterId = cluster.getId();
// create xen host
host = new HostVO(this.getHostGuid());
host.setName("devcloud xen host");
host.setType(Host.Type.Routing);
host.setPrivateIpAddress(this.getHostIp());
host.setDataCenterId(dc.getId());
host.setVersion("6.0.1");
host.setAvailable(true);
host.setSetup(true);
host.setPodId(podId);
host.setLastPinged(0);
host.setResourceState(ResourceState.Enabled);
host.setHypervisorType(HypervisorType.XenServer);
host.setClusterId(cluster.getId());
host = hostDao.persist(host);
imageStore = new ImageStoreVO();
imageStore.setName("test");
imageStore.setDataCenterId(dcId);
imageStore.setProviderName("CloudStack ImageStore Provider");
imageStore.setRole(DataStoreRole.Image);
imageStore.setUrl(this.getSecondaryStorage());
imageStore.setUuid(UUID.randomUUID().toString());
imageStore = imageStoreDao.persist(imageStore);
}
@Test(priority = -1)
public void setUp() {
ComponentContext.initComponentsLifeCycle();
host = hostDao.findByGuid(this.getHostGuid());
if (host != null) {
dcId = host.getDataCenterId();
clusterId = host.getClusterId();
podId = host.getPodId();
imageStore = this.imageStoreDao.findByName(imageStoreName);
} else {
// create data center
DataCenterVO dc =
new DataCenterVO(
UUID.randomUUID().toString(),
"test",
"8.8.8.8",
null,
"10.0.0.1",
null,
"10.0.0.1/24",
null,
null,
NetworkType.Basic,
null,
null,
true,
true,
null,
null);
dc = dcDao.persist(dc);
dcId = dc.getId();
// create pod
HostPodVO pod =
new HostPodVO(
UUID.randomUUID().toString(),
dc.getId(),
this.getHostGateway(),
this.getHostCidr(),
8,
"test");
pod = podDao.persist(pod);
podId = pod.getId();
// create xen cluster
ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
cluster.setHypervisorType(HypervisorType.VMware.toString());
cluster.setClusterType(ClusterType.ExternalManaged);
cluster.setManagedState(ManagedState.Managed);
cluster = clusterDao.persist(cluster);
clusterId = cluster.getId();
// setup vcenter
ClusterDetailsVO clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "url", null);
this.clusterDetailsDao.persist(clusterDetailVO);
clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "username", null);
this.clusterDetailsDao.persist(clusterDetailVO);
clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "password", null);
this.clusterDetailsDao.persist(clusterDetailVO);
// create xen host
host = new HostVO(this.getHostGuid());
host.setName("devcloud vmware host");
host.setType(Host.Type.Routing);
host.setPrivateIpAddress(this.getHostIp());
host.setDataCenterId(dc.getId());
host.setVersion("6.0.1");
host.setAvailable(true);
host.setSetup(true);
host.setPodId(podId);
host.setLastPinged(0);
host.setResourceState(ResourceState.Enabled);
host.setHypervisorType(HypervisorType.VMware);
host.setClusterId(cluster.getId());
host = hostDao.persist(host);
imageStore = new ImageStoreVO();
imageStore.setName(imageStoreName);
imageStore.setDataCenterId(dcId);
imageStore.setProviderName("CloudStack ImageStore Provider");
imageStore.setRole(DataStoreRole.Image);
imageStore.setUrl(this.getSecondaryStorage());
imageStore.setUuid(UUID.randomUUID().toString());
imageStore.setProtocol("nfs");
imageStore = imageStoreDao.persist(imageStore);
}
image = new VMTemplateVO();
image.setTemplateType(TemplateType.USER);
image.setUrl(this.getTemplateUrl());
image.setUniqueName(UUID.randomUUID().toString());
image.setName(UUID.randomUUID().toString());
image.setPublicTemplate(true);
image.setFeatured(true);
image.setRequiresHvm(true);
image.setBits(64);
image.setFormat(Storage.ImageFormat.VHD);
image.setEnablePassword(true);
image.setEnableSshKey(true);
image.setGuestOSId(1);
image.setBootable(true);
image.setPrepopulate(true);
image.setCrossZones(true);
image.setExtractable(true);
image = imageDataDao.persist(image);
/*
* TemplateDataStoreVO templateStore = new TemplateDataStoreVO();
*
* templateStore.setDataStoreId(imageStore.getId());
* templateStore.setDownloadPercent(100);
* templateStore.setDownloadState(Status.DOWNLOADED);
* templateStore.setDownloadUrl(imageStore.getUrl());
* templateStore.setInstallPath(this.getImageInstallPath());
* templateStore.setTemplateId(image.getId());
* templateStoreDao.persist(templateStore);
*/
DataStore store = this.dataStoreMgr.getDataStore(imageStore.getId(), DataStoreRole.Image);
TemplateInfo template = templateFactory.getTemplate(image.getId(), DataStoreRole.Image);
DataObject templateOnStore = store.create(template);
TemplateObjectTO to = new TemplateObjectTO();
to.setPath(this.getImageInstallPath());
CopyCmdAnswer answer = new CopyCmdAnswer(to);
templateOnStore.processEvent(Event.CreateOnlyRequested);
templateOnStore.processEvent(Event.OperationSuccessed, answer);
}
@Override
public boolean generateFirewallConfiguration(Long ssAHostId) {
if (ssAHostId == null) {
return true;
}
HostVO ssAHost = _hostDao.findById(ssAHostId);
Long zoneId = ssAHost.getDataCenterId();
SecondaryStorageVmVO thisSecStorageVm = _secStorageVmDao.findByInstanceName(ssAHost.getName());
if (thisSecStorageVm == null) {
s_logger.warn("secondary storage VM " + ssAHost.getName() + " doesn't exist");
return false;
}
List<SecondaryStorageVmVO> alreadyRunning =
_secStorageVmDao.getSecStorageVmListInStates(
SecondaryStorageVm.Role.templateProcessor,
State.Running,
State.Migrating,
State.Starting);
String copyPort =
_useSSlCopy ? "443" : Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
SecStorageFirewallCfgCommand cpc = new SecStorageFirewallCfgCommand();
SecStorageFirewallCfgCommand thiscpc = new SecStorageFirewallCfgCommand();
thiscpc.addPortConfig(
thisSecStorageVm.getPublicIpAddress(),
copyPort,
true,
TemplateConstants.DEFAULT_TMPLT_COPY_INTF);
for (SecondaryStorageVmVO ssVm : alreadyRunning) {
if (ssVm.getDataCenterIdToDeployIn() == zoneId) {
continue;
}
if (ssVm.getPublicIpAddress() != null) {
cpc.addPortConfig(
ssVm.getPublicIpAddress(), copyPort, true, TemplateConstants.DEFAULT_TMPLT_COPY_INTF);
}
if (ssVm.getState() != State.Running) {
continue;
}
String instanceName = ssVm.getInstanceName();
HostVO host = _hostDao.findByName(instanceName);
if (host == null) {
continue;
}
Answer answer = _agentMgr.easySend(host.getId(), thiscpc);
if (answer != null && answer.getResult()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Successfully programmed firewall rules into " + ssVm.getHostName());
}
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"failed to program firewall rules into secondary storage vm : " + ssVm.getHostName());
}
return false;
}
}
Answer answer = _agentMgr.easySend(ssAHostId, cpc);
if (answer != null && answer.getResult()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Successfully programmed firewall rules into " + thisSecStorageVm.getHostName());
}
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"failed to program firewall rules into secondary storage vm : "
+ thisSecStorageVm.getHostName());
}
return false;
}
return true;
}
// TODO: Get rid of this case once we've determined that the capacity listeners above have all the
// changes
// create capacity entries if none exist for this server
private void createCapacityEntry(StartupCommand startup, HostVO server) {
SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
capacitySC.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
capacitySC.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
if (startup instanceof StartupRoutingCommand) {
SearchCriteria<CapacityVO> capacityCPU = _capacityDao.createSearchCriteria();
capacityCPU.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
capacityCPU.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
capacityCPU.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
capacityCPU.addAnd("capacityType", SearchCriteria.Op.EQ, CapacityVO.CAPACITY_TYPE_CPU);
List<CapacityVO> capacityVOCpus = _capacityDao.search(capacitySC, null);
Float cpuovercommitratio =
Float.parseFloat(
_clusterDetailsDao
.findDetail(server.getClusterId(), "cpuOvercommitRatio")
.getValue());
Float memoryOvercommitRatio =
Float.parseFloat(
_clusterDetailsDao
.findDetail(server.getClusterId(), "memoryOvercommitRatio")
.getValue());
if (capacityVOCpus != null && !capacityVOCpus.isEmpty()) {
CapacityVO CapacityVOCpu = capacityVOCpus.get(0);
long newTotalCpu =
(long)
(server.getCpus().longValue() * server.getSpeed().longValue() * cpuovercommitratio);
if ((CapacityVOCpu.getTotalCapacity() <= newTotalCpu)
|| ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity())
<= newTotalCpu)) {
CapacityVOCpu.setTotalCapacity(newTotalCpu);
} else if ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity()
> newTotalCpu)
&& (CapacityVOCpu.getUsedCapacity() < newTotalCpu)) {
CapacityVOCpu.setReservedCapacity(0);
CapacityVOCpu.setTotalCapacity(newTotalCpu);
} else {
s_logger.debug(
"What? new cpu is :"
+ newTotalCpu
+ ", old one is "
+ CapacityVOCpu.getUsedCapacity()
+ ","
+ CapacityVOCpu.getReservedCapacity()
+ ","
+ CapacityVOCpu.getTotalCapacity());
}
_capacityDao.update(CapacityVOCpu.getId(), CapacityVOCpu);
} else {
CapacityVO capacity =
new CapacityVO(
server.getId(),
server.getDataCenterId(),
server.getPodId(),
server.getClusterId(),
0L,
(long) (server.getCpus().longValue() * server.getSpeed().longValue()),
CapacityVO.CAPACITY_TYPE_CPU);
_capacityDao.persist(capacity);
}
SearchCriteria<CapacityVO> capacityMem = _capacityDao.createSearchCriteria();
capacityMem.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
capacityMem.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
capacityMem.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
capacityMem.addAnd("capacityType", SearchCriteria.Op.EQ, CapacityVO.CAPACITY_TYPE_MEMORY);
List<CapacityVO> capacityVOMems = _capacityDao.search(capacityMem, null);
if (capacityVOMems != null && !capacityVOMems.isEmpty()) {
CapacityVO CapacityVOMem = capacityVOMems.get(0);
long newTotalMem = (long) ((server.getTotalMemory()) * memoryOvercommitRatio);
if (CapacityVOMem.getTotalCapacity() <= newTotalMem
|| (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity()
<= newTotalMem)) {
CapacityVOMem.setTotalCapacity(newTotalMem);
} else if (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity()
> newTotalMem
&& CapacityVOMem.getUsedCapacity() < newTotalMem) {
CapacityVOMem.setReservedCapacity(0);
CapacityVOMem.setTotalCapacity(newTotalMem);
} else {
s_logger.debug(
"What? new cpu is :"
+ newTotalMem
+ ", old one is "
+ CapacityVOMem.getUsedCapacity()
+ ","
+ CapacityVOMem.getReservedCapacity()
+ ","
+ CapacityVOMem.getTotalCapacity());
}
_capacityDao.update(CapacityVOMem.getId(), CapacityVOMem);
} else {
CapacityVO capacity =
new CapacityVO(
server.getId(),
server.getDataCenterId(),
server.getPodId(),
server.getClusterId(),
0L,
server.getTotalMemory(),
CapacityVO.CAPACITY_TYPE_MEMORY);
_capacityDao.persist(capacity);
}
}
}
protected Long restart(final HaWorkVO work) {
final long vmId = work.getInstanceId();
final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
if (mgr == null) {
s_logger.warn(
"Unable to find a handler for " + work.getType().toString() + ", throwing out " + vmId);
return null;
}
VMInstanceVO vm = mgr.get(vmId);
if (vm == null) {
s_logger.info("Unable to find vm: " + vmId);
return null;
}
s_logger.info("HA on " + vm.toString());
if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
s_logger.info(
"VM "
+ vm.toString()
+ " has been changed. Current State = "
+ vm.getState()
+ " Previous State = "
+ work.getPreviousState()
+ " last updated = "
+ vm.getUpdated()
+ " previous updated = "
+ work.getUpdateTime());
return null;
}
final HostVO host = _hostDao.findById(work.getHostId());
DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
HostPodVO podVO = _podDao.findById(host.getPodId());
String hostDesc =
"name: "
+ host.getName()
+ "(id:"
+ host.getId()
+ "), availability zone: "
+ dcVO.getName()
+ ", pod: "
+ podVO.getName();
short alertType = AlertManager.ALERT_TYPE_USERVM;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
}
Boolean alive = null;
if (work.getStep() == Step.Investigating) {
if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
if (vm.getState() == State.Starting && vm.getUpdated() == work.getUpdateTime()) {
_itMgr.stateTransitTo(vm, Event.AgentReportStopped, null);
}
return null;
}
Enumeration<Investigator> en = _investigators.enumeration();
Investigator investigator = null;
while (en.hasMoreElements()) {
investigator = en.nextElement();
alive = investigator.isVmAlive(vm, host);
if (alive != null) {
s_logger.debug(
investigator.getName() + " found VM " + vm.getName() + "to be alive? " + alive);
break;
}
}
if (alive != null && alive) {
s_logger.debug("VM " + vm.getName() + " is found to be alive by " + investigator.getName());
if (host.getStatus() == Status.Up) {
compareState(vm, new AgentVmInfo(vm.getInstanceName(), mgr, State.Running), false);
return null;
} else {
s_logger.debug("Rescheduling because the host is not up but the vm is alive");
return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
}
}
boolean fenced = false;
if (alive == null || !alive) {
fenced = true;
s_logger.debug("Fencing off VM that we don't know the state of");
Enumeration<FenceBuilder> enfb = _fenceBuilders.enumeration();
while (enfb.hasMoreElements()) {
final FenceBuilder fb = enfb.nextElement();
Boolean result = fb.fenceOff(vm, host);
if (result != null && !result) {
fenced = false;
}
}
}
if (alive == null && !fenced) {
s_logger.debug("We were unable to fence off the VM " + vm.toString());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}
mgr.completeStopCommand(vm);
work.setStep(Step.Scheduled);
_haDao.update(work.getId(), work);
}
// send an alert for VMs that stop unexpectedly
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"VM (name: "
+ vm.getName()
+ ", id: "
+ vmId
+ ") stopped unexpectedly on host "
+ hostDesc,
"Virtual Machine "
+ vm.getName()
+ " (id: "
+ vm.getId()
+ ") running on host ["
+ hostDesc
+ "] stopped unexpectedly.");
vm = mgr.get(vm.getId());
if (!_forceHA && !vm.isHaEnabled()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM is not HA enabled so we're done.");
}
return null; // VM doesn't require HA
}
if (!_storageMgr.canVmRestartOnAnotherServer(vm.getId())) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM can not restart on another server.");
}
return null;
}
if (work.getTimesTried() > _maxRetries) {
s_logger.warn("Retried to max times so deleting: " + vmId);
return null;
}
try {
VMInstanceVO started = mgr.start(vm.getId(), 0);
if (started != null) {
s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
return null;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
}
vm = mgr.get(vm.getId());
work.setUpdateTime(vm.getUpdated());
work.setPreviousState(vm.getState());
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
} catch (final InsufficientCapacityException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return null;
} catch (final StorageUnavailableException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return null;
} catch (ConcurrentOperationException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return null;
} catch (ExecutionException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return null;
}
}