@Override
public Status investigate(final long hostId) {
final HostVO host = _hostDao.findById(hostId);
if (host == null) {
return null;
}
final Enumeration<Investigator> en = _investigators.enumeration();
Status hostState = null;
Investigator investigator = null;
while (en.hasMoreElements()) {
investigator = en.nextElement();
hostState = investigator.isAgentAlive(host);
if (hostState != null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
investigator.getName()
+ " was able to determine host "
+ hostId
+ " is in "
+ hostState.toString());
}
return hostState;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
investigator.getName() + " unable to determine the state of the host. Moving on.");
}
}
return null;
}
@Override
public boolean processDisconnect(long agentId, Status state) {
UserContext context = UserContext.current();
context.setAccountId(1);
/* Stopped VMware Host's virtual routers */
HostVO host = _hostDao.findById(agentId);
if (host.getHypervisorType() != HypervisorType.VMware) {
return true;
}
List<DomainRouterVO> routers = _routerDao.listByHostId(agentId);
for (DomainRouterVO router : routers) {
try {
State oldState = router.getState();
_routerMgr.stopRouter(router.getId(), true);
// In case only vCenter is disconnected, we want to shut down router directly
if (oldState == State.Running) {
shutdownRouterVM(router);
}
} catch (ResourceUnavailableException e) {
s_logger.warn(
"Fail to stop router " + router.getInstanceName() + " when host disconnected!", e);
} catch (ConcurrentOperationException e) {
s_logger.warn(
"Fail to stop router " + router.getInstanceName() + " when host disconnected!", e);
}
}
return true;
}
@Override
public void shutdown(final NetworkProfile profile, final NetworkOffering offering) {
final NetworkVO networkObject = networkDao.findById(profile.getId());
if (networkObject.getBroadcastDomainType() != BroadcastDomainType.Lswitch
|| networkObject.getBroadcastUri() == null) {
s_logger.warn(
"BroadcastUri is empty or incorrect for guestnetwork " + networkObject.getDisplayText());
return;
}
final List<NiciraNvpDeviceVO> devices =
niciraNvpDao.listByPhysicalNetwork(networkObject.getPhysicalNetworkId());
if (devices.isEmpty()) {
s_logger.error(
"No NiciraNvp Controller on physical network " + networkObject.getPhysicalNetworkId());
return;
}
final NiciraNvpDeviceVO niciraNvpDevice = devices.get(0);
final HostVO niciraNvpHost = hostDao.findById(niciraNvpDevice.getHostId());
final DeleteLogicalSwitchCommand cmd =
new DeleteLogicalSwitchCommand(
BroadcastDomainType.getValue(networkObject.getBroadcastUri()));
final DeleteLogicalSwitchAnswer answer =
(DeleteLogicalSwitchAnswer) agentMgr.easySend(niciraNvpHost.getId(), cmd);
if (answer == null || !answer.getResult()) {
s_logger.error("DeleteLogicalSwitchCommand failed");
}
super.shutdown(profile, offering);
}
@Override
public List<DomainRouterVO> startRouters(
final RouterDeploymentDefinition routerDeploymentDefinition)
throws StorageUnavailableException, InsufficientCapacityException,
ConcurrentOperationException, ResourceUnavailableException {
final List<DomainRouterVO> runningRouters = new ArrayList<DomainRouterVO>();
for (DomainRouterVO router : routerDeploymentDefinition.getRouters()) {
boolean skip = false;
final State state = router.getState();
if (router.getHostId() != null && state != State.Running) {
final HostVO host = _hostDao.findById(router.getHostId());
if (host == null || host.getState() != Status.Up) {
skip = true;
}
}
if (!skip) {
if (state != State.Running) {
router =
startVirtualRouter(
router,
_accountMgr.getSystemUser(),
_accountMgr.getSystemAccount(),
routerDeploymentDefinition.getParams());
}
if (router != null) {
runningRouters.add(router);
}
}
}
return runningRouters;
}
@Override
public Status investigate(final long hostId) {
final HostVO host = _hostDao.findById(hostId);
if (host == null) {
return null;
}
Status hostState = null;
for (Investigator investigator : investigators) {
hostState = investigator.isAgentAlive(host);
if (hostState != null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
investigator.getName()
+ " was able to determine host "
+ hostId
+ " is in "
+ hostState.toString());
}
return hostState;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
investigator.getName() + " unable to determine the state of the host. Moving on.");
}
}
return null;
}
@Override
public boolean deleteCiscoVnmcResource(DeleteCiscoVnmcResourceCmd cmd) {
Long vnmcResourceId = cmd.getCiscoVnmcResourceId();
CiscoVnmcControllerVO vnmcResource = _ciscoVnmcDao.findById(vnmcResourceId);
if (vnmcResource == null) {
throw new InvalidParameterValueException(
"Could not find a Cisco VNMC appliance with id " + vnmcResourceId);
}
// Check if there any ASA 1000v appliances
Long physicalNetworkId = vnmcResource.getPhysicalNetworkId();
PhysicalNetworkVO physicalNetwork = _physicalNetworkDao.findById(physicalNetworkId);
if (physicalNetwork != null) {
List<CiscoAsa1000vDeviceVO> responseList =
_ciscoAsa1000vDao.listByPhysicalNetwork(physicalNetworkId);
if (responseList.size() > 0) {
throw new CloudRuntimeException(
"Cisco VNMC appliance with id "
+ vnmcResourceId
+ " cannot be deleted as there Cisco ASA 1000v appliances using it");
}
}
HostVO vnmcHost = _hostDao.findById(vnmcResource.getHostId());
Long hostId = vnmcHost.getId();
vnmcHost.setResourceState(ResourceState.Maintenance);
_hostDao.update(hostId, vnmcHost);
_resourceMgr.deleteHost(hostId, false, false);
_ciscoVnmcDao.remove(vnmcResourceId);
return true;
}
@Override
public boolean generateVMSetupCommand(Long ssAHostId) {
HostVO ssAHost = _hostDao.findById(ssAHostId);
if (ssAHost.getType() != Host.Type.SecondaryStorageVM) {
return false;
}
SecondaryStorageVmVO secStorageVm = _secStorageVmDao.findByInstanceName(ssAHost.getName());
if (secStorageVm == null) {
s_logger.warn("secondary storage VM " + ssAHost.getName() + " doesn't exist");
return false;
}
SecStorageVMSetupCommand setupCmd = new SecStorageVMSetupCommand();
if (_allowedInternalSites != null) {
List<String> allowedCidrs = new ArrayList<String>();
String[] cidrs = _allowedInternalSites.split(",");
for (String cidr : cidrs) {
if (NetUtils.isValidCIDR(cidr) || NetUtils.isValidIp(cidr)) {
allowedCidrs.add(cidr);
}
}
List<? extends Nic> nics =
_networkMgr.getNicsForTraffic(secStorageVm.getId(), TrafficType.Management);
Nic privateNic = nics.get(0);
String privateCidr =
NetUtils.ipAndNetMaskToCidr(privateNic.getIp4Address(), privateNic.getNetmask());
String publicCidr =
NetUtils.ipAndNetMaskToCidr(
secStorageVm.getPublicIpAddress(), secStorageVm.getPublicNetmask());
if (NetUtils.isNetworkAWithinNetworkB(privateCidr, publicCidr)
|| NetUtils.isNetworkAWithinNetworkB(publicCidr, privateCidr)) {
s_logger.info(
"private and public interface overlaps, add a default route through private interface. privateCidr: "
+ privateCidr
+ ", publicCidr: "
+ publicCidr);
allowedCidrs.add(NetUtils.ALL_CIDRS);
}
setupCmd.setAllowedInternalSites(allowedCidrs.toArray(new String[allowedCidrs.size()]));
}
String copyPasswd = _configDao.getValue("secstorage.copy.password");
setupCmd.setCopyPassword(copyPasswd);
setupCmd.setCopyUserName(TemplateConstants.DEFAULT_HTTP_AUTH_USER);
Answer answer = _agentMgr.easySend(ssAHostId, setupCmd);
if (answer != null && answer.getResult()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Successfully programmed http auth into " + secStorageVm.getHostName());
}
return true;
} else {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"failed to program http auth into secondary storage vm : "
+ secStorageVm.getHostName());
}
return false;
}
}
@Test
public void investigateHostStatusFailure() {
Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
// Set the list of investigators, CheckOnAgentInvestigator suffices for now
// Also no need to mock isAgentAlive() as actual implementation returns null
Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class);
List<Investigator> investigators = new ArrayList<Investigator>();
investigators.add(investigator);
highAvailabilityManager.setInvestigators(investigators);
assertNull(highAvailabilityManager.investigate(1l));
}
@Test
public void investigateHostStatusSuccess() {
Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
// Set the list of investigators, CheckOnAgentInvestigator suffices for now
Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class);
List<Investigator> investigators = new ArrayList<Investigator>();
investigators.add(investigator);
highAvailabilityManager.setInvestigators(investigators);
// Mock isAgentAlive to return host status as Down
Mockito.when(investigator.isAgentAlive(hostVO)).thenReturn(Status.Down);
assertTrue(highAvailabilityManager.investigate(1l) == Status.Down);
}
@Override
public CiscoVnmcResourceResponse createCiscoVnmcResourceResponse(
CiscoVnmcController ciscoVnmcResourceVO) {
HostVO ciscoVnmcHost = _hostDao.findById(ciscoVnmcResourceVO.getHostId());
CiscoVnmcResourceResponse response = new CiscoVnmcResourceResponse();
response.setId(ciscoVnmcResourceVO.getUuid());
response.setPhysicalNetworkId(ciscoVnmcResourceVO.getPhysicalNetworkId());
response.setProviderName(ciscoVnmcResourceVO.getProviderName());
response.setResourceName(ciscoVnmcHost.getName());
return response;
}
@Override
public AgentControlAnswer onConsoleAccessAuthentication(ConsoleAccessAuthenticationCommand cmd) {
long vmId = 0;
if (cmd.getVmId() != null && cmd.getVmId().isEmpty()) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Invalid vm id sent from proxy(happens when proxy session has terminated)");
}
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
try {
vmId = Long.parseLong(cmd.getVmId());
} catch (NumberFormatException e) {
s_logger.error(
"Invalid vm id " + cmd.getVmId() + " sent from console access authentication", e);
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
// TODO authentication channel between console proxy VM and management
// server needs to be secured,
// the data is now being sent through private network, but this is
// apparently not enough
VMInstanceVO vm = _instanceDao.findById(vmId);
if (vm == null) {
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
if (vm.getHostId() == null) {
s_logger.warn("VM " + vmId + " lost host info, failed authentication request");
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
HostVO host = _hostDao.findById(vm.getHostId());
if (host == null) {
s_logger.warn("VM " + vmId + "'s host does not exist, fail authentication request");
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
String sid = cmd.getSid();
if (sid == null || !sid.equals(vm.getVncPassword())) {
s_logger.warn("sid " + sid + " in url does not match stored sid " + vm.getVncPassword());
return new ConsoleAccessAuthenticationAnswer(cmd, false);
}
return new ConsoleAccessAuthenticationAnswer(cmd, true);
}
@Override
public boolean shutdown(Network network, ReservationContext context, boolean cleanup)
throws ConcurrentOperationException, ResourceUnavailableException {
unassignAsa1000vFromNetwork(network);
String vlan = network.getBroadcastUri().getHost();
long vlanId = Long.parseLong(vlan);
List<CiscoVnmcControllerVO> devices =
_ciscoVnmcDao.listByPhysicalNetwork(network.getPhysicalNetworkId());
if (!devices.isEmpty()) {
CiscoVnmcControllerVO ciscoVnmcDevice = devices.get(0);
HostVO ciscoVnmcHost = _hostDao.findById(ciscoVnmcDevice.getHostId());
cleanupLogicalEdgeFirewall(vlanId, ciscoVnmcHost.getId());
}
return true;
}
@Override
protected void injectMockito() {
if (host == null) {
return;
}
List<HostVO> results = new ArrayList<HostVO>();
results.add(host);
Mockito.when(hostDao.listAll()).thenReturn(results);
Mockito.when(hostDao.findById(Matchers.anyLong())).thenReturn(host);
Mockito.when(hostDao.findHypervisorHostInCluster(Matchers.anyLong())).thenReturn(results);
List<EndPoint> eps = new ArrayList<EndPoint>();
eps.add(
RemoteHostEndPoint.getHypervisorHostEndPoint(
host.getId(), host.getPrivateIpAddress(), host.getPublicIpAddress()));
Mockito.when(selector.selectAll(Matchers.any(DataStore.class))).thenReturn(eps);
Mockito.when(selector.select(Matchers.any(DataObject.class))).thenReturn(eps.get(0));
Mockito.when(selector.select(Matchers.any(DataObject.class), Matchers.any(DataObject.class)))
.thenReturn(eps.get(0));
}
@Override
public boolean deleteHost(Long hostId) {
List<SnapshotVO> snapshots = _snapshotDao.listByHostId(hostId);
if (snapshots != null && !snapshots.isEmpty()) {
throw new CloudRuntimeException(
"Can not delete this secondary storage due to there are still snapshots on it ");
}
List<Long> list = _templateDao.listPrivateTemplatesByHost(hostId);
if (list != null && !list.isEmpty()) {
throw new CloudRuntimeException(
"Can not delete this secondary storage due to there are still private template on it ");
}
_vmTemplateHostDao.deleteByHost(hostId);
HostVO host = _hostDao.findById(hostId);
host.setGuid(null);
_hostDao.update(hostId, host);
_hostDao.remove(hostId);
return true;
}
@Override
public boolean deleteTrafficMonitor(DeleteTrafficMonitorCmd cmd) {
long hostId = cmd.getId();
User caller = _accountMgr.getActiveUser(UserContext.current().getCallerUserId());
HostVO trafficMonitor = _hostDao.findById(hostId);
if (trafficMonitor == null) {
throw new InvalidParameterValueException(
"Could not find an traffic monitor with ID: " + hostId);
}
try {
if (_resourceMgr.maintain(hostId) && _resourceMgr.deleteHost(hostId, false, false)) {
return true;
} else {
return false;
}
} catch (AgentUnavailableException e) {
s_logger.debug(e);
return false;
}
}
@Override
public boolean release(
final Network network,
final NicProfile nic,
final VirtualMachineProfile vm,
final ReservationContext context)
throws ConcurrentOperationException, ResourceUnavailableException {
if (!canHandle(network, Service.Connectivity)) {
return false;
}
if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vswitch) {
return false;
}
if (nic.getTrafficType() != Networks.TrafficType.Guest) {
return false;
}
final HostVO host = _hostDao.findById(vm.getVirtualMachine().getHostId());
_ovsTunnelMgr.checkAndRemoveHostFromTunnelNetwork(network, host);
return true;
}
@Override
public void processCancelMaintenaceEventAfter(Long hostId) {
updateCapacityForHost(_hostDao.findById(hostId));
}
@Override
public DomainRouterVO startVirtualRouter(
final DomainRouterVO router,
final User user,
final Account caller,
final Map<Param, Object> params)
throws StorageUnavailableException, InsufficientCapacityException,
ConcurrentOperationException, ResourceUnavailableException {
if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) {
return start(router, user, caller, params, null);
}
if (router.getState() == State.Running) {
s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!");
return router;
}
//
// If another thread has already requested a VR start, there is a
// transition period for VR to transit from
// Starting to Running, there exist a race conditioning window here
// We will wait until VR is up or fail
if (router.getState() == State.Starting) {
return waitRouter(router);
}
final DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null);
DomainRouterVO result = null;
assert router.getIsRedundantRouter();
final List<Long> networkIds = _routerDao.getRouterNetworks(router.getId());
DomainRouterVO routerToBeAvoid = null;
if (networkIds.size() != 0) {
final List<DomainRouterVO> routerList = _routerDao.findByNetwork(networkIds.get(0));
for (final DomainRouterVO rrouter : routerList) {
if (rrouter.getHostId() != null
&& rrouter.getIsRedundantRouter()
&& rrouter.getState() == State.Running) {
if (routerToBeAvoid != null) {
throw new ResourceUnavailableException(
"Try to start router "
+ router.getInstanceName()
+ "("
+ router.getId()
+ ")"
+ ", but there are already two redundant routers with IP "
+ router.getPublicIpAddress()
+ ", they are "
+ rrouter.getInstanceName()
+ "("
+ rrouter.getId()
+ ") and "
+ routerToBeAvoid.getInstanceName()
+ "("
+ routerToBeAvoid.getId()
+ ")",
DataCenter.class,
rrouter.getDataCenterId());
}
routerToBeAvoid = rrouter;
}
}
}
if (routerToBeAvoid == null) {
return start(router, user, caller, params, null);
}
// We would try best to deploy the router to another place
final int retryIndex = 5;
final ExcludeList[] avoids = new ExcludeList[5];
avoids[0] = new ExcludeList();
avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn());
avoids[1] = new ExcludeList();
avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId());
avoids[2] = new ExcludeList();
final List<VolumeVO> volumes =
_volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Volume.Type.ROOT);
if (volumes != null && volumes.size() != 0) {
avoids[2].addPool(volumes.get(0).getPoolId());
}
avoids[2].addHost(routerToBeAvoid.getHostId());
avoids[3] = new ExcludeList();
avoids[3].addHost(routerToBeAvoid.getHostId());
avoids[4] = new ExcludeList();
for (int i = 0; i < retryIndex; i++) {
if (s_logger.isTraceEnabled()) {
s_logger.trace(
"Try to deploy redundant virtual router:"
+ router.getHostName()
+ ", for "
+ i
+ " time");
}
plan.setAvoids(avoids[i]);
try {
result = start(router, user, caller, params, plan);
} catch (final InsufficientServerCapacityException ex) {
result = null;
}
if (result != null) {
break;
}
}
return result;
}
@Override
public Boolean isVmAlive(VirtualMachine vm, Host host) {
if (!vm.getType().isUsedBySystem()) {
s_logger.debug("Not a System Vm, unable to determine state of " + vm + " returning null");
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Testing if " + vm + " is alive");
}
if (vm.getHostId() == null) {
s_logger.debug("There's no host id for " + vm);
return null;
}
HostVO vmHost = _hostDao.findById(vm.getHostId());
if (vmHost == null) {
s_logger.debug("Unable to retrieve the host by using id " + vm.getHostId());
return null;
}
List<? extends Nic> nics = _networkMgr.getNicsForTraffic(vm.getId(), TrafficType.Management);
if (nics.size() == 0) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Unable to find a management nic, cannot ping this system VM, unable to determine state of "
+ vm
+ " returning null");
}
return null;
}
for (Nic nic : nics) {
if (nic.getIp4Address() == null) {
continue;
}
// get the data center IP address, find a host on the pod, use that host to ping the data
// center IP address
List<Long> otherHosts = findHostByPod(vmHost.getPodId(), vm.getHostId());
for (Long otherHost : otherHosts) {
Status vmState = testIpAddress(otherHost, nic.getIp4Address());
if (vmState == null) {
// can't get information from that host, try the next one
continue;
}
if (vmState == Status.Up) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"successfully pinged vm's private IP ("
+ vm.getPrivateIpAddress()
+ "), returning that the VM is up");
}
return Boolean.TRUE;
} else if (vmState == Status.Down) {
// We can't ping the VM directly...if we can ping the host, then report the VM down.
// If we can't ping the host, then we don't have enough information.
Status vmHostState = testIpAddress(otherHost, vmHost.getPrivateIpAddress());
if ((vmHostState != null) && (vmHostState == Status.Up)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"successfully pinged vm's host IP ("
+ vmHost.getPrivateIpAddress()
+ "), but could not ping VM, returning that the VM is down");
}
return Boolean.FALSE;
}
}
}
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("unable to determine state of " + vm + " returning null");
}
return null;
}
protected Long restart(HaWorkVO work) {
List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
if (items.size() > 0) {
StringBuilder str =
new StringBuilder(
"Cancelling this work item because newer ones have been scheduled. Work Ids = [");
for (HaWorkVO item : items) {
str.append(item.getId()).append(", ");
}
str.delete(str.length() - 2, str.length()).append("]");
s_logger.info(str.toString());
return null;
}
items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
if (items.size() > 0) {
StringBuilder str =
new StringBuilder(
"Waiting because there's HA work being executed on an item currently. Work Ids =[");
for (HaWorkVO item : items) {
str.append(item.getId()).append(", ");
}
str.delete(str.length() - 2, str.length()).append("]");
s_logger.info(str.toString());
return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
}
long vmId = work.getInstanceId();
VMInstanceVO vm = _itMgr.findByIdAndType(work.getType(), work.getInstanceId());
if (vm == null) {
s_logger.info("Unable to find vm: " + vmId);
return null;
}
s_logger.info("HA on " + vm);
if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
s_logger.info(
"VM "
+ vm
+ " has been changed. Current State = "
+ vm.getState()
+ " Previous State = "
+ work.getPreviousState()
+ " last updated = "
+ vm.getUpdated()
+ " previous updated = "
+ work.getUpdateTime());
return null;
}
short alertType = AlertManager.ALERT_TYPE_USERVM;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
} else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_SSVM;
}
HostVO host = _hostDao.findById(work.getHostId());
boolean isHostRemoved = false;
if (host == null) {
host = _hostDao.findByIdIncludingRemoved(work.getHostId());
if (host != null) {
s_logger.debug(
"VM "
+ vm.toString()
+ " is now no longer on host "
+ work.getHostId()
+ " as the host is removed");
isHostRemoved = true;
}
}
DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
HostPodVO podVO = _podDao.findById(host.getPodId());
String hostDesc =
"name: "
+ host.getName()
+ "(id:"
+ host.getId()
+ "), availability zone: "
+ dcVO.getName()
+ ", pod: "
+ podVO.getName();
Boolean alive = null;
if (work.getStep() == Step.Investigating) {
if (!isHostRemoved) {
if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
return null;
}
Enumeration<Investigator> en = _investigators.enumeration();
Investigator investigator = null;
while (en.hasMoreElements()) {
investigator = en.nextElement();
alive = investigator.isVmAlive(vm, host);
s_logger.info(investigator.getName() + " found " + vm + "to be alive? " + alive);
if (alive != null) {
break;
}
}
boolean fenced = false;
if (alive == null) {
s_logger.debug("Fencing off VM that we don't know the state of");
Enumeration<FenceBuilder> enfb = _fenceBuilders.enumeration();
while (enfb.hasMoreElements()) {
FenceBuilder fb = enfb.nextElement();
Boolean result = fb.fenceOff(vm, host);
s_logger.info("Fencer " + fb.getName() + " returned " + result);
if (result != null && result) {
fenced = true;
break;
}
}
} else if (!alive) {
fenced = true;
} else {
s_logger.debug(
"VM " + vm.getHostName() + " is found to be alive by " + investigator.getName());
if (host.getStatus() == Status.Up) {
s_logger.info(vm + " is alive and host is up. No need to restart it.");
return null;
} else {
s_logger.debug("Rescheduling because the host is not up but the vm is alive");
return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
}
}
if (!fenced) {
s_logger.debug("We were unable to fence off the VM " + vm);
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}
try {
_itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
} catch (ResourceUnavailableException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (OperationTimedoutException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (ConcurrentOperationException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
}
work.setStep(Step.Scheduled);
_haDao.update(work.getId(), work);
} else {
s_logger.debug(
"How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
try {
_itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
} catch (ResourceUnavailableException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (OperationTimedoutException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
} catch (ConcurrentOperationException e) {
assert false : "How do we hit this when force is true?";
throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
}
}
}
vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());
if (!_forceHA && !vm.isHaEnabled()) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM is not HA enabled so we're done.");
}
return null; // VM doesn't require HA
}
if (!_storageMgr.canVmRestartOnAnotherServer(vm.getId())) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("VM can not restart on another server.");
}
return null;
}
if (work.getTimesTried() > _maxRetries) {
s_logger.warn("Retried to max times so deleting: " + vmId);
return null;
}
try {
VMInstanceVO started =
_itMgr.advanceStart(
vm,
new HashMap<VirtualMachineProfile.Param, Object>(),
_accountMgr.getSystemUser(),
_accountMgr.getSystemAccount());
if (started != null) {
s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
return null;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
}
} catch (final InsufficientCapacityException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"Insufficient capacity to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (final ResourceUnavailableException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (ConcurrentOperationException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
} catch (OperationTimedoutException e) {
s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterIdToDeployIn(),
vm.getPodIdToDeployIn(),
"Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
"The Storage is unavailable for trying to restart VM, name: "
+ vm.getHostName()
+ ", id: "
+ vmId
+ " which was running on host "
+ hostDesc);
}
vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());
work.setUpdateTime(vm.getUpdated());
work.setPreviousState(vm.getState());
return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}
/**
* compareState does as its name suggests and compares the states between management server and
* agent. It returns whether something should be cleaned up
*/
protected Command compareState(VMInstanceVO vm, final AgentVmInfo info, final boolean fullSync) {
State agentState = info.state;
final String agentName = info.name;
final State serverState = vm.getState();
final String serverName = vm.getName();
Command command = null;
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"VM "
+ serverName
+ ": server state = "
+ serverState.toString()
+ " and agent state = "
+ agentState.toString());
}
if (agentState == State.Error) {
agentState = State.Stopped;
short alertType = AlertManager.ALERT_TYPE_USERVM;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
}
HostPodVO podVO = _podDao.findById(vm.getPodId());
DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
HostVO hostVO = _hostDao.findById(vm.getHostId());
String hostDesc =
"name: "
+ hostVO.getName()
+ " (id:"
+ hostVO.getId()
+ "), availability zone: "
+ dcVO.getName()
+ ", pod: "
+ podVO.getName();
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"VM (name: "
+ vm.getName()
+ ", id: "
+ vm.getId()
+ ") stopped on host "
+ hostDesc
+ " due to storage failure",
"Virtual Machine "
+ vm.getName()
+ " (id: "
+ vm.getId()
+ ") running on host ["
+ vm.getHostId()
+ "] stopped due to storage failure.");
}
if (serverState == State.Migrating) {
s_logger.debug("Skipping vm in migrating state: " + vm.toString());
return null;
}
if (agentState == serverState) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Both states are " + agentState.toString() + " for " + serverName);
}
assert (agentState == State.Stopped || agentState == State.Running)
: "If the states we send up is changed, this must be changed.";
_itMgr.stateTransitTo(
vm,
agentState == State.Stopped
? VirtualMachine.Event.AgentReportStopped
: VirtualMachine.Event.AgentReportRunning,
vm.getHostId());
if (agentState == State.Stopped) {
s_logger.debug("State matches but the agent said stopped so let's send a cleanup anyways.");
return info.mgr.cleanup(vm, agentName);
}
return null;
}
if (agentState == State.Stopped) {
// This state means the VM on the agent was detected previously
// and now is gone. This is slightly different than if the VM
// was never completed but we still send down a Stop Command
// to ensure there's cleanup.
if (serverState == State.Running) {
// Our records showed that it should be running so let's restart it.
vm = info.mgr.get(vm.getId());
scheduleRestart(vm, false);
command = info.mgr.cleanup(vm, agentName);
} else if (serverState == State.Stopping) {
if (fullSync) {
s_logger.debug("VM is in stopping state on full sync. Updating the status to stopped");
vm = info.mgr.get(vm.getId());
info.mgr.completeStopCommand(vm);
command = info.mgr.cleanup(vm, agentName);
} else {
s_logger.debug("Ignoring VM in stopping mode: " + vm.getName());
}
} else if (serverState == State.Starting) {
s_logger.debug("Ignoring VM in starting mode: " + vm.getName());
} else {
s_logger.debug("Sending cleanup to a stopped vm: " + agentName);
_itMgr.stateTransitTo(vm, VirtualMachine.Event.AgentReportStopped, null);
command = info.mgr.cleanup(vm, agentName);
}
} else if (agentState == State.Running) {
if (serverState == State.Starting) {
if (fullSync) {
s_logger.debug("VM state is starting on full sync so updating it to running");
vm = info.mgr.get(vm.getId());
info.mgr.completeStartCommand(vm);
}
} else if (serverState == State.Stopping) {
if (fullSync) {
s_logger.debug("VM state is in stopping on fullsync so resend stop.");
vm = info.mgr.get(vm.getId());
info.mgr.completeStopCommand(vm);
command = info.mgr.cleanup(vm, agentName);
} else {
s_logger.debug("VM is in stopping state so no action.");
}
} else if (serverState == State.Destroyed
|| serverState == State.Stopped
|| serverState == State.Expunging) {
s_logger.debug("VM state is in stopped so stopping it on the agent");
vm = info.mgr.get(vm.getId());
command = info.mgr.cleanup(vm, agentName);
} else {
_itMgr.stateTransitTo(vm, VirtualMachine.Event.AgentReportRunning, vm.getHostId());
}
} /*else if (agentState == State.Unknown) {
if (serverState == State.Running) {
if (fullSync) {
vm = info.handler.get(vm.getId());
}
scheduleRestart(vm, false);
} else if (serverState == State.Starting) {
if (fullSync) {
vm = info.handler.get(vm.getId());
}
scheduleRestart(vm, false);
} else if (serverState == State.Stopping) {
if (fullSync) {
s_logger.debug("VM state is stopping in full sync. Resending stop");
command = info.handler.cleanup(vm, agentName);
}
}
}*/
return command;
}
public Long migrate(final HaWorkVO work) {
final long vmId = work.getInstanceId();
final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
VMInstanceVO vm = mgr.get(vmId);
if (vm == null || vm.getRemoved() != null) {
s_logger.debug("Unable to find the vm " + vmId);
return null;
}
s_logger.info("Migrating vm: " + vm.toString());
if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
s_logger.info("VM is not longer running on the current hostId");
return null;
}
short alertType = AlertManager.ALERT_TYPE_USERVM_MIGRATE;
if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
} else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
}
HostVO fromHost = _hostDao.findById(vm.getHostId());
String fromHostName = ((fromHost == null) ? "unknown" : fromHost.getName());
HostVO toHost = null;
if (work.getStep() == Step.Scheduled) {
if (vm.getState() != State.Running) {
s_logger.info(
"VM's state is not ready for migration. "
+ vm.toString()
+ " State is "
+ vm.getState().toString());
return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
}
DataCenterVO dcVO = _dcDao.findById(fromHost.getDataCenterId());
HostPodVO podVO = _podDao.findById(fromHost.getPodId());
try {
toHost = mgr.prepareForMigration(vm);
if (toHost == null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Unable to find a host for migrating vm " + vmId);
}
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Unable to find a suitable host");
}
} catch (final InsufficientCapacityException e) {
s_logger.warn("Unable to mgirate due to insufficient capacity " + vm.toString());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Insufficient capacity");
} catch (final StorageUnavailableException e) {
s_logger.warn("Storage is unavailable: " + vm.toString());
_alertMgr.sendAlert(
alertType,
vm.getDataCenterId(),
vm.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHostName
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Storage is gone.");
}
if (toHost == null) {
_agentMgr.maintenanceFailed(vm.getHostId());
return null;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Migrating from " + work.getHostId() + " to " + toHost.getId());
}
work.setStep(Step.Migrating);
work.setHostId(toHost.getId());
_haDao.update(work.getId(), work);
}
if (work.getStep() == Step.Migrating) {
vm = mgr.get(vmId); // let's see if anything has changed.
boolean migrated = false;
if (vm == null
|| vm.getRemoved() != null
|| vm.getHostId() == null
|| !_itMgr.stateTransitTo(vm, Event.MigrationRequested, vm.getHostId())) {
s_logger.info("Migration cancelled because state has changed: " + vm.toString());
} else {
try {
boolean isWindows =
_guestOSCategoryDao
.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId())
.getName()
.equalsIgnoreCase("Windows");
MigrateCommand cmd =
new MigrateCommand(vm.getInstanceName(), toHost.getPrivateIpAddress(), isWindows);
Answer answer = _agentMgr.send(fromHost.getId(), cmd);
if (answer != null && answer.getResult()) {
migrated = true;
_storageMgr.unshare(vm, fromHost);
work.setStep(Step.Investigating);
_haDao.update(work.getId(), work);
}
} catch (final AgentUnavailableException e) {
s_logger.debug("host became unavailable");
} catch (final OperationTimedoutException e) {
s_logger.debug("operation timed out");
if (e.isActive()) {
scheduleRestart(vm, true);
}
}
}
if (!migrated) {
s_logger.info("Migration was unsuccessful. Cleaning up: " + vm.toString());
DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
HostPodVO podVO = _podDao.findById(vm.getPodId());
_alertMgr.sendAlert(
alertType,
fromHost.getDataCenterId(),
fromHost.getPodId(),
"Unable to migrate vm "
+ vm.getName()
+ " from host "
+ fromHost.getName()
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Migrate Command failed. Please check logs.");
_itMgr.stateTransitTo(vm, Event.MigrationFailedOnSource, toHost.getId());
_agentMgr.maintenanceFailed(vm.getHostId());
Command cleanup = mgr.cleanup(vm, null);
_agentMgr.easySend(toHost.getId(), cleanup);
_storageMgr.unshare(vm, toHost);
return null;
}
}
if (toHost == null) {
toHost = _hostDao.findById(work.getHostId());
}
DataCenterVO dcVO = _dcDao.findById(toHost.getDataCenterId());
HostPodVO podVO = _podDao.findById(toHost.getPodId());
try {
if (!mgr.completeMigration(vm, toHost)) {
_alertMgr.sendAlert(
alertType,
toHost.getDataCenterId(),
toHost.getPodId(),
"Unable to migrate "
+ vmId
+ " to host "
+ toHost.getName()
+ " in zone "
+ dcVO.getName()
+ " and pod "
+ podVO.getName(),
"Migration not completed");
s_logger.warn("Unable to complete migration: " + vm.toString());
} else {
s_logger.info("Migration is complete: " + vm.toString());
}
return null;
} catch (final AgentUnavailableException e) {
s_logger.warn("Agent is unavailable for " + vm.toString());
} catch (final OperationTimedoutException e) {
s_logger.warn("Operation timed outfor " + vm.toString());
}
_itMgr.stateTransitTo(vm, Event.MigrationFailedOnDest, toHost.getId());
return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
}
@Override
public Network implement(
final Network network,
final NetworkOffering offering,
final DeployDestination dest,
final ReservationContext context)
throws InsufficientVirtualNetworkCapacityException {
assert network.getState() == State.Implementing : "Why are we implementing " + network;
final long dcId = dest.getDataCenter().getId();
Long physicalNetworkId = network.getPhysicalNetworkId();
// physical network id can be null in Guest Network in Basic zone, so locate the physical
// network
if (physicalNetworkId == null) {
physicalNetworkId =
networkModel.findPhysicalNetworkId(dcId, offering.getTags(), offering.getTrafficType());
}
final NetworkVO implemented =
new NetworkVO(
network.getTrafficType(),
network.getMode(),
network.getBroadcastDomainType(),
network.getNetworkOfferingId(),
State.Allocated,
network.getDataCenterId(),
physicalNetworkId,
offering.getRedundantRouter());
if (network.getGateway() != null) {
implemented.setGateway(network.getGateway());
}
if (network.getCidr() != null) {
implemented.setCidr(network.getCidr());
}
// Name is either the given name or the uuid
String name = network.getName();
if (name == null || name.isEmpty()) {
name = ((NetworkVO) network).getUuid();
}
if (name.length() > MAX_NAME_LENGTH) {
name = name.substring(0, MAX_NAME_LENGTH - 1);
}
final List<NiciraNvpDeviceVO> devices = niciraNvpDao.listByPhysicalNetwork(physicalNetworkId);
if (devices.isEmpty()) {
s_logger.error("No NiciraNvp Controller on physical network " + physicalNetworkId);
return null;
}
final NiciraNvpDeviceVO niciraNvpDevice = devices.get(0);
final HostVO niciraNvpHost = hostDao.findById(niciraNvpDevice.getHostId());
hostDao.loadDetails(niciraNvpHost);
final String transportzoneuuid = niciraNvpHost.getDetail("transportzoneuuid");
final String transportzoneisotype = niciraNvpHost.getDetail("transportzoneisotype");
final CreateLogicalSwitchCommand cmd =
new CreateLogicalSwitchCommand(
transportzoneuuid,
transportzoneisotype,
name,
context.getDomain().getName() + "-" + context.getAccount().getAccountName());
final CreateLogicalSwitchAnswer answer =
(CreateLogicalSwitchAnswer) agentMgr.easySend(niciraNvpHost.getId(), cmd);
if (answer == null || !answer.getResult()) {
s_logger.error("CreateLogicalSwitchCommand failed");
return null;
}
try {
implemented.setBroadcastUri(new URI("lswitch", answer.getLogicalSwitchUuid(), null));
implemented.setBroadcastDomainType(BroadcastDomainType.Lswitch);
s_logger.info(
"Implemented OK, network linked to = " + implemented.getBroadcastUri().toString());
} catch (final URISyntaxException e) {
s_logger.error(
"Unable to store logical switch id in broadcast uri, uuid = " + implemented.getUuid(), e);
return null;
}
return implemented;
}
@DB
protected void CheckAndCreateTunnel(VirtualMachine instance, DeployDestination dest) {
if (!_isEnabled) {
return;
}
if (instance.getType() != VirtualMachine.Type.User
&& instance.getType() != VirtualMachine.Type.DomainRouter) {
return;
}
long hostId = dest.getHost().getId();
long accountId = instance.getAccountId();
List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
List<DomainRouterVO> routers =
_routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
List<VMInstanceVO> ins = new ArrayList<VMInstanceVO>();
if (vms != null) {
ins.addAll(vms);
}
if (routers.size() != 0) {
ins.addAll(routers);
}
List<Pair<Long, Integer>> toHosts = new ArrayList<Pair<Long, Integer>>();
List<Pair<Long, Integer>> fromHosts = new ArrayList<Pair<Long, Integer>>();
int key;
for (VMInstanceVO v : ins) {
Long rh = v.getHostId();
if (rh == null || rh.longValue() == hostId) {
continue;
}
OvsTunnelAccountVO ta =
_tunnelAccountDao.getByFromToAccount(hostId, rh.longValue(), accountId);
if (ta == null) {
key = getGreKey(hostId, rh.longValue(), accountId);
if (key == -1) {
s_logger.warn(
String.format(
"Cannot get GRE key for from=%1$s to=%2$s accountId=%3$s, tunnel create failed",
hostId, rh.longValue(), accountId));
continue;
}
Pair<Long, Integer> p = new Pair<Long, Integer>(rh, Integer.valueOf(key));
if (!toHosts.contains(p)) {
toHosts.add(p);
}
}
ta = _tunnelAccountDao.getByFromToAccount(rh.longValue(), hostId, accountId);
if (ta == null) {
key = getGreKey(rh.longValue(), hostId, accountId);
if (key == -1) {
s_logger.warn(
String.format(
"Cannot get GRE key for from=%1$s to=%2$s accountId=%3$s, tunnel create failed",
rh.longValue(), hostId, accountId));
continue;
}
Pair<Long, Integer> p = new Pair<Long, Integer>(rh, Integer.valueOf(key));
if (!fromHosts.contains(p)) {
fromHosts.add(p);
}
}
}
try {
String myIp = dest.getHost().getPrivateIpAddress();
for (Pair<Long, Integer> i : toHosts) {
HostVO rHost = _hostDao.findById(i.first());
Commands cmds =
new Commands(
new OvsCreateTunnelCommand(
rHost.getPrivateIpAddress(),
i.second().toString(),
Long.valueOf(hostId),
i.first(),
accountId,
myIp));
s_logger.debug("Ask host " + hostId + " to create gre tunnel to " + i.first());
Answer[] answers = _agentMgr.send(hostId, cmds);
handleCreateTunnelAnswer(answers);
}
for (Pair<Long, Integer> i : fromHosts) {
HostVO rHost = _hostDao.findById(i.first());
Commands cmd2s =
new Commands(
new OvsCreateTunnelCommand(
myIp,
i.second().toString(),
i.first(),
Long.valueOf(hostId),
accountId,
rHost.getPrivateIpAddress()));
s_logger.debug("Ask host " + i.first() + " to create gre tunnel to " + hostId);
Answer[] answers = _agentMgr.send(i.first(), cmd2s);
handleCreateTunnelAnswer(answers);
}
} catch (Exception e) {
s_logger.debug("Ovs Tunnel network created tunnel failed", e);
}
}
HostVO findHost(VMInstanceVO vm) {
return _hostDao.findById(vm.getHostId());
}
private HypervisorType getHypervisorType(long hostId) {
HostVO host = _hostDao.findById(hostId);
if (host != null) return host.getHypervisorType();
return HypervisorType.None;
}
@Override
public boolean implement(
Network network, NetworkOffering offering, DeployDestination dest, ReservationContext context)
throws ConcurrentOperationException, ResourceUnavailableException,
InsufficientCapacityException {
DataCenter zone = _configMgr.getZone(network.getDataCenterId());
if (zone.getNetworkType() == NetworkType.Basic) {
s_logger.debug("Not handling network implement in zone of type " + NetworkType.Basic);
return false;
}
if (!canHandle(network)) {
return false;
}
List<CiscoVnmcControllerVO> devices =
_ciscoVnmcDao.listByPhysicalNetwork(network.getPhysicalNetworkId());
if (devices.isEmpty()) {
s_logger.error("No Cisco Vnmc device on network " + network.getName());
return false;
}
List<CiscoAsa1000vDeviceVO> asaList =
_ciscoAsa1000vDao.listByPhysicalNetwork(network.getPhysicalNetworkId());
if (asaList.isEmpty()) {
s_logger.debug("No Cisco ASA 1000v device on network " + network.getName());
return false;
}
NetworkAsa1000vMapVO asaForNetwork = _networkAsa1000vMapDao.findByNetworkId(network.getId());
if (asaForNetwork != null) {
s_logger.debug("Cisco ASA 1000v device already associated with network " + network.getName());
return true;
}
if (!_networkModel.isProviderSupportServiceInNetwork(
network.getId(), Service.SourceNat, Provider.CiscoVnmc)) {
s_logger.error(
"SourceNat service is not provided by Cisco Vnmc device on network " + network.getName());
return false;
}
Transaction txn = Transaction.currentTxn();
boolean status = false;
try {
txn.start();
// ensure that there is an ASA 1000v assigned to this network
CiscoAsa1000vDevice assignedAsa = assignAsa1000vToNetwork(network);
if (assignedAsa == null) {
s_logger.error("Unable to assign ASA 1000v device to network " + network.getName());
return false;
}
ClusterVO asaCluster = _clusterDao.findById(assignedAsa.getClusterId());
ClusterVSMMapVO clusterVsmMap = _clusterVsmMapDao.findByClusterId(assignedAsa.getClusterId());
if (clusterVsmMap == null) {
s_logger.error(
"Vmware cluster "
+ asaCluster.getName()
+ " has no Cisco Nexus VSM device associated with it");
return false;
}
CiscoNexusVSMDeviceVO vsmDevice = _vsmDeviceDao.findById(clusterVsmMap.getVsmId());
if (vsmDevice == null) {
s_logger.error(
"Unable to load details of Cisco Nexus VSM device associated with cluster "
+ asaCluster.getName());
return false;
}
CiscoVnmcControllerVO ciscoVnmcDevice = devices.get(0);
HostVO ciscoVnmcHost = _hostDao.findById(ciscoVnmcDevice.getHostId());
_hostDao.loadDetails(ciscoVnmcHost);
Account owner = context.getAccount();
PublicIp sourceNatIp = _ipAddrMgr.assignSourceNatIpAddressToGuestNetwork(owner, network);
String vlan = network.getBroadcastUri().getHost();
long vlanId = Long.parseLong(vlan);
List<VlanVO> vlanVOList =
_vlanDao.listVlansByPhysicalNetworkId(network.getPhysicalNetworkId());
List<String> publicGateways = new ArrayList<String>();
for (VlanVO vlanVO : vlanVOList) {
publicGateways.add(vlanVO.getVlanGateway());
}
// due to VNMC limitation of not allowing source NAT ip as the outside ip of firewall,
// an additional public ip needs to acquired for assigning as firewall outside ip.
// In case there are already additional ip addresses available (network restart) use one
// of them such that it is not the source NAT ip
IpAddress outsideIp = null;
List<IPAddressVO> publicIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), null);
for (IPAddressVO ip : publicIps) {
if (!ip.isSourceNat()) {
outsideIp = ip;
break;
}
}
if (outsideIp == null) { // none available, acquire one
try {
Account caller = CallContext.current().getCallingAccount();
long callerUserId = CallContext.current().getCallingUserId();
outsideIp = _ipAddrMgr.allocateIp(owner, false, caller, callerUserId, zone);
} catch (ResourceAllocationException e) {
s_logger.error("Unable to allocate additional public Ip address. Exception details " + e);
return false;
}
try {
outsideIp =
_ipAddrMgr.associateIPToGuestNetwork(outsideIp.getId(), network.getId(), true);
} catch (ResourceAllocationException e) {
s_logger.error(
"Unable to assign allocated additional public Ip "
+ outsideIp.getAddress().addr()
+ " to network with vlan "
+ vlanId
+ ". Exception details "
+ e);
return false;
}
}
// create logical edge firewall in VNMC
String gatewayNetmask = NetUtils.getCidrNetmask(network.getCidr());
// due to ASA limitation of allowing single subnet to be assigned to firewall interfaces,
// all public ip addresses must be from same subnet, this essentially means single public
// subnet in zone
if (!createLogicalEdgeFirewall(
vlanId,
network.getGateway(),
gatewayNetmask,
outsideIp.getAddress().addr(),
sourceNatIp.getNetmask(),
publicGateways,
ciscoVnmcHost.getId())) {
s_logger.error(
"Failed to create logical edge firewall in Cisco VNMC device for network "
+ network.getName());
return false;
}
// create stuff in VSM for ASA device
if (!configureNexusVsmForAsa(
vlanId,
network.getGateway(),
vsmDevice.getUserName(),
vsmDevice.getPassword(),
vsmDevice.getipaddr(),
assignedAsa.getInPortProfile(),
ciscoVnmcHost.getId())) {
s_logger.error(
"Failed to configure Cisco Nexus VSM "
+ vsmDevice.getipaddr()
+ " for ASA device for network "
+ network.getName());
return false;
}
// configure source NAT
if (!configureSourceNat(vlanId, network.getCidr(), sourceNatIp, ciscoVnmcHost.getId())) {
s_logger.error(
"Failed to configure source NAT in Cisco VNMC device for network " + network.getName());
return false;
}
// associate Asa 1000v instance with logical edge firewall
if (!associateAsaWithLogicalEdgeFirewall(
vlanId, assignedAsa.getManagementIp(), ciscoVnmcHost.getId())) {
s_logger.error(
"Failed to associate Cisco ASA 1000v ("
+ assignedAsa.getManagementIp()
+ ") with logical edge firewall in VNMC for network "
+ network.getName());
return false;
}
status = true;
txn.commit();
} finally {
if (!status) {
txn.rollback();
// FIXME: also undo changes in VNMC, VSM if anything failed
}
}
return true;
}
@DB
@Override
public boolean releaseVmCapacity(
VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId) {
ServiceOfferingVO svo = _offeringsDao.findById(vm.getServiceOfferingId());
CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, CapacityVO.CAPACITY_TYPE_CPU);
CapacityVO capacityMemory =
_capacityDao.findByHostIdType(hostId, CapacityVO.CAPACITY_TYPE_MEMORY);
Long clusterId = null;
if (hostId != null) {
HostVO host = _hostDao.findById(hostId);
clusterId = host.getClusterId();
}
if (capacityCpu == null || capacityMemory == null || svo == null) {
return false;
}
Transaction txn = Transaction.currentTxn();
try {
txn.start();
capacityCpu = _capacityDao.lockRow(capacityCpu.getId(), true);
capacityMemory = _capacityDao.lockRow(capacityMemory.getId(), true);
long usedCpu = capacityCpu.getUsedCapacity();
long usedMem = capacityMemory.getUsedCapacity();
long reservedCpu = capacityCpu.getReservedCapacity();
long reservedMem = capacityMemory.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
float cpuOvercommitRatio =
Float.parseFloat(
_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio").getValue());
float memoryOvercommitRatio =
Float.parseFloat(
_clusterDetailsDao.findDetail(clusterId, "memoryOvercommitRatio").getValue());
int vmCPU = (int) (svo.getCpu() * svo.getSpeed());
long vmMem = (long) (svo.getRamSize() * 1024L * 1024L);
long actualTotalMem = capacityMemory.getTotalCapacity();
long totalMem = (long) (actualTotalMem * memoryOvercommitRatio);
long totalCpu = (long) (actualTotalCpu * cpuOvercommitRatio);
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Hosts's actual total CPU: "
+ actualTotalCpu
+ " and CPU after applying overprovisioning: "
+ totalCpu);
s_logger.debug(
"Hosts's actual total RAM: "
+ actualTotalMem
+ " and RAM after applying overprovisioning: "
+ totalMem);
}
if (!moveFromReserved) {
/* move resource from used */
if (usedCpu >= vmCPU) {
capacityCpu.setUsedCapacity(usedCpu - vmCPU);
}
if (usedMem >= vmMem) {
capacityMemory.setUsedCapacity(usedMem - vmMem);
}
if (moveToReservered) {
if (reservedCpu + vmCPU <= totalCpu) {
capacityCpu.setReservedCapacity(reservedCpu + vmCPU);
}
if (reservedMem + vmMem <= totalMem) {
capacityMemory.setReservedCapacity(reservedMem + vmMem);
}
}
} else {
if (reservedCpu >= vmCPU) {
capacityCpu.setReservedCapacity(reservedCpu - vmCPU);
}
if (reservedMem >= vmMem) {
capacityMemory.setReservedCapacity(reservedMem - vmMem);
}
}
s_logger.debug(
"release cpu from host: "
+ hostId
+ ", old used: "
+ usedCpu
+ ",reserved: "
+ reservedCpu
+ ", actual total: "
+ actualTotalCpu
+ ", total with overprovisioning: "
+ totalCpu
+ "; new used: "
+ capacityCpu.getUsedCapacity()
+ ",reserved:"
+ capacityCpu.getReservedCapacity()
+ "; movedfromreserved: "
+ moveFromReserved
+ ",moveToReservered"
+ moveToReservered);
s_logger.debug(
"release mem from host: "
+ hostId
+ ", old used: "
+ usedMem
+ ",reserved: "
+ reservedMem
+ ", total: "
+ totalMem
+ "; new used: "
+ capacityMemory.getUsedCapacity()
+ ",reserved:"
+ capacityMemory.getReservedCapacity()
+ "; movedfromreserved: "
+ moveFromReserved
+ ",moveToReservered"
+ moveToReservered);
_capacityDao.update(capacityCpu.getId(), capacityCpu);
_capacityDao.update(capacityMemory.getId(), capacityMemory);
txn.commit();
return true;
} catch (Exception e) {
s_logger.debug("Failed to transit vm's state, due to " + e.getMessage());
txn.rollback();
return false;
}
}
@DB
@Override
public void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost) {
long hostId = vm.getHostId();
HostVO host = _hostDao.findById(hostId);
long clusterId = host.getClusterId();
float cpuOvercommitRatio =
Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio").getValue());
float memoryOvercommitRatio =
Float.parseFloat(
_clusterDetailsDao.findDetail(clusterId, "memoryOvercommitRatio").getValue());
ServiceOfferingVO svo = _offeringsDao.findById(vm.getServiceOfferingId());
CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, CapacityVO.CAPACITY_TYPE_CPU);
CapacityVO capacityMem = _capacityDao.findByHostIdType(hostId, CapacityVO.CAPACITY_TYPE_MEMORY);
if (capacityCpu == null || capacityMem == null || svo == null) {
return;
}
int cpu = (int) (svo.getCpu() * svo.getSpeed());
long ram = (long) (svo.getRamSize() * 1024L * 1024L);
Transaction txn = Transaction.currentTxn();
try {
txn.start();
capacityCpu = _capacityDao.lockRow(capacityCpu.getId(), true);
capacityMem = _capacityDao.lockRow(capacityMem.getId(), true);
long usedCpu = capacityCpu.getUsedCapacity();
long usedMem = capacityMem.getUsedCapacity();
long reservedCpu = capacityCpu.getReservedCapacity();
long reservedMem = capacityMem.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
long actualTotalMem = capacityMem.getTotalCapacity();
long totalCpu = (long) (actualTotalCpu * cpuOvercommitRatio);
long totalMem = (long) (actualTotalMem * memoryOvercommitRatio);
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Hosts's actual total CPU: "
+ actualTotalCpu
+ " and CPU after applying overprovisioning: "
+ totalCpu);
}
long freeCpu = totalCpu - (reservedCpu + usedCpu);
long freeMem = totalMem - (reservedMem + usedMem);
if (s_logger.isDebugEnabled()) {
s_logger.debug("We are allocating VM, increasing the used capacity of this host:" + hostId);
s_logger.debug(
"Current Used CPU: " + usedCpu + " , Free CPU:" + freeCpu + " ,Requested CPU: " + cpu);
s_logger.debug(
"Current Used RAM: " + usedMem + " , Free RAM:" + freeMem + " ,Requested RAM: " + ram);
}
capacityCpu.setUsedCapacity(usedCpu + cpu);
capacityMem.setUsedCapacity(usedMem + ram);
if (fromLastHost) {
/* alloc from reserved */
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"We are allocating VM to the last host again, so adjusting the reserved capacity if it is not less than required");
s_logger.debug("Reserved CPU: " + reservedCpu + " , Requested CPU: " + cpu);
s_logger.debug("Reserved RAM: " + reservedMem + " , Requested RAM: " + ram);
}
if (reservedCpu >= cpu && reservedMem >= ram) {
capacityCpu.setReservedCapacity(reservedCpu - cpu);
capacityMem.setReservedCapacity(reservedMem - ram);
}
} else {
/* alloc from free resource */
if (!((reservedCpu + usedCpu + cpu <= totalCpu)
&& (reservedMem + usedMem + ram <= totalMem))) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Host doesnt seem to have enough free capacity, but increasing the used capacity anyways, since the VM is already starting on this host ");
}
}
}
s_logger.debug(
"CPU STATS after allocation: for host: "
+ hostId
+ ", old used: "
+ usedCpu
+ ", old reserved: "
+ reservedCpu
+ ", actual total: "
+ actualTotalCpu
+ ", total with overprovisioning: "
+ totalCpu
+ "; new used:"
+ capacityCpu.getUsedCapacity()
+ ", reserved:"
+ capacityCpu.getReservedCapacity()
+ "; requested cpu:"
+ cpu
+ ",alloc_from_last:"
+ fromLastHost);
s_logger.debug(
"RAM STATS after allocation: for host: "
+ hostId
+ ", old used: "
+ usedMem
+ ", old reserved: "
+ reservedMem
+ ", total: "
+ totalMem
+ "; new used: "
+ capacityMem.getUsedCapacity()
+ ", reserved: "
+ capacityMem.getReservedCapacity()
+ "; requested mem: "
+ ram
+ ",alloc_from_last:"
+ fromLastHost);
_capacityDao.update(capacityCpu.getId(), capacityCpu);
_capacityDao.update(capacityMem.getId(), capacityMem);
txn.commit();
} catch (Exception e) {
txn.rollback();
return;
}
}
@Override
public boolean applyStaticNats(Network network, List<? extends StaticNat> rules)
throws ResourceUnavailableException {
if (!_networkModel.isProviderSupportServiceInNetwork(
network.getId(), Service.StaticNat, Provider.CiscoVnmc)) {
s_logger.error(
"Static NAT service is not provided by Cisco Vnmc device on network "
+ network.getName());
return false;
}
// Find VNMC host for physical network
List<CiscoVnmcControllerVO> devices =
_ciscoVnmcDao.listByPhysicalNetwork(network.getPhysicalNetworkId());
if (devices.isEmpty()) {
s_logger.error("No Cisco Vnmc device on network " + network.getName());
return true;
}
// Find if ASA 1000v is associated with network
NetworkAsa1000vMapVO asaForNetwork = _networkAsa1000vMapDao.findByNetworkId(network.getId());
if (asaForNetwork == null) {
s_logger.debug("Cisco ASA 1000v device is not associated with network " + network.getName());
return true;
}
if (network.getState() == Network.State.Allocated) {
s_logger.debug(
"External firewall was asked to apply static NAT rules for network with ID "
+ network.getId()
+ "; this network is not implemented. Skipping backend commands.");
return true;
}
CiscoVnmcControllerVO ciscoVnmcDevice = devices.get(0);
HostVO ciscoVnmcHost = _hostDao.findById(ciscoVnmcDevice.getHostId());
List<StaticNatRuleTO> rulesTO = new ArrayList<StaticNatRuleTO>();
for (StaticNat rule : rules) {
IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId());
StaticNatRuleTO ruleTO =
new StaticNatRuleTO(
rule.getSourceIpAddressId(),
sourceIp.getAddress().addr(),
null,
null,
rule.getDestIpAddress(),
null,
null,
null,
rule.isForRevoke(),
false);
rulesTO.add(ruleTO);
}
if (!rulesTO.isEmpty()) {
SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, null);
cmd.setContextParam(
NetworkElementCommand.GUEST_VLAN_TAG, network.getBroadcastUri().getHost());
cmd.setContextParam(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr());
Answer answer = _agentMgr.easySend(ciscoVnmcHost.getId(), cmd);
if (answer == null || !answer.getResult()) {
String details = (answer != null) ? answer.getDetails() : "details unavailable";
String msg =
"Unable to apply static NAT rules to Cisco ASA 1000v appliance due to: "
+ details
+ ".";
s_logger.error(msg);
throw new ResourceUnavailableException(msg, DataCenter.class, network.getDataCenterId());
}
}
return true;
}