@Override
public void run() {
try {
List<VolumeVO> volumes = _volsDao.listAll();
Map<Long, List<VolumeCommand>> commandsByPool = new HashMap<Long, List<VolumeCommand>>();
for (VolumeVO volume : volumes) {
List<VolumeCommand> commands = commandsByPool.get(volume.getPoolId());
if (commands == null) {
commands = new ArrayList<VolumeCommand>();
commandsByPool.put(volume.getPoolId(), commands);
}
VolumeCommand vCommand = new VolumeCommand();
vCommand.volumeId = volume.getId();
vCommand.command = new GetFileStatsCommand(volume);
commands.add(vCommand);
}
ConcurrentHashMap<Long, VolumeStats> volumeStats =
new ConcurrentHashMap<Long, VolumeStats>();
for (Iterator<Long> iter = commandsByPool.keySet().iterator(); iter.hasNext(); ) {
Long poolId = iter.next();
List<VolumeCommand> commandsList = commandsByPool.get(poolId);
long[] volumeIdArray = new long[commandsList.size()];
Commands commands = new Commands(OnError.Continue);
for (int i = 0; i < commandsList.size(); i++) {
VolumeCommand vCommand = commandsList.get(i);
volumeIdArray[i] = vCommand.volumeId;
commands.addCommand(vCommand.command);
}
List<StoragePoolHostVO> poolhosts = _storagePoolHostDao.listByPoolId(poolId);
for (StoragePoolHostVO poolhost : poolhosts) {
Answer[] answers = _agentMgr.send(poolhost.getHostId(), commands);
if (answers != null) {
long totalBytes = 0L;
for (int i = 0; i < answers.length; i++) {
if (answers[i].getResult()) {
VolumeStats vStats = (VolumeStats) answers[i];
volumeStats.put(volumeIdArray[i], vStats);
totalBytes += vStats.getBytesUsed();
}
}
break;
}
}
}
// We replace the existing volumeStats so that it does not grow with no bounds
_volumeStats = volumeStats;
} catch (AgentUnavailableException e) {
s_logger.debug(e.getMessage());
} catch (Throwable t) {
s_logger.error("Error trying to retrieve volume stats", t);
}
}
private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds)
throws AgentUnavailableException {
Answer[] answers = null;
try {
answers = _agentMgr.send(elbVm.getHostId(), cmds);
} catch (OperationTimedoutException e) {
s_logger.warn("ELB: Timed Out", e);
throw new AgentUnavailableException(
"Unable to send commands to virtual elbVm ", elbVm.getHostId(), e);
}
if (answers == null) {
return false;
}
if (answers.length != cmds.size()) {
return false;
}
// FIXME: Have to return state for individual command in the future
if (answers.length > 0) {
Answer ans = answers[0];
return ans.getResult();
}
return true;
}
@Override
public boolean finalizeCommandsOnStart(
Commands cmds, VirtualMachineProfile<SecondaryStorageVmVO> profile) {
NicProfile managementNic = null;
NicProfile controlNic = null;
for (NicProfile nic : profile.getNics()) {
if (nic.getTrafficType() == TrafficType.Management) {
managementNic = nic;
} else if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) {
controlNic = nic;
}
}
if (controlNic == null) {
if (managementNic == null) {
s_logger.error(
"Management network doesn't exist for the secondaryStorageVm "
+ profile.getVirtualMachine());
return false;
}
controlNic = managementNic;
}
CheckSshCommand check =
new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922, 5, 20);
cmds.addCommand("checkSsh", check);
return true;
}
protected boolean setupVpcGuestNetwork(
final Network network,
final VirtualRouter router,
final boolean add,
final NicProfile guestNic)
throws ConcurrentOperationException, ResourceUnavailableException {
boolean result = true;
if (router.getState() == State.Running) {
final SetupGuestNetworkCommand setupCmd =
_commandSetupHelper.createSetupGuestNetworkCommand(
(DomainRouterVO) router, add, guestNic);
final Commands cmds = new Commands(Command.OnError.Stop);
cmds.addCommand("setupguestnetwork", setupCmd);
_nwHelper.sendCommandsToRouter(router, cmds);
final Answer setupAnswer = cmds.getAnswer("setupguestnetwork");
final String setup = add ? "set" : "destroy";
if (!(setupAnswer != null && setupAnswer.getResult())) {
s_logger.warn("Unable to " + setup + " guest network on router " + router);
result = false;
}
return result;
} else if (router.getState() == State.Stopped || router.getState() == State.Stopping) {
s_logger.debug(
"Router "
+ router.getInstanceName()
+ " is in "
+ router.getState()
+ ", so not sending setup guest network command to the backend");
return true;
} else {
s_logger.warn(
"Unable to setup guest network on virtual router "
+ router
+ " is not in the right state "
+ router.getState());
throw new ResourceUnavailableException(
"Unable to setup guest network on the backend,"
+ " virtual router "
+ router
+ " is not in the right state",
DataCenter.class,
router.getDataCenterId());
}
}
@Override
public boolean finalizeStart(
VirtualMachineProfile<DomainRouterVO> profile,
long hostId,
Commands cmds,
ReservationContext context) {
CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh");
if (answer == null || !answer.getResult()) {
s_logger.warn("Unable to ssh to the ELB VM: " + answer.getDetails());
return false;
}
return true;
}
private void createApplyLoadBalancingRulesCommands(
List<LoadBalancingRule> rules, DomainRouterVO elbVm, Commands cmds, long guestNetworkId) {
LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()];
int i = 0;
for (LoadBalancingRule rule : rules) {
boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke));
String protocol = rule.getProtocol();
String algorithm = rule.getAlgorithm();
String elbIp = _networkModel.getIp(rule.getSourceIpAddressId()).getAddress().addr();
int srcPort = rule.getSourcePortStart();
String uuid = rule.getUuid();
List<LbDestination> destinations = rule.getDestinations();
LoadBalancerTO lb =
new LoadBalancerTO(
uuid, elbIp, srcPort, protocol, algorithm, revoked, false, false, destinations);
lbs[i++] = lb;
}
LoadBalancerConfigCommand cmd =
new LoadBalancerConfigCommand(
lbs,
elbVm.getPublicIpAddress(),
_nicDao.getIpAddress(guestNetworkId, elbVm.getId()),
elbVm.getPrivateIpAddress(),
null,
null);
cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, elbVm.getPrivateIpAddress());
cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, elbVm.getInstanceName());
// FIXME: why are we setting attributes directly? Ick!! There should be accessors and
// the constructor should set defaults.
cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key());
cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key());
cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key());
cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key());
cmds.addCommand(cmd);
}
@Override
public boolean sendCommandsToRouter(final VirtualRouter router, final Commands cmds)
throws AgentUnavailableException, ResourceUnavailableException {
if (!checkRouterVersion(router)) {
s_logger.debug(
"Router requires upgrade. Unable to send command to router:"
+ router.getId()
+ ", router template version : "
+ router.getTemplateVersion()
+ ", minimal required version : "
+ NetworkOrchestrationService.MinVRVersion.valueIn(router.getDataCenterId()));
throw new ResourceUnavailableException(
"Unable to send command. Router requires upgrade", VirtualRouter.class, router.getId());
}
Answer[] answers = null;
try {
answers = _agentMgr.send(router.getHostId(), cmds);
} catch (final OperationTimedoutException e) {
s_logger.warn("Timed Out", e);
throw new AgentUnavailableException(
"Unable to send commands to virtual router ", router.getHostId(), e);
}
if (answers == null || answers.length != cmds.size()) {
return false;
}
// FIXME: Have to return state for individual command in the future
boolean result = true;
for (final Answer answer : answers) {
if (!answer.getResult()) {
result = false;
break;
}
}
return result;
}
@Override
public boolean finalizeCommandsOnStart(
Commands cmds, VirtualMachineProfile<DomainRouterVO> profile) {
DomainRouterVO elbVm = profile.getVirtualMachine();
DataCenterVO dcVo = _dcDao.findById(elbVm.getDataCenterId());
NicProfile controlNic = null;
Long guestNetworkId = null;
if (profile.getHypervisorType() == HypervisorType.VMware
&& dcVo.getNetworkType() == NetworkType.Basic) {
// TODO this is a ugly to test hypervisor type here
// for basic network mode, we will use the guest NIC for control NIC
for (NicProfile nic : profile.getNics()) {
if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) {
controlNic = nic;
guestNetworkId = nic.getNetworkId();
}
}
} else {
for (NicProfile nic : profile.getNics()) {
if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) {
controlNic = nic;
} else if (nic.getTrafficType() == TrafficType.Guest) {
guestNetworkId = nic.getNetworkId();
}
}
}
if (controlNic == null) {
s_logger.error("Control network doesn't exist for the ELB vm " + elbVm);
return false;
}
cmds.addCommand(
"checkSsh",
new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922));
// Re-apply load balancing rules
List<LoadBalancerVO> lbs = _elbVmMapDao.listLbsForElbVm(elbVm.getId());
List<LoadBalancingRule> lbRules = new ArrayList<LoadBalancingRule>();
for (LoadBalancerVO lb : lbs) {
List<LbDestination> dstList = _lbMgr.getExistingDestinations(lb.getId());
List<LbStickinessPolicy> policyList = _lbMgr.getStickinessPolicies(lb.getId());
List<LbHealthCheckPolicy> hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId());
LoadBalancingRule loadBalancing =
new LoadBalancingRule(lb, dstList, policyList, hcPolicyList);
lbRules.add(loadBalancing);
}
s_logger.debug(
"Found "
+ lbRules.size()
+ " load balancing rule(s) to apply as a part of ELB vm "
+ elbVm
+ " start.");
if (!lbRules.isEmpty()) {
createApplyLoadBalancingRulesCommands(lbRules, elbVm, cmds, guestNetworkId);
}
return true;
}
@Override
public boolean startRemoteAccessVpn(final RemoteAccessVpn vpn, final VirtualRouter router)
throws ResourceUnavailableException {
if (router.getState() != State.Running) {
s_logger.warn(
"Unable to apply remote access VPN configuration, virtual router is not in the right state "
+ router.getState());
throw new ResourceUnavailableException(
"Unable to apply remote access VPN configuration,"
+ " virtual router is not in the right state",
DataCenter.class,
router.getDataCenterId());
}
final Commands cmds = new Commands(Command.OnError.Stop);
_commandSetupHelper.createApplyVpnCommands(true, vpn, router, cmds);
try {
_agentMgr.send(router.getHostId(), cmds);
} catch (final OperationTimedoutException e) {
s_logger.debug("Failed to start remote access VPN: ", e);
throw new AgentUnavailableException(
"Unable to send commands to virtual router ", router.getHostId(), e);
}
Answer answer = cmds.getAnswer("users");
if (!answer.getResult()) {
s_logger.error(
"Unable to start vpn: unable add users to vpn in zone "
+ router.getDataCenterId()
+ " for account "
+ vpn.getAccountId()
+ " on domR: "
+ router.getInstanceName()
+ " due to "
+ answer.getDetails());
throw new ResourceUnavailableException(
"Unable to start vpn: Unable to add users to vpn in zone "
+ router.getDataCenterId()
+ " for account "
+ vpn.getAccountId()
+ " on domR: "
+ router.getInstanceName()
+ " due to "
+ answer.getDetails(),
DataCenter.class,
router.getDataCenterId());
}
answer = cmds.getAnswer("startVpn");
if (!answer.getResult()) {
s_logger.error(
"Unable to start vpn in zone "
+ router.getDataCenterId()
+ " for account "
+ vpn.getAccountId()
+ " on domR: "
+ router.getInstanceName()
+ " due to "
+ answer.getDetails());
throw new ResourceUnavailableException(
"Unable to start vpn in zone "
+ router.getDataCenterId()
+ " for account "
+ vpn.getAccountId()
+ " on domR: "
+ router.getInstanceName()
+ " due to "
+ answer.getDetails(),
DataCenter.class,
router.getDataCenterId());
}
return true;
}
@Override
public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachineProfile profile) {
final DomainRouterVO domainRouterVO = _routerDao.findById(profile.getId());
final boolean isVpc = domainRouterVO.getVpcId() != null;
if (!isVpc) {
return super.finalizeCommandsOnStart(cmds, profile);
}
if (domainRouterVO.getState() == State.Starting || domainRouterVO.getState() == State.Running) {
// 1) FORM SSH CHECK COMMAND
final NicProfile controlNic = getControlNic(profile);
if (controlNic == null) {
s_logger.error("Control network doesn't exist for the router " + domainRouterVO);
return false;
}
finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, domainRouterVO, controlNic);
// 2) FORM PLUG NIC COMMANDS
final List<Pair<Nic, Network>> guestNics = new ArrayList<Pair<Nic, Network>>();
final List<Pair<Nic, Network>> publicNics = new ArrayList<Pair<Nic, Network>>();
final Map<String, String> vlanMacAddress = new HashMap<String, String>();
final List<? extends Nic> routerNics = _nicDao.listByVmId(profile.getId());
for (final Nic routerNic : routerNics) {
final Network network = _networkModel.getNetwork(routerNic.getNetworkId());
if (network.getTrafficType() == TrafficType.Guest) {
final Pair<Nic, Network> guestNic = new Pair<Nic, Network>(routerNic, network);
guestNics.add(guestNic);
} else if (network.getTrafficType() == TrafficType.Public) {
final Pair<Nic, Network> publicNic = new Pair<Nic, Network>(routerNic, network);
publicNics.add(publicNic);
final String vlanTag = BroadcastDomainType.getValue(routerNic.getBroadcastUri());
vlanMacAddress.put(vlanTag, routerNic.getMacAddress());
}
}
final List<Command> usageCmds = new ArrayList<Command>();
// 3) PREPARE PLUG NIC COMMANDS
try {
// add VPC router to public networks
final List<PublicIp> sourceNat = new ArrayList<PublicIp>(1);
for (final Pair<Nic, Network> nicNtwk : publicNics) {
final Nic publicNic = nicNtwk.first();
final Network publicNtwk = nicNtwk.second();
final IPAddressVO userIp =
_ipAddressDao.findByIpAndSourceNetworkId(
publicNtwk.getId(), publicNic.getIPv4Address());
if (userIp.isSourceNat()) {
final PublicIp publicIp =
PublicIp.createFromAddrAndVlan(userIp, _vlanDao.findById(userIp.getVlanId()));
sourceNat.add(publicIp);
if (domainRouterVO.getPublicIpAddress() == null) {
final DomainRouterVO routerVO = _routerDao.findById(domainRouterVO.getId());
routerVO.setPublicIpAddress(publicNic.getIPv4Address());
routerVO.setPublicNetmask(publicNic.getIPv4Netmask());
routerVO.setPublicMacAddress(publicNic.getMacAddress());
_routerDao.update(routerVO.getId(), routerVO);
}
}
final PlugNicCommand plugNicCmd =
new PlugNicCommand(
_nwHelper.getNicTO(
domainRouterVO,
publicNic.getNetworkId(),
publicNic.getBroadcastUri().toString()),
domainRouterVO.getInstanceName(),
domainRouterVO.getType());
cmds.addCommand(plugNicCmd);
final VpcVO vpc = _vpcDao.findById(domainRouterVO.getVpcId());
final NetworkUsageCommand netUsageCmd =
new NetworkUsageCommand(
domainRouterVO.getPrivateIpAddress(),
domainRouterVO.getInstanceName(),
true,
publicNic.getIPv4Address(),
vpc.getCidr());
usageCmds.add(netUsageCmd);
UserStatisticsVO stats =
_userStatsDao.findBy(
domainRouterVO.getAccountId(),
domainRouterVO.getDataCenterId(),
publicNtwk.getId(),
publicNic.getIPv4Address(),
domainRouterVO.getId(),
domainRouterVO.getType().toString());
if (stats == null) {
stats =
new UserStatisticsVO(
domainRouterVO.getAccountId(),
domainRouterVO.getDataCenterId(),
publicNic.getIPv4Address(),
domainRouterVO.getId(),
domainRouterVO.getType().toString(),
publicNtwk.getId());
_userStatsDao.persist(stats);
}
}
// create ip assoc for source nat
if (!sourceNat.isEmpty()) {
_commandSetupHelper.createVpcAssociatePublicIPCommands(
domainRouterVO, sourceNat, cmds, vlanMacAddress);
}
// add VPC router to guest networks
for (final Pair<Nic, Network> nicNtwk : guestNics) {
final Nic guestNic = nicNtwk.first();
// plug guest nic
final PlugNicCommand plugNicCmd =
new PlugNicCommand(
_nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null),
domainRouterVO.getInstanceName(),
domainRouterVO.getType());
cmds.addCommand(plugNicCmd);
if (!_networkModel.isPrivateGateway(guestNic.getNetworkId())) {
// set guest network
final VirtualMachine vm = _vmDao.findById(domainRouterVO.getId());
final NicProfile nicProfile =
_networkModel.getNicProfile(vm, guestNic.getNetworkId(), null);
final SetupGuestNetworkCommand setupCmd =
_commandSetupHelper.createSetupGuestNetworkCommand(
domainRouterVO, true, nicProfile);
cmds.addCommand(setupCmd);
} else {
// set private network
final PrivateIpVO ipVO =
_privateIpDao.findByIpAndSourceNetworkId(
guestNic.getNetworkId(), guestNic.getIPv4Address());
final Network network = _networkDao.findById(guestNic.getNetworkId());
BroadcastDomainType.getValue(network.getBroadcastUri());
final String netmask = NetUtils.getCidrNetmask(network.getCidr());
final PrivateIpAddress ip =
new PrivateIpAddress(
ipVO,
network.getBroadcastUri().toString(),
network.getGateway(),
netmask,
guestNic.getMacAddress());
final List<PrivateIpAddress> privateIps = new ArrayList<PrivateIpAddress>(1);
privateIps.add(ip);
_commandSetupHelper.createVpcAssociatePrivateIPCommands(
domainRouterVO, privateIps, cmds, true);
final Long privateGwAclId =
_vpcGatewayDao.getNetworkAclIdForPrivateIp(
ipVO.getVpcId(), ipVO.getNetworkId(), ipVO.getIpAddress());
if (privateGwAclId != null) {
// set network acl on private gateway
final List<NetworkACLItemVO> networkACLs =
_networkACLItemDao.listByACL(privateGwAclId);
s_logger.debug(
"Found "
+ networkACLs.size()
+ " network ACLs to apply as a part of VPC VR "
+ domainRouterVO
+ " start for private gateway ip = "
+ ipVO.getIpAddress());
_commandSetupHelper.createNetworkACLsCommands(
networkACLs, domainRouterVO, cmds, ipVO.getNetworkId(), true);
}
}
}
} catch (final Exception ex) {
s_logger.warn(
"Failed to add router " + domainRouterVO + " to network due to exception ", ex);
return false;
}
// 4) RE-APPLY ALL STATIC ROUTE RULES
final List<? extends StaticRoute> routes =
_staticRouteDao.listByVpcId(domainRouterVO.getVpcId());
final List<StaticRouteProfile> staticRouteProfiles =
new ArrayList<StaticRouteProfile>(routes.size());
final Map<Long, VpcGateway> gatewayMap = new HashMap<Long, VpcGateway>();
for (final StaticRoute route : routes) {
VpcGateway gateway = gatewayMap.get(route.getVpcGatewayId());
if (gateway == null) {
gateway = _entityMgr.findById(VpcGateway.class, route.getVpcGatewayId());
gatewayMap.put(gateway.getId(), gateway);
}
staticRouteProfiles.add(new StaticRouteProfile(route, gateway));
}
s_logger.debug(
"Found "
+ staticRouteProfiles.size()
+ " static routes to apply as a part of vpc route "
+ domainRouterVO
+ " start");
if (!staticRouteProfiles.isEmpty()) {
_commandSetupHelper.createStaticRouteCommands(staticRouteProfiles, domainRouterVO, cmds);
}
// 5) RE-APPLY ALL REMOTE ACCESS VPNs
final RemoteAccessVpnVO vpn =
_vpnDao.findByAccountAndVpc(domainRouterVO.getAccountId(), domainRouterVO.getVpcId());
if (vpn != null) {
_commandSetupHelper.createApplyVpnCommands(true, vpn, domainRouterVO, cmds);
}
// 6) REPROGRAM GUEST NETWORK
boolean reprogramGuestNtwks = true;
if (profile.getParameter(Param.ReProgramGuestNetworks) != null
&& (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) {
reprogramGuestNtwks = false;
}
final VirtualRouterProvider vrProvider =
_vrProviderDao.findById(domainRouterVO.getElementId());
if (vrProvider == null) {
throw new CloudRuntimeException(
"Cannot find related virtual router provider of router: "
+ domainRouterVO.getHostName());
}
final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString());
if (provider == null) {
throw new CloudRuntimeException(
"Cannot find related provider of virtual router provider: "
+ vrProvider.getType().toString());
}
for (final Pair<Nic, Network> nicNtwk : guestNics) {
final Nic guestNic = nicNtwk.first();
final AggregationControlCommand startCmd =
new AggregationControlCommand(
Action.Start,
domainRouterVO.getInstanceName(),
controlNic.getIPv4Address(),
_routerControlHelper.getRouterIpInNetwork(
guestNic.getNetworkId(), domainRouterVO.getId()));
cmds.addCommand(startCmd);
if (reprogramGuestNtwks) {
finalizeIpAssocForNetwork(
cmds, domainRouterVO, provider, guestNic.getNetworkId(), vlanMacAddress);
finalizeNetworkRulesForNetwork(cmds, domainRouterVO, provider, guestNic.getNetworkId());
}
finalizeUserDataAndDhcpOnStart(cmds, domainRouterVO, provider, guestNic.getNetworkId());
final AggregationControlCommand finishCmd =
new AggregationControlCommand(
Action.Finish,
domainRouterVO.getInstanceName(),
controlNic.getIPv4Address(),
_routerControlHelper.getRouterIpInNetwork(
guestNic.getNetworkId(), domainRouterVO.getId()));
cmds.addCommand(finishCmd);
}
// Add network usage commands
cmds.addCommands(usageCmds);
}
return true;
}