@Override
  public boolean generateVMSetupCommand(Long ssAHostId) {
    HostVO ssAHost = _hostDao.findById(ssAHostId);
    if (ssAHost.getType() != Host.Type.SecondaryStorageVM) {
      return false;
    }
    SecondaryStorageVmVO secStorageVm = _secStorageVmDao.findByInstanceName(ssAHost.getName());
    if (secStorageVm == null) {
      s_logger.warn("secondary storage VM " + ssAHost.getName() + " doesn't exist");
      return false;
    }

    SecStorageVMSetupCommand setupCmd = new SecStorageVMSetupCommand();
    if (_allowedInternalSites != null) {
      List<String> allowedCidrs = new ArrayList<String>();
      String[] cidrs = _allowedInternalSites.split(",");
      for (String cidr : cidrs) {
        if (NetUtils.isValidCIDR(cidr) || NetUtils.isValidIp(cidr)) {
          allowedCidrs.add(cidr);
        }
      }
      List<? extends Nic> nics =
          _networkMgr.getNicsForTraffic(secStorageVm.getId(), TrafficType.Management);
      Nic privateNic = nics.get(0);
      String privateCidr =
          NetUtils.ipAndNetMaskToCidr(privateNic.getIp4Address(), privateNic.getNetmask());
      String publicCidr =
          NetUtils.ipAndNetMaskToCidr(
              secStorageVm.getPublicIpAddress(), secStorageVm.getPublicNetmask());
      if (NetUtils.isNetworkAWithinNetworkB(privateCidr, publicCidr)
          || NetUtils.isNetworkAWithinNetworkB(publicCidr, privateCidr)) {
        s_logger.info(
            "private and public interface overlaps, add a default route through private interface. privateCidr: "
                + privateCidr
                + ", publicCidr: "
                + publicCidr);
        allowedCidrs.add(NetUtils.ALL_CIDRS);
      }
      setupCmd.setAllowedInternalSites(allowedCidrs.toArray(new String[allowedCidrs.size()]));
    }
    String copyPasswd = _configDao.getValue("secstorage.copy.password");
    setupCmd.setCopyPassword(copyPasswd);
    setupCmd.setCopyUserName(TemplateConstants.DEFAULT_HTTP_AUTH_USER);
    Answer answer = _agentMgr.easySend(ssAHostId, setupCmd);
    if (answer != null && answer.getResult()) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Successfully programmed http auth into " + secStorageVm.getHostName());
      }
      return true;
    } else {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "failed to program http auth into secondary storage vm : "
                + secStorageVm.getHostName());
      }
      return false;
    }
  }
  @Override
  public Boolean isVmAlive(VirtualMachine vm, Host host) {
    if (vm.getType() != VirtualMachine.Type.User) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Not a User Vm, unable to determine state of " + vm + " returning null");
      }
      return null;
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("testing if " + vm + " is alive");
    }
    // to verify that the VM is alive, we ask the domR (router) to ping the VM (private IP)
    UserVmVO userVm = _userVmDao.findById(vm.getId());

    List<? extends Nic> nics = _networkMgr.getNicsForTraffic(userVm.getId(), TrafficType.Guest);

    for (Nic nic : nics) {
      if (nic.getIp4Address() == null) {
        continue;
      }

      List<VirtualRouter> routers = _vnaMgr.getRoutersForNetwork(nic.getNetworkId());
      if (routers == null || routers.isEmpty()) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug(
              "Unable to find a router in network " + nic.getNetworkId() + " to ping " + vm);
        }
        continue;
      }

      Boolean result = null;
      for (VirtualRouter router : routers) {
        result = testUserVM(vm, nic, router);
        if (result != null) {
          break;
        }
      }

      if (result == null) {
        continue;
      }

      return result;
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Returning null since we're unable to determine state of " + vm);
    }
    return null;
  }
  @Override
  public List<LbDestination> getExistingDestinations(long lbId) {
    List<LbDestination> dstList = new ArrayList<LbDestination>();
    List<LoadBalancerVMMapVO> lbVmMaps = _lb2VmMapDao.listByLoadBalancerId(lbId);
    LoadBalancerVO lb = _lbDao.findById(lbId);

    String dstIp = null;
    for (LoadBalancerVMMapVO lbVmMap : lbVmMaps) {
      UserVm vm = _vmDao.findById(lbVmMap.getInstanceId());
      Nic nic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(lb.getNetworkId(), vm.getId());
      dstIp = nic.getIp4Address();
      LbDestination lbDst =
          new LbDestination(
              lb.getDefaultPortStart(), lb.getDefaultPortEnd(), dstIp, lbVmMap.isRevoke());
      dstList.add(lbDst);
    }
    return dstList;
  }
  protected Map<PortAndProto, Set<String>> generateRulesForVM(Long userVmId) {

    Map<PortAndProto, Set<String>> allowed = new TreeMap<PortAndProto, Set<String>>();

    List<SecurityGroupVMMapVO> groupsForVm = _securityGroupVMMapDao.listByInstanceId(userVmId);
    for (SecurityGroupVMMapVO mapVO : groupsForVm) {
      List<IngressRuleVO> rules = _ingressRuleDao.listBySecurityGroupId(mapVO.getSecurityGroupId());
      for (IngressRuleVO rule : rules) {
        PortAndProto portAndProto =
            new PortAndProto(rule.getProtocol(), rule.getStartPort(), rule.getEndPort());
        Set<String> cidrs = allowed.get(portAndProto);
        if (cidrs == null) {
          cidrs = new TreeSet<String>(new CidrComparator());
        }
        if (rule.getAllowedNetworkId() != null) {
          List<SecurityGroupVMMapVO> allowedInstances =
              _securityGroupVMMapDao.listBySecurityGroup(rule.getAllowedNetworkId(), State.Running);
          for (SecurityGroupVMMapVO ngmapVO : allowedInstances) {
            Nic defaultNic = _networkMgr.getDefaultNic(ngmapVO.getInstanceId());
            if (defaultNic != null) {
              String cidr = defaultNic.getIp4Address();
              cidr = cidr + "/32";
              cidrs.add(cidr);
            }
          }
        } else if (rule.getAllowedSourceIpCidr() != null) {
          cidrs.add(rule.getAllowedSourceIpCidr());
        }
        if (cidrs.size() > 0) {
          allowed.put(portAndProto, cidrs);
        }
      }
    }

    return allowed;
  }
  private Boolean testUserVM(VirtualMachine vm, Nic nic, VirtualRouter router) {
    String privateIp = nic.getIp4Address();
    String routerPrivateIp = router.getPrivateIpAddress();

    List<Long> otherHosts = new ArrayList<Long>();
    if (vm.getHypervisorType() == HypervisorType.XenServer
        || vm.getHypervisorType() == HypervisorType.KVM) {
      otherHosts.add(router.getHostId());
    } else {
      otherHosts = findHostByPod(router.getPodIdToDeployIn(), null);
    }
    for (Long hostId : otherHosts) {
      try {
        Answer pingTestAnswer =
            _agentMgr.easySend(hostId, new PingTestCommand(routerPrivateIp, privateIp));
        if (pingTestAnswer != null && pingTestAnswer.getResult()) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "user vm's "
                    + vm.getHostName()
                    + " ip address "
                    + privateIp
                    + "  has been successfully pinged from the Virtual Router "
                    + router.getHostName()
                    + ", returning that vm is alive");
          }
          return Boolean.TRUE;
        }
      } catch (Exception e) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug("Couldn't reach due to", e);
        }
        continue;
      }
    }
    if (s_logger.isDebugEnabled()) {
      s_logger.debug(vm + " could not be pinged, returning that it is unknown");
    }
    return null;
  }
Example #6
0
  @Override
  public LinkedHashMap<Network, List<? extends NicProfile>> configureGuestNic(
      final RouterDeploymentDefinition routerDeploymentDefinition)
      throws ConcurrentOperationException, InsufficientAddressCapacityException {

    // Form networks
    final LinkedHashMap<Network, List<? extends NicProfile>> networks =
        new LinkedHashMap<Network, List<? extends NicProfile>>(3);
    // 1) Guest network
    final Network guestNetwork = routerDeploymentDefinition.getGuestNetwork();

    if (guestNetwork != null) {
      s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork);
      String defaultNetworkStartIp = null, defaultNetworkStartIpv6 = null;
      if (!routerDeploymentDefinition.isPublicNetwork()) {
        final Nic placeholder =
            _networkModel.getPlaceholderNicForRouter(
                guestNetwork, routerDeploymentDefinition.getPodId());
        if (guestNetwork.getCidr() != null) {
          if (placeholder != null && placeholder.getIPv4Address() != null) {
            s_logger.debug(
                "Requesting ipv4 address "
                    + placeholder.getIPv4Address()
                    + " stored in placeholder nic for the network "
                    + guestNetwork);
            defaultNetworkStartIp = placeholder.getIPv4Address();
          } else {
            final String startIp = _networkModel.getStartIpAddress(guestNetwork.getId());
            if (startIp != null
                && _ipAddressDao
                        .findByIpAndSourceNetworkId(guestNetwork.getId(), startIp)
                        .getAllocatedTime()
                    == null) {
              defaultNetworkStartIp = startIp;
            } else if (s_logger.isDebugEnabled()) {
              s_logger.debug(
                  "First ipv4 "
                      + startIp
                      + " in network id="
                      + guestNetwork.getId()
                      + " is already allocated, can't use it for domain router; will get random ip address from the range");
            }
          }
        }

        if (guestNetwork.getIp6Cidr() != null) {
          if (placeholder != null && placeholder.getIPv6Address() != null) {
            s_logger.debug(
                "Requesting ipv6 address "
                    + placeholder.getIPv6Address()
                    + " stored in placeholder nic for the network "
                    + guestNetwork);
            defaultNetworkStartIpv6 = placeholder.getIPv6Address();
          } else {
            final String startIpv6 = _networkModel.getStartIpv6Address(guestNetwork.getId());
            if (startIpv6 != null
                && _ipv6Dao.findByNetworkIdAndIp(guestNetwork.getId(), startIpv6) == null) {
              defaultNetworkStartIpv6 = startIpv6;
            } else if (s_logger.isDebugEnabled()) {
              s_logger.debug(
                  "First ipv6 "
                      + startIpv6
                      + " in network id="
                      + guestNetwork.getId()
                      + " is already allocated, can't use it for domain router; will get random ipv6 address from the range");
            }
          }
        }
      }

      final NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp, defaultNetworkStartIpv6);
      if (routerDeploymentDefinition.isPublicNetwork()) {
        if (routerDeploymentDefinition.isRedundant()) {
          gatewayNic.setIPv4Address(_ipAddrMgr.acquireGuestIpAddress(guestNetwork, null));
        } else {
          gatewayNic.setIPv4Address(guestNetwork.getGateway());
        }
        gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri());
        gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType());
        gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri());
        gatewayNic.setMode(guestNetwork.getMode());
        final String gatewayCidr = guestNetwork.getCidr();
        gatewayNic.setIPv4Netmask(NetUtils.getCidrNetmask(gatewayCidr));
      } else {
        gatewayNic.setDefaultNic(true);
      }

      networks.put(guestNetwork, new ArrayList<NicProfile>(Arrays.asList(gatewayNic)));
    }
    return networks;
  }
  @Override
  @DB
  public NicProfile createPrivateNicProfileForGateway(
      final VpcGateway privateGateway, final VirtualRouter router) {
    final Network privateNetwork = _networkModel.getNetwork(privateGateway.getNetworkId());

    PrivateIpVO ipVO =
        _privateIpDao.allocateIpAddress(
            privateNetwork.getDataCenterId(),
            privateNetwork.getId(),
            privateGateway.getIp4Address());

    final Long vpcId = privateGateway.getVpcId();
    final Vpc activeVpc = _vpcMgr.getActiveVpc(vpcId);
    if (activeVpc.isRedundant() && ipVO == null) {
      ipVO = _privateIpDao.findByIpAndVpcId(vpcId, privateGateway.getIp4Address());
    }

    Nic privateNic = null;

    if (ipVO != null) {
      privateNic =
          _nicDao.findByIp4AddressAndNetworkId(ipVO.getIpAddress(), privateNetwork.getId());
    }

    NicProfile privateNicProfile = new NicProfile();

    if (privateNic != null) {
      privateNicProfile =
          new NicProfile(
              privateNic,
              privateNetwork,
              privateNic.getBroadcastUri(),
              privateNic.getIsolationUri(),
              _networkModel.getNetworkRate(privateNetwork.getId(), router.getId()),
              _networkModel.isSecurityGroupSupportedInNetwork(privateNetwork),
              _networkModel.getNetworkTag(router.getHypervisorType(), privateNetwork));

      if (router.getIsRedundantRouter()) {
        String newMacAddress =
            NetUtils.long2Mac(NetUtils.createSequenceBasedMacAddress(ipVO.getMacAddress()));
        privateNicProfile.setMacAddress(newMacAddress);
      }
    } else {
      final String netmask = NetUtils.getCidrNetmask(privateNetwork.getCidr());
      final PrivateIpAddress ip =
          new PrivateIpAddress(
              ipVO,
              privateNetwork.getBroadcastUri().toString(),
              privateNetwork.getGateway(),
              netmask,
              NetUtils.long2Mac(NetUtils.createSequenceBasedMacAddress(ipVO.getMacAddress())));

      final URI netUri = BroadcastDomainType.fromString(ip.getBroadcastUri());
      privateNicProfile.setIPv4Address(ip.getIpAddress());
      privateNicProfile.setIPv4Gateway(ip.getGateway());
      privateNicProfile.setIPv4Netmask(ip.getNetmask());
      privateNicProfile.setIsolationUri(netUri);
      privateNicProfile.setBroadcastUri(netUri);
      // can we solve this in setBroadcastUri()???
      // or more plugable construct is desirable
      privateNicProfile.setBroadcastType(BroadcastDomainType.getSchemeValue(netUri));
      privateNicProfile.setFormat(AddressFormat.Ip4);
      privateNicProfile.setReservationId(String.valueOf(ip.getBroadcastUri()));
      privateNicProfile.setMacAddress(ip.getMacAddress());
    }

    return privateNicProfile;
  }
  @Override
  public Boolean isVmAlive(VirtualMachine vm, Host host) {
    if (!vm.getType().isUsedBySystem()) {
      s_logger.debug("Not a System Vm, unable to determine state of " + vm + " returning null");
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Testing if " + vm + " is alive");
    }

    if (vm.getHostId() == null) {
      s_logger.debug("There's no host id for " + vm);
      return null;
    }

    HostVO vmHost = _hostDao.findById(vm.getHostId());
    if (vmHost == null) {
      s_logger.debug("Unable to retrieve the host by using id " + vm.getHostId());
      return null;
    }

    List<? extends Nic> nics = _networkMgr.getNicsForTraffic(vm.getId(), TrafficType.Management);
    if (nics.size() == 0) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "Unable to find a management nic, cannot ping this system VM, unable to determine state of "
                + vm
                + " returning null");
      }
      return null;
    }

    for (Nic nic : nics) {
      if (nic.getIp4Address() == null) {
        continue;
      }
      // get the data center IP address, find a host on the pod, use that host to ping the data
      // center IP address
      List<Long> otherHosts = findHostByPod(vmHost.getPodId(), vm.getHostId());
      for (Long otherHost : otherHosts) {
        Status vmState = testIpAddress(otherHost, nic.getIp4Address());
        if (vmState == null) {
          // can't get information from that host, try the next one
          continue;
        }
        if (vmState == Status.Up) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "successfully pinged vm's private IP ("
                    + vm.getPrivateIpAddress()
                    + "), returning that the VM is up");
          }
          return Boolean.TRUE;
        } else if (vmState == Status.Down) {
          // We can't ping the VM directly...if we can ping the host, then report the VM down.
          // If we can't ping the host, then we don't have enough information.
          Status vmHostState = testIpAddress(otherHost, vmHost.getPrivateIpAddress());
          if ((vmHostState != null) && (vmHostState == Status.Up)) {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug(
                  "successfully pinged vm's host IP ("
                      + vmHost.getPrivateIpAddress()
                      + "), but could not ping VM, returning that the VM is down");
            }
            return Boolean.FALSE;
          }
        }
      }
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("unable to determine state of " + vm + " returning null");
    }
    return null;
  }
  @Override
  @DB
  @ActionEvent(
      eventType = EventTypes.EVENT_ASSIGN_TO_LOAD_BALANCER_RULE,
      eventDescription = "assigning to load balancer",
      async = true)
  public boolean assignToLoadBalancer(long loadBalancerId, List<Long> instanceIds) {
    UserContext ctx = UserContext.current();
    Account caller = ctx.getCaller();

    LoadBalancerVO loadBalancer = _lbDao.findById(loadBalancerId);
    if (loadBalancer == null) {
      throw new InvalidParameterValueException(
          "Failed to assign to load balancer "
              + loadBalancerId
              + ", the load balancer was not found.");
    }

    List<LoadBalancerVMMapVO> mappedInstances =
        _lb2VmMapDao.listByLoadBalancerId(loadBalancerId, false);
    Set<Long> mappedInstanceIds = new HashSet<Long>();
    for (LoadBalancerVMMapVO mappedInstance : mappedInstances) {
      mappedInstanceIds.add(Long.valueOf(mappedInstance.getInstanceId()));
    }

    List<UserVm> vmsToAdd = new ArrayList<UserVm>();

    for (Long instanceId : instanceIds) {
      if (mappedInstanceIds.contains(instanceId)) {
        throw new InvalidParameterValueException(
            "VM " + instanceId + " is already mapped to load balancer.");
      }

      UserVm vm = _vmDao.findById(instanceId);
      if (vm == null || vm.getState() == State.Destroyed || vm.getState() == State.Expunging) {
        throw new InvalidParameterValueException("Invalid instance id: " + instanceId);
      }

      _rulesMgr.checkRuleAndUserVm(loadBalancer, vm, caller);

      if (vm.getAccountId() != loadBalancer.getAccountId()) {
        throw new PermissionDeniedException(
            "Cannot add virtual machines that do not belong to the same owner.");
      }

      // Let's check to make sure the vm has a nic in the same network as the load balancing rule.
      List<? extends Nic> nics = _networkMgr.getNics(vm.getId());
      Nic nicInSameNetwork = null;
      for (Nic nic : nics) {
        if (nic.getNetworkId() == loadBalancer.getNetworkId()) {
          nicInSameNetwork = nic;
          break;
        }
      }

      if (nicInSameNetwork == null) {
        throw new InvalidParameterValueException(
            "VM " + instanceId + " cannot be added because it doesn't belong in the same network.");
      }

      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Adding " + vm + " to the load balancer pool");
      }
      vmsToAdd.add(vm);
    }

    Transaction txn = Transaction.currentTxn();
    txn.start();

    for (UserVm vm : vmsToAdd) {
      LoadBalancerVMMapVO map = new LoadBalancerVMMapVO(loadBalancer.getId(), vm.getId(), false);
      map = _lb2VmMapDao.persist(map);
    }
    txn.commit();

    try {
      loadBalancer.setState(FirewallRule.State.Add);
      _lbDao.persist(loadBalancer);
      applyLoadBalancerConfig(loadBalancerId);
    } catch (ResourceUnavailableException e) {
      s_logger.warn("Unable to apply the load balancer config because resource is unavaliable.", e);
      return false;
    }

    return true;
  }
  @Override
  public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachineProfile profile) {
    final DomainRouterVO domainRouterVO = _routerDao.findById(profile.getId());

    final boolean isVpc = domainRouterVO.getVpcId() != null;
    if (!isVpc) {
      return super.finalizeCommandsOnStart(cmds, profile);
    }

    if (domainRouterVO.getState() == State.Starting || domainRouterVO.getState() == State.Running) {
      // 1) FORM SSH CHECK COMMAND
      final NicProfile controlNic = getControlNic(profile);
      if (controlNic == null) {
        s_logger.error("Control network doesn't exist for the router " + domainRouterVO);
        return false;
      }

      finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, domainRouterVO, controlNic);

      // 2) FORM PLUG NIC COMMANDS
      final List<Pair<Nic, Network>> guestNics = new ArrayList<Pair<Nic, Network>>();
      final List<Pair<Nic, Network>> publicNics = new ArrayList<Pair<Nic, Network>>();
      final Map<String, String> vlanMacAddress = new HashMap<String, String>();

      final List<? extends Nic> routerNics = _nicDao.listByVmId(profile.getId());
      for (final Nic routerNic : routerNics) {
        final Network network = _networkModel.getNetwork(routerNic.getNetworkId());
        if (network.getTrafficType() == TrafficType.Guest) {
          final Pair<Nic, Network> guestNic = new Pair<Nic, Network>(routerNic, network);
          guestNics.add(guestNic);
        } else if (network.getTrafficType() == TrafficType.Public) {
          final Pair<Nic, Network> publicNic = new Pair<Nic, Network>(routerNic, network);
          publicNics.add(publicNic);
          final String vlanTag = BroadcastDomainType.getValue(routerNic.getBroadcastUri());
          vlanMacAddress.put(vlanTag, routerNic.getMacAddress());
        }
      }

      final List<Command> usageCmds = new ArrayList<Command>();

      // 3) PREPARE PLUG NIC COMMANDS
      try {
        // add VPC router to public networks
        final List<PublicIp> sourceNat = new ArrayList<PublicIp>(1);
        for (final Pair<Nic, Network> nicNtwk : publicNics) {
          final Nic publicNic = nicNtwk.first();
          final Network publicNtwk = nicNtwk.second();
          final IPAddressVO userIp =
              _ipAddressDao.findByIpAndSourceNetworkId(
                  publicNtwk.getId(), publicNic.getIPv4Address());

          if (userIp.isSourceNat()) {
            final PublicIp publicIp =
                PublicIp.createFromAddrAndVlan(userIp, _vlanDao.findById(userIp.getVlanId()));
            sourceNat.add(publicIp);

            if (domainRouterVO.getPublicIpAddress() == null) {
              final DomainRouterVO routerVO = _routerDao.findById(domainRouterVO.getId());
              routerVO.setPublicIpAddress(publicNic.getIPv4Address());
              routerVO.setPublicNetmask(publicNic.getIPv4Netmask());
              routerVO.setPublicMacAddress(publicNic.getMacAddress());
              _routerDao.update(routerVO.getId(), routerVO);
            }
          }
          final PlugNicCommand plugNicCmd =
              new PlugNicCommand(
                  _nwHelper.getNicTO(
                      domainRouterVO,
                      publicNic.getNetworkId(),
                      publicNic.getBroadcastUri().toString()),
                  domainRouterVO.getInstanceName(),
                  domainRouterVO.getType());
          cmds.addCommand(plugNicCmd);
          final VpcVO vpc = _vpcDao.findById(domainRouterVO.getVpcId());
          final NetworkUsageCommand netUsageCmd =
              new NetworkUsageCommand(
                  domainRouterVO.getPrivateIpAddress(),
                  domainRouterVO.getInstanceName(),
                  true,
                  publicNic.getIPv4Address(),
                  vpc.getCidr());
          usageCmds.add(netUsageCmd);
          UserStatisticsVO stats =
              _userStatsDao.findBy(
                  domainRouterVO.getAccountId(),
                  domainRouterVO.getDataCenterId(),
                  publicNtwk.getId(),
                  publicNic.getIPv4Address(),
                  domainRouterVO.getId(),
                  domainRouterVO.getType().toString());
          if (stats == null) {
            stats =
                new UserStatisticsVO(
                    domainRouterVO.getAccountId(),
                    domainRouterVO.getDataCenterId(),
                    publicNic.getIPv4Address(),
                    domainRouterVO.getId(),
                    domainRouterVO.getType().toString(),
                    publicNtwk.getId());
            _userStatsDao.persist(stats);
          }
        }

        // create ip assoc for source nat
        if (!sourceNat.isEmpty()) {
          _commandSetupHelper.createVpcAssociatePublicIPCommands(
              domainRouterVO, sourceNat, cmds, vlanMacAddress);
        }

        // add VPC router to guest networks
        for (final Pair<Nic, Network> nicNtwk : guestNics) {
          final Nic guestNic = nicNtwk.first();
          // plug guest nic
          final PlugNicCommand plugNicCmd =
              new PlugNicCommand(
                  _nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null),
                  domainRouterVO.getInstanceName(),
                  domainRouterVO.getType());
          cmds.addCommand(plugNicCmd);
          if (!_networkModel.isPrivateGateway(guestNic.getNetworkId())) {
            // set guest network
            final VirtualMachine vm = _vmDao.findById(domainRouterVO.getId());
            final NicProfile nicProfile =
                _networkModel.getNicProfile(vm, guestNic.getNetworkId(), null);
            final SetupGuestNetworkCommand setupCmd =
                _commandSetupHelper.createSetupGuestNetworkCommand(
                    domainRouterVO, true, nicProfile);
            cmds.addCommand(setupCmd);
          } else {

            // set private network
            final PrivateIpVO ipVO =
                _privateIpDao.findByIpAndSourceNetworkId(
                    guestNic.getNetworkId(), guestNic.getIPv4Address());
            final Network network = _networkDao.findById(guestNic.getNetworkId());
            BroadcastDomainType.getValue(network.getBroadcastUri());
            final String netmask = NetUtils.getCidrNetmask(network.getCidr());
            final PrivateIpAddress ip =
                new PrivateIpAddress(
                    ipVO,
                    network.getBroadcastUri().toString(),
                    network.getGateway(),
                    netmask,
                    guestNic.getMacAddress());

            final List<PrivateIpAddress> privateIps = new ArrayList<PrivateIpAddress>(1);
            privateIps.add(ip);
            _commandSetupHelper.createVpcAssociatePrivateIPCommands(
                domainRouterVO, privateIps, cmds, true);

            final Long privateGwAclId =
                _vpcGatewayDao.getNetworkAclIdForPrivateIp(
                    ipVO.getVpcId(), ipVO.getNetworkId(), ipVO.getIpAddress());

            if (privateGwAclId != null) {
              // set network acl on private gateway
              final List<NetworkACLItemVO> networkACLs =
                  _networkACLItemDao.listByACL(privateGwAclId);
              s_logger.debug(
                  "Found "
                      + networkACLs.size()
                      + " network ACLs to apply as a part of VPC VR "
                      + domainRouterVO
                      + " start for private gateway ip = "
                      + ipVO.getIpAddress());

              _commandSetupHelper.createNetworkACLsCommands(
                  networkACLs, domainRouterVO, cmds, ipVO.getNetworkId(), true);
            }
          }
        }
      } catch (final Exception ex) {
        s_logger.warn(
            "Failed to add router " + domainRouterVO + " to network due to exception ", ex);
        return false;
      }

      // 4) RE-APPLY ALL STATIC ROUTE RULES
      final List<? extends StaticRoute> routes =
          _staticRouteDao.listByVpcId(domainRouterVO.getVpcId());
      final List<StaticRouteProfile> staticRouteProfiles =
          new ArrayList<StaticRouteProfile>(routes.size());
      final Map<Long, VpcGateway> gatewayMap = new HashMap<Long, VpcGateway>();
      for (final StaticRoute route : routes) {
        VpcGateway gateway = gatewayMap.get(route.getVpcGatewayId());
        if (gateway == null) {
          gateway = _entityMgr.findById(VpcGateway.class, route.getVpcGatewayId());
          gatewayMap.put(gateway.getId(), gateway);
        }
        staticRouteProfiles.add(new StaticRouteProfile(route, gateway));
      }

      s_logger.debug(
          "Found "
              + staticRouteProfiles.size()
              + " static routes to apply as a part of vpc route "
              + domainRouterVO
              + " start");
      if (!staticRouteProfiles.isEmpty()) {
        _commandSetupHelper.createStaticRouteCommands(staticRouteProfiles, domainRouterVO, cmds);
      }

      // 5) RE-APPLY ALL REMOTE ACCESS VPNs
      final RemoteAccessVpnVO vpn =
          _vpnDao.findByAccountAndVpc(domainRouterVO.getAccountId(), domainRouterVO.getVpcId());
      if (vpn != null) {
        _commandSetupHelper.createApplyVpnCommands(true, vpn, domainRouterVO, cmds);
      }

      // 6) REPROGRAM GUEST NETWORK
      boolean reprogramGuestNtwks = true;
      if (profile.getParameter(Param.ReProgramGuestNetworks) != null
          && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) {
        reprogramGuestNtwks = false;
      }

      final VirtualRouterProvider vrProvider =
          _vrProviderDao.findById(domainRouterVO.getElementId());
      if (vrProvider == null) {
        throw new CloudRuntimeException(
            "Cannot find related virtual router provider of router: "
                + domainRouterVO.getHostName());
      }
      final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString());
      if (provider == null) {
        throw new CloudRuntimeException(
            "Cannot find related provider of virtual router provider: "
                + vrProvider.getType().toString());
      }

      for (final Pair<Nic, Network> nicNtwk : guestNics) {
        final Nic guestNic = nicNtwk.first();
        final AggregationControlCommand startCmd =
            new AggregationControlCommand(
                Action.Start,
                domainRouterVO.getInstanceName(),
                controlNic.getIPv4Address(),
                _routerControlHelper.getRouterIpInNetwork(
                    guestNic.getNetworkId(), domainRouterVO.getId()));
        cmds.addCommand(startCmd);
        if (reprogramGuestNtwks) {
          finalizeIpAssocForNetwork(
              cmds, domainRouterVO, provider, guestNic.getNetworkId(), vlanMacAddress);
          finalizeNetworkRulesForNetwork(cmds, domainRouterVO, provider, guestNic.getNetworkId());
        }

        finalizeUserDataAndDhcpOnStart(cmds, domainRouterVO, provider, guestNic.getNetworkId());
        final AggregationControlCommand finishCmd =
            new AggregationControlCommand(
                Action.Finish,
                domainRouterVO.getInstanceName(),
                controlNic.getIPv4Address(),
                _routerControlHelper.getRouterIpInNetwork(
                    guestNic.getNetworkId(), domainRouterVO.getId()));
        cmds.addCommand(finishCmd);
      }

      // Add network usage commands
      cmds.addCommands(usageCmds);
    }
    return true;
  }