private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds)
      throws AgentUnavailableException {
    Answer[] answers = null;
    try {
      answers = _agentMgr.send(elbVm.getHostId(), cmds);
    } catch (OperationTimedoutException e) {
      s_logger.warn("ELB: Timed Out", e);
      throw new AgentUnavailableException(
          "Unable to send commands to virtual elbVm ", elbVm.getHostId(), e);
    }

    if (answers == null) {
      return false;
    }

    if (answers.length != cmds.size()) {
      return false;
    }

    // FIXME: Have to return state for individual command in the future
    if (answers.length > 0) {
      Answer ans = answers[0];
      return ans.getResult();
    }
    return true;
  }
  protected Status testIpAddress(Long hostId, String testHostIp) {
    try {
      Answer pingTestAnswer = _agentMgr.send(hostId, new PingTestCommand(testHostIp));
      if (pingTestAnswer == null) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug("host (" + testHostIp + ") returns null answer");
        }
        return null;
      }

      if (pingTestAnswer.getResult()) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug(
              "host (" + testHostIp + ") has been successfully pinged, returning that host is up");
        }
        // computing host is available, but could not reach agent, return false
        return Status.Up;
      } else {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug(
              "host (" + testHostIp + ") cannot be pinged, returning null ('I don't know')");
        }
        return null;
      }
    } catch (AgentUnavailableException e) {
      return null;
    } catch (OperationTimedoutException e) {
      return null;
    }
  }
  @Override
  public void CheckAndDestroyTunnel(VirtualMachine vm) {
    if (!_isEnabled) {
      return;
    }

    List<UserVmVO> userVms = _userVmDao.listByAccountIdAndHostId(vm.getAccountId(), vm.getHostId());
    if (vm.getType() == VirtualMachine.Type.User) {
      if (userVms.size() > 1) {
        return;
      }

      List<DomainRouterVO> routers =
          _routerDao.findBy(vm.getAccountId(), vm.getDataCenterIdToDeployIn());
      for (DomainRouterVO router : routers) {
        if (router.getHostId() == vm.getHostId()) {
          return;
        }
      }
    } else if (vm.getType() == VirtualMachine.Type.DomainRouter && userVms.size() != 0) {
      return;
    }

    try {
      /* Now we are last one on host, destroy all tunnels of my account */
      Command cmd = new OvsDestroyTunnelCommand(vm.getAccountId(), "[]");
      Answer ans = _agentMgr.send(vm.getHostId(), cmd);
      handleDestroyTunnelAnswer(ans, vm.getHostId(), 0, vm.getAccountId());

      /* Then ask hosts have peer tunnel with me to destroy them */
      List<OvsTunnelAccountVO> peers =
          _tunnelAccountDao.listByToAccount(vm.getHostId(), vm.getAccountId());
      for (OvsTunnelAccountVO p : peers) {
        cmd = new OvsDestroyTunnelCommand(p.getAccount(), p.getPortName());
        ans = _agentMgr.send(p.getFrom(), cmd);
        handleDestroyTunnelAnswer(ans, p.getFrom(), p.getTo(), p.getAccount());
      }
    } catch (Exception e) {
      s_logger.warn(
          String.format(
              "Destroy tunnel(account:%1$s, hostId:%2$s) failed",
              vm.getAccountId(), vm.getHostId()),
          e);
    }
  }
  protected Long stopVM(final HaWorkVO work) {
    final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
    final VMInstanceVO vm = mgr.get(work.getInstanceId());
    s_logger.info("Stopping " + vm.toString());
    try {
      if (work.getWorkType() == WorkType.Stop) {
        if (vm.getHostId() != null) {
          if (mgr.stop(vm, 0)) {
            s_logger.info("Successfully stopped " + vm.toString());
            return null;
          }
        } else {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(vm.toString() + " has already been stopped");
          }
          return null;
        }
      } else if (work.getWorkType() == WorkType.CheckStop) {
        if ((vm.getState() != State.Stopping)
            || vm.getHostId() == null
            || vm.getHostId().longValue() != work.getHostId()) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                vm.toString()
                    + " is different now.  Scheduled Host: "
                    + work.getHostId()
                    + " Current Host: "
                    + (vm.getHostId() != null ? vm.getHostId() : "none")
                    + " State: "
                    + vm.getState());
          }
          return null;
        } else {
          Command cmd = mgr.cleanup(vm, null);
          Answer ans = _agentMgr.send(work.getHostId(), cmd);
          if (ans.getResult()) {
            mgr.completeStopCommand(vm);
            s_logger.info("Successfully stopped " + vm.toString());
            return null;
          }
          s_logger.debug(
              "Stop for " + vm.toString() + " was unsuccessful. Detail: " + ans.getDetails());
        }
      } else {
        assert false
            : "Who decided there's other steps but didn't modify the guy who does the work?";
      }
    } catch (final AgentUnavailableException e) {
      s_logger.debug("Agnet is not available" + e.getMessage());
    } catch (OperationTimedoutException e) {
      s_logger.debug("operation timed out: " + e.getMessage());
    }

    work.setTimesTried(work.getTimesTried() + 1);
    return (System.currentTimeMillis() >> 10) + _stopRetryInterval;
  }
Example #5
0
  @Override
  public Boolean fenceOff(VirtualMachine vm, Host host) {
    if (host.getHypervisorType() != HypervisorType.KVM
        && host.getHypervisorType() != HypervisorType.LXC) {
      s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
      return null;
    }

    List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
    FenceCommand fence = new FenceCommand(vm, host);

    int i = 0;
    for (HostVO h : hosts) {
      if (h.getHypervisorType() == HypervisorType.KVM
          || h.getHypervisorType() == HypervisorType.LXC) {
        if (h.getStatus() != Status.Up) {
          continue;
        }

        i++;

        if (h.getId() == host.getId()) {
          continue;
        }
        FenceAnswer answer;
        try {
          answer = (FenceAnswer) _agentMgr.send(h.getId(), fence);
        } catch (AgentUnavailableException e) {
          s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
          continue;
        } catch (OperationTimedoutException e) {
          s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
          continue;
        }
        if (answer != null && answer.getResult()) {
          return true;
        }
      }
    }

    _alertMgr.sendAlert(
        AlertManager.AlertType.ALERT_TYPE_HOST,
        host.getDataCenterId(),
        host.getPodId(),
        "Unable to fence off host: " + host.getId(),
        "Fencing off host "
            + host.getId()
            + " did not succeed after asking "
            + i
            + " hosts. "
            + "Check Agent logs for more information.");

    s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());

    return false;
  }
 @Override
 public Answer sendCommand(HypervisorHostEndPoint host, Command command) {
   Answer answer;
   try {
     answer = agentMgr.send(host.getId(), command);
     return answer;
   } catch (AgentUnavailableException e) {
     return null;
   } catch (OperationTimedoutException e) {
     return null;
   }
 }
Example #7
0
  @Override
  public Boolean fenceOff(VMInstanceVO vm, HostVO host) {
    if (host.getHypervisorType() != HypervisorType.Ovm) {
      s_logger.debug("Don't know how to fence non Ovm hosts " + host.getHypervisorType());
      return null;
    }

    List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
    FenceCommand fence = new FenceCommand(vm, host);

    for (HostVO h : hosts) {
      if (h.getHypervisorType() != HypervisorType.Ovm) {
        continue;
      }

      if (h.getStatus() != Status.Up) {
        continue;
      }

      if (h.getId() == host.getId()) {
        continue;
      }

      FenceAnswer answer;
      try {
        answer = (FenceAnswer) _agentMgr.send(h.getId(), fence);
      } catch (AgentUnavailableException e) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
        }
        continue;
      } catch (OperationTimedoutException e) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
        }
        continue;
      }

      if (answer != null && answer.getResult()) {
        return true;
      }
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
    }

    return false;
  }
  @Override
  public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForceDeleteStorage)
      throws UnableDeleteHostException {
    if (host.getType() != Host.Type.Routing || host.getHypervisorType() != HypervisorType.KVM) {
      return null;
    }

    _resourceMgr.deleteRoutingHost(host, isForced, isForceDeleteStorage);
    try {
      ShutdownCommand cmd = new ShutdownCommand(ShutdownCommand.DeleteHost, null);
      _agentMgr.send(host.getId(), cmd);
    } catch (AgentUnavailableException e) {
      s_logger.warn("Sending ShutdownCommand failed: ", e);
    } catch (OperationTimedoutException e) {
      s_logger.warn("Sending ShutdownCommand failed: ", e);
    }

    return new DeleteHostAnswer(true);
  }
  protected void handleVmMigrated(VMInstanceVO vm) {
    if (!isVmSecurityGroupEnabled(vm.getId())) return;
    if (vm.getType() != VirtualMachine.Type.User) {
      Commands cmds = null;
      NetworkRulesSystemVmCommand nrc =
          new NetworkRulesSystemVmCommand(vm.getInstanceName(), vm.getType());
      cmds = new Commands(nrc);
      try {
        _agentMgr.send(vm.getHostId(), cmds);
      } catch (AgentUnavailableException e) {
        s_logger.debug(e.toString());
      } catch (OperationTimedoutException e) {
        s_logger.debug(e.toString());
      }

    } else {
      List<Long> affectedVms = new ArrayList<Long>();
      affectedVms.add(vm.getId());
      scheduleRulesetUpdateToHosts(affectedVms, true, null);
    }
  }
  @Override
  public void processConnect(Host host, StartupCommand cmd, boolean forRebalance) {
    if (s_logger.isInfoEnabled()) s_logger.info("Received a host startup notification");

    if (cmd instanceof StartupRoutingCommand) {
      // if (Boolean.toString(true).equals(host.getDetail("can_bridge_firewall"))) {
      try {
        int interval =
            MIN_TIME_BETWEEN_CLEANUPS + _cleanupRandom.nextInt(MIN_TIME_BETWEEN_CLEANUPS / 2);
        CleanupNetworkRulesCmd cleanupCmd = new CleanupNetworkRulesCmd(interval);
        Commands c = new Commands(cleanupCmd);
        _agentMgr.send(host.getId(), c, this);
        if (s_logger.isInfoEnabled())
          s_logger.info("Scheduled network rules cleanup, interval=" + cleanupCmd.getInterval());
      } catch (AgentUnavailableException e) {
        // usually hypervisors that do not understand sec group rules.
        s_logger.debug("Unable to schedule network rules cleanup for host " + host.getId(), e);
      }
      if (_workTracker != null) {
        _workTracker.processConnect(host.getId());
      }
    }
  }
Example #11
0
  @Override
  public boolean sendCommandsToRouter(final VirtualRouter router, final Commands cmds)
      throws AgentUnavailableException, ResourceUnavailableException {
    if (!checkRouterVersion(router)) {
      s_logger.debug(
          "Router requires upgrade. Unable to send command to router:"
              + router.getId()
              + ", router template version : "
              + router.getTemplateVersion()
              + ", minimal required version : "
              + NetworkOrchestrationService.MinVRVersion.valueIn(router.getDataCenterId()));
      throw new ResourceUnavailableException(
          "Unable to send command. Router requires upgrade", VirtualRouter.class, router.getId());
    }
    Answer[] answers = null;
    try {
      answers = _agentMgr.send(router.getHostId(), cmds);
    } catch (final OperationTimedoutException e) {
      s_logger.warn("Timed Out", e);
      throw new AgentUnavailableException(
          "Unable to send commands to virtual router ", router.getHostId(), e);
    }

    if (answers == null || answers.length != cmds.size()) {
      return false;
    }

    // FIXME: Have to return state for individual command in the future
    boolean result = true;
    for (final Answer answer : answers) {
      if (!answer.getResult()) {
        result = false;
        break;
      }
    }
    return result;
  }
  protected Long destroyVM(HaWorkVO work) {
    final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
    final VMInstanceVO vm = mgr.get(work.getInstanceId());
    s_logger.info("Destroying " + vm.toString());
    try {
      if (vm.getState() != State.Destroyed) {
        s_logger.info("VM is no longer in Destroyed state " + vm.toString());
        return null;
      }

      if (vm.getHostId() != null) {
        Command cmd = mgr.cleanup(vm, null);
        Answer ans = _agentMgr.send(work.getHostId(), cmd);
        if (ans.getResult()) {
          mgr.completeStopCommand(vm);
          if (mgr.destroy(vm)) {
            s_logger.info("Successfully stopped " + vm.toString());
            return null;
          }
        }
        s_logger.debug(
            "Stop for " + vm.toString() + " was unsuccessful. Detail: " + ans.getDetails());
      } else {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug(vm.toString() + " has already been stopped");
        }
        return null;
      }
    } catch (final AgentUnavailableException e) {
      s_logger.debug("Agnet is not available" + e.getMessage());
    } catch (OperationTimedoutException e) {
      s_logger.debug("operation timed out: " + e.getMessage());
    }

    work.setTimesTried(work.getTimesTried() + 1);
    return (System.currentTimeMillis() >> 10) + _stopRetryInterval;
  }
  @DB
  public void work() {
    if (s_logger.isTraceEnabled()) {
      s_logger.trace("Checking the database");
    }
    final SecurityGroupWorkVO work = _workDao.take(_serverId);
    if (work == null) {
      if (s_logger.isTraceEnabled()) {
        s_logger.trace("Security Group work: no work found");
      }
      return;
    }
    Long userVmId = work.getInstanceId();
    if (work.getStep() == Step.Done) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "Security Group work: found a job in done state, rescheduling for vm: " + userVmId);
      }
      ArrayList<Long> affectedVms = new ArrayList<Long>();
      affectedVms.add(userVmId);
      scheduleRulesetUpdateToHosts(affectedVms, false, _timeBetweenCleanups * 1000l);
      return;
    }
    UserVm vm = null;
    Long seqnum = null;
    s_logger.debug("Working on " + work);
    final Transaction txn = Transaction.currentTxn();
    txn.start();
    boolean locked = false;
    try {
      vm = _userVMDao.acquireInLockTable(work.getInstanceId());
      if (vm == null) {
        vm = _userVMDao.findById(work.getInstanceId());
        if (vm == null) {
          s_logger.info("VM " + work.getInstanceId() + " is removed");
          locked = true;
          return;
        }
        s_logger.warn("Unable to acquire lock on vm id=" + userVmId);
        return;
      }
      locked = true;
      Long agentId = null;
      VmRulesetLogVO log = _rulesetLogDao.findByVmId(userVmId);
      if (log == null) {
        s_logger.warn("Cannot find log record for vm id=" + userVmId);
        return;
      }
      seqnum = log.getLogsequence();

      if (vm != null && vm.getState() == State.Running) {
        Map<PortAndProto, Set<String>> rules = generateRulesForVM(userVmId);
        agentId = vm.getHostId();
        if (agentId != null) {
          SecurityIngressRulesCmd cmd =
              generateRulesetCmd(
                  vm.getInstanceName(),
                  vm.getPrivateIpAddress(),
                  vm.getPrivateMacAddress(),
                  vm.getId(),
                  generateRulesetSignature(rules),
                  seqnum,
                  rules);
          Commands cmds = new Commands(cmd);
          try {
            _agentMgr.send(agentId, cmds, _answerListener);
          } catch (AgentUnavailableException e) {
            s_logger.debug(
                "Unable to send updates for vm: " + userVmId + "(agentid=" + agentId + ")");
            _workDao.updateStep(work.getInstanceId(), seqnum, Step.Done);
          }
        }
      }
    } finally {
      if (locked) {
        _userVMDao.releaseFromLockTable(userVmId);
        _workDao.updateStep(work.getId(), Step.Done);
      }
      txn.commit();
    }
  }
  @Override
  public final void processConnect(
      final Host agent, final StartupCommand cmd, final boolean forRebalance)
      throws ConnectionException {
    // Limit the commands we can process
    if (!(cmd instanceof StartupRoutingCommand)) {
      return;
    }

    StartupRoutingCommand startup = (StartupRoutingCommand) cmd;

    // assert
    if (startup.getHypervisorType() != HypervisorType.Hyperv) {
      s_logger.debug("Not Hyper-V hypervisor, so moving on.");
      return;
    }

    long agentId = agent.getId();
    HostVO host = _hostDao.findById(agentId);

    // Our Hyper-V machines are not participating in pools, and the pool id
    // we provide them is not persisted.
    // This means the pool id can vary.
    ClusterVO cluster = _clusterDao.findById(host.getClusterId());
    if (cluster.getGuid() == null) {
      cluster.setGuid(startup.getPool());
      _clusterDao.update(cluster.getId(), cluster);
    }

    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Setting up host " + agentId);
    }

    HostEnvironment env = new HostEnvironment();
    SetupCommand setup = new SetupCommand(env);
    if (!host.isSetup()) {
      setup.setNeedSetup(true);
    }

    try {
      SetupAnswer answer = (SetupAnswer) _agentMgr.send(agentId, setup);
      if (answer != null && answer.getResult()) {
        host.setSetup(true);
        // TODO: clean up magic numbers below
        host.setLastPinged((System.currentTimeMillis() >> 10) - 5 * 60);
        _hostDao.update(host.getId(), host);
        if (answer.needReconnect()) {
          throw new ConnectionException(false, "Reinitialize agent after setup.");
        }
        return;
      } else {
        String reason = answer.getDetails();
        if (reason == null) {
          reason = " details were null";
        }
        s_logger.warn("Unable to setup agent " + agentId + " due to " + reason);
      }
      // Error handling borrowed from XcpServerDiscoverer, may need to be
      // updated.
    } catch (AgentUnavailableException e) {
      s_logger.warn("Unable to setup agent " + agentId + " because it became unavailable.", e);
    } catch (OperationTimedoutException e) {
      s_logger.warn("Unable to setup agent " + agentId + " because it timed out", e);
    }
    throw new ConnectionException(true, "Reinitialize agent after setup.");
  }
 public void send(Long hostId, Command cmd, Listener listener) throws AgentUnavailableException {
   _agentMgr.send(hostId, new Commands(cmd), listener);
 }
  public Long migrate(final HaWorkVO work) {
    final long vmId = work.getInstanceId();

    final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());

    VMInstanceVO vm = mgr.get(vmId);
    if (vm == null || vm.getRemoved() != null) {
      s_logger.debug("Unable to find the vm " + vmId);
      return null;
    }

    s_logger.info("Migrating vm: " + vm.toString());
    if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
      s_logger.info("VM is not longer running on the current hostId");
      return null;
    }

    short alertType = AlertManager.ALERT_TYPE_USERVM_MIGRATE;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
    }

    HostVO fromHost = _hostDao.findById(vm.getHostId());
    String fromHostName = ((fromHost == null) ? "unknown" : fromHost.getName());
    HostVO toHost = null;
    if (work.getStep() == Step.Scheduled) {
      if (vm.getState() != State.Running) {
        s_logger.info(
            "VM's state is not ready for migration. "
                + vm.toString()
                + " State is "
                + vm.getState().toString());
        return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
      }

      DataCenterVO dcVO = _dcDao.findById(fromHost.getDataCenterId());
      HostPodVO podVO = _podDao.findById(fromHost.getPodId());

      try {
        toHost = mgr.prepareForMigration(vm);
        if (toHost == null) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Unable to find a host for migrating vm " + vmId);
          }
          _alertMgr.sendAlert(
              alertType,
              vm.getDataCenterId(),
              vm.getPodId(),
              "Unable to migrate vm "
                  + vm.getName()
                  + " from host "
                  + fromHostName
                  + " in zone "
                  + dcVO.getName()
                  + " and pod "
                  + podVO.getName(),
              "Unable to find a suitable host");
        }
      } catch (final InsufficientCapacityException e) {
        s_logger.warn("Unable to mgirate due to insufficient capacity " + vm.toString());
        _alertMgr.sendAlert(
            alertType,
            vm.getDataCenterId(),
            vm.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHostName
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Insufficient capacity");
      } catch (final StorageUnavailableException e) {
        s_logger.warn("Storage is unavailable: " + vm.toString());
        _alertMgr.sendAlert(
            alertType,
            vm.getDataCenterId(),
            vm.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHostName
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Storage is gone.");
      }

      if (toHost == null) {
        _agentMgr.maintenanceFailed(vm.getHostId());
        return null;
      }

      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Migrating from " + work.getHostId() + " to " + toHost.getId());
      }
      work.setStep(Step.Migrating);
      work.setHostId(toHost.getId());
      _haDao.update(work.getId(), work);
    }

    if (work.getStep() == Step.Migrating) {
      vm = mgr.get(vmId); // let's see if anything has changed.
      boolean migrated = false;
      if (vm == null
          || vm.getRemoved() != null
          || vm.getHostId() == null
          || !_itMgr.stateTransitTo(vm, Event.MigrationRequested, vm.getHostId())) {
        s_logger.info("Migration cancelled because state has changed: " + vm.toString());
      } else {
        try {
          boolean isWindows =
              _guestOSCategoryDao
                  .findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId())
                  .getName()
                  .equalsIgnoreCase("Windows");
          MigrateCommand cmd =
              new MigrateCommand(vm.getInstanceName(), toHost.getPrivateIpAddress(), isWindows);
          Answer answer = _agentMgr.send(fromHost.getId(), cmd);
          if (answer != null && answer.getResult()) {
            migrated = true;
            _storageMgr.unshare(vm, fromHost);
            work.setStep(Step.Investigating);
            _haDao.update(work.getId(), work);
          }
        } catch (final AgentUnavailableException e) {
          s_logger.debug("host became unavailable");
        } catch (final OperationTimedoutException e) {
          s_logger.debug("operation timed out");
          if (e.isActive()) {
            scheduleRestart(vm, true);
          }
        }
      }

      if (!migrated) {
        s_logger.info("Migration was unsuccessful.  Cleaning up: " + vm.toString());

        DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
        HostPodVO podVO = _podDao.findById(vm.getPodId());
        _alertMgr.sendAlert(
            alertType,
            fromHost.getDataCenterId(),
            fromHost.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHost.getName()
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Migrate Command failed.  Please check logs.");

        _itMgr.stateTransitTo(vm, Event.MigrationFailedOnSource, toHost.getId());
        _agentMgr.maintenanceFailed(vm.getHostId());

        Command cleanup = mgr.cleanup(vm, null);
        _agentMgr.easySend(toHost.getId(), cleanup);
        _storageMgr.unshare(vm, toHost);

        return null;
      }
    }

    if (toHost == null) {
      toHost = _hostDao.findById(work.getHostId());
    }
    DataCenterVO dcVO = _dcDao.findById(toHost.getDataCenterId());
    HostPodVO podVO = _podDao.findById(toHost.getPodId());

    try {
      if (!mgr.completeMigration(vm, toHost)) {
        _alertMgr.sendAlert(
            alertType,
            toHost.getDataCenterId(),
            toHost.getPodId(),
            "Unable to migrate "
                + vmId
                + " to host "
                + toHost.getName()
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Migration not completed");
        s_logger.warn("Unable to complete migration: " + vm.toString());
      } else {
        s_logger.info("Migration is complete: " + vm.toString());
      }
      return null;
    } catch (final AgentUnavailableException e) {
      s_logger.warn("Agent is unavailable for " + vm.toString());
    } catch (final OperationTimedoutException e) {
      s_logger.warn("Operation timed outfor " + vm.toString());
    }
    _itMgr.stateTransitTo(vm, Event.MigrationFailedOnDest, toHost.getId());
    return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
  }
  @Override
  public boolean addUserData(NicProfile nic, VirtualMachineProfile profile) {
    UserVmVO vm = _vmDao.findById(profile.getVirtualMachine().getId());
    _vmDao.loadDetails(vm);

    String serviceOffering =
        _serviceOfferingDao
            .findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId())
            .getDisplayText();
    String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
    NicVO nvo = _nicDao.findById(nic.getId());
    VmDataCommand cmd =
        new VmDataCommand(
            nvo.getIPv4Address(), vm.getInstanceName(), _ntwkModel.getExecuteInSeqNtwkElmtCmd());
    // if you add new metadata files, also edit
    // systemvm/patches/debian/config/var/www/html/latest/.htaccess
    cmd.addVmData("userdata", "user-data", vm.getUserData());
    cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering));
    cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName));
    cmd.addVmData("metadata", "local-ipv4", nic.getIPv4Address());
    cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vm.getInstanceName()));
    cmd.addVmData("metadata", "public-ipv4", nic.getIPv4Address());
    cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vm.getInstanceName()));
    cmd.addVmData("metadata", "instance-id", String.valueOf(vm.getId()));
    cmd.addVmData("metadata", "vm-id", String.valueOf(vm.getInstanceName()));
    cmd.addVmData("metadata", "public-keys", null);
    String cloudIdentifier = _configDao.getValue("cloud.identifier");
    if (cloudIdentifier == null) {
      cloudIdentifier = "";
    } else {
      cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}";
    }
    cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier);

    List<PhysicalNetworkVO> phys = _phynwDao.listByZone(vm.getDataCenterId());
    if (phys.isEmpty()) {
      throw new CloudRuntimeException(
          String.format("Cannot find physical network in zone %s", vm.getDataCenterId()));
    }
    if (phys.size() > 1) {
      throw new CloudRuntimeException(
          String.format(
              "Baremetal only supports one physical network in zone, but zone %s has %s physical networks",
              vm.getDataCenterId(), phys.size()));
    }
    PhysicalNetworkVO phy = phys.get(0);

    QueryBuilder<BaremetalPxeVO> sc = QueryBuilder.create(BaremetalPxeVO.class);
    // TODO: handle both kickstart and PING
    // sc.addAnd(sc.getEntity().getPodId(), Op.EQ, vm.getPodIdToDeployIn());
    sc.and(sc.entity().getPhysicalNetworkId(), Op.EQ, phy.getId());
    BaremetalPxeVO pxeVo = sc.find();
    if (pxeVo == null) {
      throw new CloudRuntimeException(
          "No PXE server found in pod: "
              + vm.getPodIdToDeployIn()
              + ", you need to add it before starting VM");
    }

    try {
      Answer ans = _agentMgr.send(pxeVo.getHostId(), cmd);
      if (!ans.getResult()) {
        s_logger.debug(
            String.format(
                "Add userdata to vm:%s failed because %s", vm.getInstanceName(), ans.getDetails()));
        return false;
      } else {
        return true;
      }
    } catch (Exception e) {
      s_logger.debug(String.format("Add userdata to vm:%s failed", vm.getInstanceName()), e);
      return false;
    }
  }
  @DB
  protected void CheckAndCreateTunnel(VirtualMachine instance, DeployDestination dest) {
    if (!_isEnabled) {
      return;
    }

    if (instance.getType() != VirtualMachine.Type.User
        && instance.getType() != VirtualMachine.Type.DomainRouter) {
      return;
    }

    long hostId = dest.getHost().getId();
    long accountId = instance.getAccountId();
    List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
    List<DomainRouterVO> routers =
        _routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
    List<VMInstanceVO> ins = new ArrayList<VMInstanceVO>();
    if (vms != null) {
      ins.addAll(vms);
    }
    if (routers.size() != 0) {
      ins.addAll(routers);
    }
    List<Pair<Long, Integer>> toHosts = new ArrayList<Pair<Long, Integer>>();
    List<Pair<Long, Integer>> fromHosts = new ArrayList<Pair<Long, Integer>>();
    int key;

    for (VMInstanceVO v : ins) {
      Long rh = v.getHostId();
      if (rh == null || rh.longValue() == hostId) {
        continue;
      }

      OvsTunnelAccountVO ta =
          _tunnelAccountDao.getByFromToAccount(hostId, rh.longValue(), accountId);
      if (ta == null) {
        key = getGreKey(hostId, rh.longValue(), accountId);
        if (key == -1) {
          s_logger.warn(
              String.format(
                  "Cannot get GRE key for from=%1$s to=%2$s accountId=%3$s, tunnel create failed",
                  hostId, rh.longValue(), accountId));
          continue;
        }

        Pair<Long, Integer> p = new Pair<Long, Integer>(rh, Integer.valueOf(key));
        if (!toHosts.contains(p)) {
          toHosts.add(p);
        }
      }

      ta = _tunnelAccountDao.getByFromToAccount(rh.longValue(), hostId, accountId);
      if (ta == null) {
        key = getGreKey(rh.longValue(), hostId, accountId);
        if (key == -1) {
          s_logger.warn(
              String.format(
                  "Cannot get GRE key for from=%1$s to=%2$s accountId=%3$s, tunnel create failed",
                  rh.longValue(), hostId, accountId));
          continue;
        }

        Pair<Long, Integer> p = new Pair<Long, Integer>(rh, Integer.valueOf(key));
        if (!fromHosts.contains(p)) {
          fromHosts.add(p);
        }
      }
    }

    try {
      String myIp = dest.getHost().getPrivateIpAddress();
      for (Pair<Long, Integer> i : toHosts) {
        HostVO rHost = _hostDao.findById(i.first());
        Commands cmds =
            new Commands(
                new OvsCreateTunnelCommand(
                    rHost.getPrivateIpAddress(),
                    i.second().toString(),
                    Long.valueOf(hostId),
                    i.first(),
                    accountId,
                    myIp));
        s_logger.debug("Ask host " + hostId + " to create gre tunnel to " + i.first());
        Answer[] answers = _agentMgr.send(hostId, cmds);
        handleCreateTunnelAnswer(answers);
      }

      for (Pair<Long, Integer> i : fromHosts) {
        HostVO rHost = _hostDao.findById(i.first());
        Commands cmd2s =
            new Commands(
                new OvsCreateTunnelCommand(
                    myIp,
                    i.second().toString(),
                    i.first(),
                    Long.valueOf(hostId),
                    accountId,
                    rHost.getPrivateIpAddress()));
        s_logger.debug("Ask host " + i.first() + " to create gre tunnel to " + hostId);
        Answer[] answers = _agentMgr.send(i.first(), cmd2s);
        handleCreateTunnelAnswer(answers);
      }
    } catch (Exception e) {
      s_logger.debug("Ovs Tunnel network created tunnel failed", e);
    }
  }