Пример #1
0
  private void updateNetworkLabels(HostVO host) {
    // check if networkLabels need to be updated in details
    // we send only private and storage network label to the resource.
    String privateNetworkLabel =
        _networkMgr.getDefaultManagementTrafficLabel(
            host.getDataCenterId(), host.getHypervisorType());
    String storageNetworkLabel =
        _networkMgr.getDefaultStorageTrafficLabel(host.getDataCenterId(), host.getHypervisorType());

    String privateDevice = host.getDetail("private.network.device");
    String storageDevice = host.getDetail("storage.network.device1");

    boolean update = false;

    if (privateNetworkLabel != null && !privateNetworkLabel.equalsIgnoreCase(privateDevice)) {
      host.setDetail("private.network.device", privateNetworkLabel);
      update = true;
    }
    if (storageNetworkLabel != null && !storageNetworkLabel.equalsIgnoreCase(storageDevice)) {
      host.setDetail("storage.network.device1", storageNetworkLabel);
      update = true;
    }
    if (update) {
      _hostDao.saveDetails(host);
    }
  }
  @Override
  public void scheduleRestartForVmsOnHost(final HostVO host) {

    if (host.getType() != Host.Type.Routing) {
      return;
    }
    s_logger.warn("Scheduling restart for VMs on host " + host.getId());

    final List<VMInstanceVO> vms = _instanceDao.listByHostId(host.getId());
    final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());

    // send an email alert that the host is down
    StringBuilder sb = null;
    if ((vms != null) && !vms.isEmpty()) {
      sb = new StringBuilder();
      sb.append("  Starting HA on the following VMs: ");
      // collect list of vm names for the alert email
      VMInstanceVO vm = vms.get(0);
      if (vm.isHaEnabled()) {
        sb.append(" " + vm.getName());
      }
      for (int i = 1; i < vms.size(); i++) {
        vm = vms.get(i);
        if (vm.isHaEnabled()) {
          sb.append(" " + vm.getName());
        }
      }
    }

    // send an email alert that the host is down, include VMs
    HostPodVO podVO = _podDao.findById(host.getPodId());
    String hostDesc =
        "name: "
            + host.getName()
            + " (id:"
            + host.getId()
            + "), availability zone: "
            + dcVO.getName()
            + ", pod: "
            + podVO.getName();

    _alertMgr.sendAlert(
        AlertManager.ALERT_TYPE_HOST,
        host.getDataCenterId(),
        host.getPodId(),
        "Host is down, " + hostDesc,
        "Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : ""));

    for (final VMInstanceVO vm : vms) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Notifying HA Mgr of to investigate vm " + vm.getId() + "-" + vm.getName());
      }
      scheduleRestart(vm, true);
    }
  }
  // TODO: add test for method
  @Override
  public final HostVO createHostVOForDirectConnectAgent(
      final HostVO host,
      final StartupCommand[] startup,
      final ServerResource resource,
      final Map<String, String> details,
      final List<String> hostTags) {
    StartupCommand firstCmd = startup[0];
    if (!(firstCmd instanceof StartupRoutingCommand)) {
      return null;
    }

    StartupRoutingCommand ssCmd = ((StartupRoutingCommand) firstCmd);
    if (ssCmd.getHypervisorType() != HypervisorType.Hyperv) {
      return null;
    }

    s_logger.info(
        "Host: "
            + host.getName()
            + " connected with hypervisor type: "
            + HypervisorType.Hyperv
            + ". Checking CIDR...");

    HostPodVO pod = _podDao.findById(host.getPodId());
    DataCenterVO dc = _dcDao.findById(host.getDataCenterId());

    _resourceMgr.checkCIDR(pod, dc, ssCmd.getPrivateIpAddress(), ssCmd.getPrivateNetmask());

    return _resourceMgr.fillRoutingHostVO(host, ssCmd, HypervisorType.Hyperv, details, hostTags);
  }
Пример #4
0
  protected HashMap<String, Object> buildConfigParams(HostVO host) {
    HashMap<String, Object> params = new HashMap<String, Object>(host.getDetails().size() + 5);
    params.putAll(host.getDetails());

    params.put("guid", host.getGuid());
    params.put("zone", Long.toString(host.getDataCenterId()));
    if (host.getPodId() != null) {
      params.put("pod", Long.toString(host.getPodId()));
    }
    if (host.getClusterId() != null) {
      params.put("cluster", Long.toString(host.getClusterId()));
      String guid = null;
      ClusterVO cluster = _clusterDao.findById(host.getClusterId());
      if (cluster.getGuid() == null) {
        guid = host.getDetail("pool");
      } else {
        guid = cluster.getGuid();
      }
      if (guid != null && !guid.isEmpty()) {
        params.put("pool", guid);
      }
    }

    params.put("ipaddress", host.getPrivateIpAddress());
    params.put("secondary.storage.vm", "false");
    params.put(
        "max.template.iso.size", _configDao.getValue(Config.MaxTemplateAndIsoSize.toString()));
    params.put("migratewait", _configDao.getValue(Config.MigrateWait.toString()));
    return params;
  }
  @Override
  public boolean prepareCreateTemplate(Long pxeServerId, UserVm vm, String templateUrl) {
    List<NicVO> nics = _nicDao.listByVmId(vm.getId());
    if (nics.size() != 1) {
      throw new CloudRuntimeException("Wrong nic number " + nics.size() + " of vm " + vm.getId());
    }

    /* use last host id when VM stopped */
    Long hostId = (vm.getHostId() == null ? vm.getLastHostId() : vm.getHostId());
    HostVO host = _hostDao.findById(hostId);
    DataCenterVO dc = _dcDao.findById(host.getDataCenterId());
    NicVO nic = nics.get(0);
    String mask = nic.getNetmask();
    String mac = nic.getMacAddress();
    String ip = nic.getIp4Address();
    String gateway = nic.getGateway();
    String dns = dc.getDns1();
    if (dns == null) {
      dns = dc.getDns2();
    }

    try {
      prepareCreateTemplateCommand cmd =
          new prepareCreateTemplateCommand(ip, mac, mask, gateway, dns, templateUrl);
      Answer ans = _agentMgr.send(pxeServerId, cmd);
      return ans.getResult();
    } catch (Exception e) {
      s_logger.debug("Prepare for creating baremetal template failed", e);
      return false;
    }
  }
Пример #6
0
  @Override
  public void handleSysTemplateDownload(HostVO host) {
    List<HypervisorType> hypers =
        _resourceMgr.listAvailHypervisorInZone(host.getId(), host.getDataCenterId());
    HypervisorType hostHyper = host.getHypervisorType();
    if (hypers.contains(hostHyper)) {
      return;
    }

    Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
    List<HostVO> ssHosts =
        _resourceMgr.listAllUpAndEnabledHostsInOneZoneByType(
            Host.Type.SecondaryStorage, host.getDataCenterId());
    if (ssHosts == null || ssHosts.isEmpty()) {
      return;
    }
    /*Download all the templates in zone with the same hypervisortype*/
    for (HostVO ssHost : ssHosts) {
      List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
      List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();

      for (VMTemplateVO rtngTmplt : rtngTmplts) {
        if (rtngTmplt.getHypervisorType() == hostHyper) {
          toBeDownloaded.add(rtngTmplt);
        }
      }

      for (VMTemplateVO builtinTmplt : defaultBuiltin) {
        if (builtinTmplt.getHypervisorType() == hostHyper) {
          toBeDownloaded.add(builtinTmplt);
        }
      }

      for (VMTemplateVO template : toBeDownloaded) {
        VMTemplateHostVO tmpltHost =
            _vmTemplateHostDao.findByHostTemplate(ssHost.getId(), template.getId());
        if (tmpltHost == null || tmpltHost.getDownloadState() != Status.DOWNLOADED) {
          downloadTemplateToStorage(template, ssHost);
        }
      }
    }
  }
Пример #7
0
  @DB
  public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) {
    if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED)
        || (dnldStatus == Status.ABANDONED)) {
      VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId());
      synchronized (_listenerVolumeMap) {
        _listenerVolumeMap.remove(volumeHost);
      }
    }

    VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId());

    Transaction txn = Transaction.currentTxn();
    txn.start();

    if (dnldStatus == Status.DOWNLOADED) {

      // Create usage event
      long size = -1;
      if (volumeHost != null) {
        size = volumeHost.getSize();
        volume.setSize(size);
        this._volumeDao.update(volume.getId(), volume);
      } else {
        s_logger.warn("Failed to get size for volume" + volume.getName());
      }
      String eventType = EventTypes.EVENT_VOLUME_UPLOAD;
      if (volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
        UsageEventUtils.publishUsageEvent(
            eventType,
            volume.getAccountId(),
            host.getDataCenterId(),
            volume.getId(),
            volume.getName(),
            null,
            0l,
            size,
            volume.getClass().getName(),
            volume.getUuid());
      }
    } else if (dnldStatus == Status.DOWNLOAD_ERROR
        || dnldStatus == Status.ABANDONED
        || dnldStatus == Status.UNKNOWN) {
      // Decrement the volume and secondary storage space count
      _resourceLimitMgr.decrementResourceCount(
          volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume);
      _resourceLimitMgr.recalculateResourceCount(
          volume.getAccountId(),
          volume.getDomainId(),
          com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
    }
    txn.commit();
  }
  @Override
  protected HashMap<String, Object> buildConfigParams(HostVO host) {
    HashMap<String, Object> params = super.buildConfigParams(host);

    Map<String, String> clusterDetails = _clusterDetailsDao.findDetails(host.getClusterId());
    // Get zone wide traffic labels from guest traffic and public traffic
    String guestTrafficLabel =
        _netmgr.getDefaultGuestTrafficLabel(host.getDataCenterId(), HypervisorType.VMware);
    String publicTrafficLabel =
        _netmgr.getDefaultPublicTrafficLabel(host.getDataCenterId(), HypervisorType.VMware);
    _readGlobalConfigParameters();
    VirtualSwitchType defaultVirtualSwitchType = getDefaultVirtualSwitchType();

    params.put(
        "guestTrafficInfo",
        getTrafficInfo(
            TrafficType.Guest, guestTrafficLabel, clusterDetails, defaultVirtualSwitchType));
    params.put(
        "publicTrafficInfo",
        getTrafficInfo(
            TrafficType.Public, publicTrafficLabel, clusterDetails, defaultVirtualSwitchType));

    return params;
  }
Пример #9
0
  @DB
  public void handleDownloadEvent(HostVO host, VMTemplateVO template, Status dnldStatus) {
    if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED)
        || (dnldStatus == Status.ABANDONED)) {
      VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId());
      synchronized (_listenerMap) {
        _listenerMap.remove(vmTemplateHost);
      }
    }

    VMTemplateHostVO vmTemplateHost =
        _vmTemplateHostDao.findByHostTemplate(host.getId(), template.getId());

    Transaction txn = Transaction.currentTxn();
    txn.start();

    if (dnldStatus == Status.DOWNLOADED) {
      long size = -1;
      if (vmTemplateHost != null) {
        size = vmTemplateHost.getPhysicalSize();
        template.setSize(size);
        this._templateDao.update(template.getId(), template);
      } else {
        s_logger.warn("Failed to get size for template" + template.getName());
      }
      String eventType = EventTypes.EVENT_TEMPLATE_CREATE;
      if ((template.getFormat()).equals(ImageFormat.ISO)) {
        eventType = EventTypes.EVENT_ISO_CREATE;
      }
      if (template.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
        UsageEventUtils.publishUsageEvent(
            eventType,
            template.getAccountId(),
            host.getDataCenterId(),
            template.getId(),
            template.getName(),
            null,
            template.getSourceTemplateId(),
            size,
            template.getClass().getName(),
            template.getUuid());
      }
    }
    txn.commit();
  }
 @Override
 public void processConnect(HostVO agent, StartupCommand cmd, boolean forRebalance) {
   if ((cmd instanceof StartupStorageCommand)) {
     StartupStorageCommand scmd = (StartupStorageCommand) cmd;
     if (scmd.getResourceType() == Storage.StorageResourceType.SECONDARY_STORAGE) {
       _ssVmMgr.generateSetupCommand(agent.getId());
       return;
     }
   } else if (cmd instanceof StartupSecondaryStorageCommand) {
     if (s_logger.isInfoEnabled()) {
       s_logger.info("Received a host startup notification " + cmd);
     }
     _ssVmMgr.onAgentConnect(agent.getDataCenterId(), cmd);
     _ssVmMgr.generateSetupCommand(agent.getId());
     _ssVmMgr.generateFirewallConfiguration(agent.getId());
     _ssVmMgr.generateVMSetupCommand(agent.getId());
     return;
   }
   return;
 }
    @Override
    @DB
    public boolean processAnswers(long agentId, long seq, Answer[] answers) {
      /*
       * Do not collect Direct Network usage stats if the Traffic Monitor is not owned by this mgmt server
       */
      HostVO host = _hostDao.findById(agentId);
      if (host != null) {
        if ((host.getManagementServerId() == null)
            || (mgmtSrvrId != host.getManagementServerId())) {
          s_logger.warn(
              "Not the owner. Not collecting Direct Network usage from  TrafficMonitor : "
                  + agentId);
          return false;
        }
      } else {
        s_logger.warn(
            "Agent not found. Not collecting Direct Network usage from  TrafficMonitor : "
                + agentId);
        return false;
      }

      GlobalLock scanLock =
          GlobalLock.getInternLock("direct.network.usage.collect" + host.getDataCenterId());
      try {
        if (scanLock.lock(10)) {
          try {
            return collectDirectNetworkUsage(host);
          } finally {
            scanLock.unlock();
          }
        }
      } finally {
        scanLock.releaseRef();
      }
      return false;
    }
Пример #12
0
  private String generateCopyUrl(HostVO sourceServer, VMTemplateHostVO srcTmpltHost) {
    List<SecondaryStorageVmVO> ssVms =
        _secStorageVmDao.getSecStorageVmListInStates(
            SecondaryStorageVm.Role.templateProcessor,
            sourceServer.getDataCenterId(),
            State.Running);
    if (ssVms.size() > 0) {
      SecondaryStorageVmVO ssVm = ssVms.get(0);
      if (ssVm.getPublicIpAddress() == null) {
        s_logger.warn("A running secondary storage vm has a null public ip?");
        return null;
      }
      return generateCopyUrl(
          ssVm.getPublicIpAddress(), sourceServer.getParent(), srcTmpltHost.getInstallPath());
    }

    VMTemplateVO tmplt = _templateDao.findById(srcTmpltHost.getTemplateId());
    HypervisorType hyperType = tmplt.getHypervisorType();
    /*No secondary storage vm yet*/
    if (hyperType != null && hyperType == HypervisorType.KVM) {
      return "file://" + sourceServer.getParent() + "/" + srcTmpltHost.getInstallPath();
    }
    return null;
  }
  @Override
  @DB
  public boolean delete(TemplateProfile profile) {
    boolean success = true;

    VMTemplateVO template = profile.getTemplate();
    Long zoneId = profile.getZoneId();
    Long templateId = template.getId();

    String zoneName;
    List<HostVO> secondaryStorageHosts;
    if (!template.isCrossZones() && zoneId != null) {
      DataCenterVO zone = _dcDao.findById(zoneId);
      zoneName = zone.getName();
      secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(zoneId);
    } else {
      zoneName = "(all zones)";
      secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInAllZones();
    }

    s_logger.debug(
        "Attempting to mark template host refs for template: "
            + template.getName()
            + " as destroyed in zone: "
            + zoneName);

    // Make sure the template is downloaded to all the necessary secondary storage hosts
    for (HostVO secondaryStorageHost : secondaryStorageHosts) {
      long hostId = secondaryStorageHost.getId();
      List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
      for (VMTemplateHostVO templateHostVO : templateHostVOs) {
        if (templateHostVO.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
          String errorMsg = "Please specify a template that is not currently being downloaded.";
          s_logger.debug(
              "Template: "
                  + template.getName()
                  + " is currently being downloaded to secondary storage host: "
                  + secondaryStorageHost.getName()
                  + "; cant' delete it.");
          throw new CloudRuntimeException(errorMsg);
        }
      }
    }

    Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId());
    String eventType = "";

    if (template.getFormat().equals(ImageFormat.ISO)) {
      eventType = EventTypes.EVENT_ISO_DELETE;
    } else {
      eventType = EventTypes.EVENT_TEMPLATE_DELETE;
    }

    // Iterate through all necessary secondary storage hosts and mark the template on each host as
    // destroyed
    for (HostVO secondaryStorageHost : secondaryStorageHosts) {
      long hostId = secondaryStorageHost.getId();
      long sZoneId = secondaryStorageHost.getDataCenterId();
      List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
      for (VMTemplateHostVO templateHostVO : templateHostVOs) {
        VMTemplateHostVO lock = _tmpltHostDao.acquireInLockTable(templateHostVO.getId());
        try {
          if (lock == null) {
            s_logger.debug(
                "Failed to acquire lock when deleting templateHostVO with ID: "
                    + templateHostVO.getId());
            success = false;
            break;
          }
          UsageEventVO usageEvent =
              new UsageEventVO(eventType, account.getId(), sZoneId, templateId, null);
          _usageEventDao.persist(usageEvent);
          templateHostVO.setDestroyed(true);
          _tmpltHostDao.update(templateHostVO.getId(), templateHostVO);
          String installPath = templateHostVO.getInstallPath();
          if (installPath != null) {
            Answer answer =
                _agentMgr.sendToSecStorage(
                    secondaryStorageHost,
                    new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(), installPath));

            if (answer == null || !answer.getResult()) {
              s_logger.debug(
                  "Failed to delete "
                      + templateHostVO
                      + " due to "
                      + ((answer == null) ? "answer is null" : answer.getDetails()));
            } else {
              _tmpltHostDao.remove(templateHostVO.getId());
              s_logger.debug("Deleted template at: " + installPath);
            }
          } else {
            _tmpltHostDao.remove(templateHostVO.getId());
          }
          VMTemplateZoneVO templateZone = _tmpltZoneDao.findByZoneTemplate(sZoneId, templateId);

          if (templateZone != null) {
            _tmpltZoneDao.remove(templateZone.getId());
          }
        } finally {
          if (lock != null) {
            _tmpltHostDao.releaseFromLockTable(lock.getId());
          }
        }
      }

      if (!success) {
        break;
      }
    }

    s_logger.debug(
        "Successfully marked template host refs for template: "
            + template.getName()
            + " as destroyed in zone: "
            + zoneName);

    // If there are no more non-destroyed template host entries for this template, delete it
    if (success && (_tmpltHostDao.listByTemplateId(templateId).size() == 0)) {
      long accountId = template.getAccountId();

      VMTemplateVO lock = _tmpltDao.acquireInLockTable(templateId);

      try {
        if (lock == null) {
          s_logger.debug("Failed to acquire lock when deleting template with ID: " + templateId);
          success = false;
        } else if (_tmpltDao.remove(templateId)) {
          // Decrement the number of templates
          _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template);
        }

      } finally {
        if (lock != null) {
          _tmpltDao.releaseFromLockTable(lock.getId());
        }
      }

      s_logger.debug(
          "Removed template: "
              + template.getName()
              + " because all of its template host refs were marked as destroyed.");
    }

    return success;
  }
Пример #14
0
  @Override
  public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer)
      throws StorageUnavailableException {

    boolean downloadJobExists = false;
    VMTemplateHostVO destTmpltHost = null;
    VMTemplateHostVO srcTmpltHost = null;

    srcTmpltHost = _vmTemplateHostDao.findByHostTemplate(sourceServer.getId(), template.getId());
    if (srcTmpltHost == null) {
      throw new InvalidParameterValueException(
          "Template " + template.getName() + " not associated with " + sourceServer.getName());
    }

    String url = generateCopyUrl(sourceServer, srcTmpltHost);
    if (url == null) {
      s_logger.warn(
          "Unable to start/resume copy of template "
              + template.getUniqueName()
              + " to "
              + destServer.getName()
              + ", no secondary storage vm in running state in source zone");
      throw new CloudRuntimeException(
          "No secondary VM in running state in zone " + sourceServer.getDataCenterId());
    }
    destTmpltHost = _vmTemplateHostDao.findByHostTemplate(destServer.getId(), template.getId());
    if (destTmpltHost == null) {
      destTmpltHost =
          new VMTemplateHostVO(
              destServer.getId(),
              template.getId(),
              new Date(),
              0,
              VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED,
              null,
              null,
              "jobid0000",
              null,
              url);
      destTmpltHost.setCopy(true);
      destTmpltHost.setPhysicalSize(srcTmpltHost.getPhysicalSize());
      _vmTemplateHostDao.persist(destTmpltHost);
    } else if ((destTmpltHost.getJobId() != null) && (destTmpltHost.getJobId().length() > 2)) {
      downloadJobExists = true;
    }

    Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
    if (srcTmpltHost.getSize() > maxTemplateSizeInBytes) {
      throw new CloudRuntimeException(
          "Cant copy the template as the template's size "
              + srcTmpltHost.getSize()
              + " is greater than max.template.iso.size "
              + maxTemplateSizeInBytes);
    }

    if (destTmpltHost != null) {
      start();
      String sourceChecksum =
          this.templateMgr.getChecksum(srcTmpltHost.getHostId(), srcTmpltHost.getInstallPath());
      DownloadCommand dcmd =
          new DownloadCommand(
              destServer.getStorageUrl(),
              url,
              template,
              TemplateConstants.DEFAULT_HTTP_AUTH_USER,
              _copyAuthPasswd,
              maxTemplateSizeInBytes);
      if (downloadJobExists) {
        dcmd =
            new DownloadProgressCommand(dcmd, destTmpltHost.getJobId(), RequestType.GET_OR_RESTART);
      }
      dcmd.setProxy(getHttpProxy());
      dcmd.setChecksum(
          sourceChecksum); // We need to set the checksum as the source template might be a
                           // compressed url and have cksum for compressed image. Bug #10775
      HostVO ssAhost = _ssvmMgr.pickSsvmHost(destServer);
      if (ssAhost == null) {
        s_logger.warn(
            "There is no secondary storage VM for secondary storage host " + destServer.getName());
        return false;
      }
      DownloadListener dl =
          new DownloadListener(
              ssAhost,
              destServer,
              template,
              _timer,
              _vmTemplateHostDao,
              destTmpltHost.getId(),
              this,
              dcmd,
              _templateDao,
              _resourceLimitMgr,
              _alertMgr,
              _accountMgr);
      if (downloadJobExists) {
        dl.setCurrState(destTmpltHost.getDownloadState());
      }
      DownloadListener old = null;
      synchronized (_listenerMap) {
        old = _listenerMap.put(destTmpltHost, dl);
      }
      if (old != null) {
        old.abandon();
      }

      try {
        send(ssAhost.getId(), dcmd, dl);
        return true;
      } catch (AgentUnavailableException e) {
        s_logger.warn(
            "Unable to start /resume COPY of template "
                + template.getUniqueName()
                + " to "
                + destServer.getName(),
            e);
        dl.setDisconnected();
        dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
        e.printStackTrace();
      }
    }

    return false;
  }
    @Override
    public void run() {
      try {
        SearchCriteria<HostVO> sc = _hostDao.createSearchCriteria();
        sc.addAnd("status", SearchCriteria.Op.EQ, Status.Up.toString());
        sc.addAnd("type", SearchCriteria.Op.EQ, Host.Type.Storage.toString());

        ConcurrentHashMap<Long, StorageStats> storageStats =
            new ConcurrentHashMap<Long, StorageStats>();
        List<HostVO> hosts = _hostDao.search(sc, null);
        for (HostVO host : hosts) {
          GetStorageStatsCommand command = new GetStorageStatsCommand(host.getGuid());
          Answer answer = _agentMgr.easySend(host.getId(), command);
          if (answer != null && answer.getResult()) {
            storageStats.put(host.getId(), (StorageStats) answer);
          }
        }

        sc = _hostDao.createSearchCriteria();
        sc.addAnd("status", SearchCriteria.Op.EQ, Status.Up.toString());
        sc.addAnd("type", SearchCriteria.Op.EQ, Host.Type.SecondaryStorage.toString());

        hosts = _hostDao.search(sc, null);
        for (HostVO host : hosts) {
          GetStorageStatsCommand command = new GetStorageStatsCommand(host.getGuid());
          Answer answer = _agentMgr.easySend(host.getId(), command);
          if (answer != null && answer.getResult()) {
            storageStats.put(host.getId(), (StorageStats) answer);
          }
        }
        _storageStats = storageStats;

        ConcurrentHashMap<Long, StorageStats> storagePoolStats =
            new ConcurrentHashMap<Long, StorageStats>();

        List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
        for (StoragePoolVO pool : storagePools) {
          GetStorageStatsCommand command =
              new GetStorageStatsCommand(pool.getUuid(), pool.getPoolType(), pool.getPath());
          Answer answer = _storageManager.sendToPool(pool, command);
          if (answer != null && answer.getResult()) {
            storagePoolStats.put(pool.getId(), (StorageStats) answer);
          }
        }
        _storagePoolStats = storagePoolStats;

        // a list to store the new capacity entries that will be committed once everything is
        // calculated
        List<CapacityVO> newCapacities = new ArrayList<CapacityVO>();

        // Updating the storage entries and creating new ones if they dont exist.
        Transaction txn = Transaction.open(Transaction.CLOUD_DB);
        try {
          if (s_logger.isTraceEnabled()) {
            s_logger.trace("recalculating system storage capacity");
          }
          txn.start();
          for (Long hostId : storageStats.keySet()) {
            StorageStats stats = storageStats.get(hostId);
            short capacityType = -1;
            HostVO host = _hostDao.findById(hostId);
            host.setTotalSize(stats.getCapacityBytes());
            _hostDao.update(host.getId(), host);

            SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
            capacitySC.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, hostId);
            capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, host.getDataCenterId());

            if (Host.Type.SecondaryStorage.equals(host.getType())) {
              capacityType = CapacityVO.CAPACITY_TYPE_SECONDARY_STORAGE;
            } else if (Host.Type.Storage.equals(host.getType())) {
              capacityType = CapacityVO.CAPACITY_TYPE_STORAGE;
            }
            if (-1 != capacityType) {
              capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);
              List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);
              if (capacities.size() == 0) { // Create a new one
                CapacityVO capacity =
                    new CapacityVO(
                        host.getId(),
                        host.getDataCenterId(),
                        host.getPodId(),
                        stats.getByteUsed(),
                        stats.getCapacityBytes(),
                        capacityType);
                _capacityDao.persist(capacity);
              } else { // Update if it already exists.
                CapacityVO capacity = capacities.get(0);
                capacity.setUsedCapacity(stats.getByteUsed());
                capacity.setTotalCapacity(stats.getCapacityBytes());
                _capacityDao.update(capacity.getId(), capacity);
              }
            }
          } // End of for
          txn.commit();
        } catch (Exception ex) {
          txn.rollback();
          s_logger.error("Unable to start transaction for storage capacity update");
        } finally {
          txn.close();
        }

        for (Long poolId : storagePoolStats.keySet()) {
          StorageStats stats = storagePoolStats.get(poolId);
          StoragePoolVO pool = _storagePoolDao.findById(poolId);

          if (pool == null) {
            continue;
          }

          pool.setCapacityBytes(stats.getCapacityBytes());
          long available = stats.getCapacityBytes() - stats.getByteUsed();
          if (available < 0) {
            available = 0;
          }
          pool.setAvailableBytes(available);
          _storagePoolDao.update(pool.getId(), pool);

          _storageManager.createCapacityEntry(pool, 0L);
        }
      } catch (Throwable t) {
        s_logger.error("Error trying to retrieve storage stats", t);
      }
    }
Пример #16
0
  private void downloadVolumeToStorage(
      VolumeVO volume, HostVO sserver, String url, String checkSum, ImageFormat format) {
    boolean downloadJobExists = false;
    VolumeHostVO volumeHost = null;

    volumeHost = _volumeHostDao.findByHostVolume(sserver.getId(), volume.getId());
    if (volumeHost == null) {
      volumeHost =
          new VolumeHostVO(
              sserver.getId(),
              volume.getId(),
              sserver.getDataCenterId(),
              new Date(),
              0,
              VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED,
              null,
              null,
              "jobid0000",
              null,
              url,
              checkSum,
              format);
      _volumeHostDao.persist(volumeHost);
    } else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) {
      downloadJobExists = true;
    }

    Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes();
    String secUrl = sserver.getStorageUrl();
    if (volumeHost != null) {
      start();
      DownloadCommand dcmd =
          new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format);
      if (downloadJobExists) {
        dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART);
        dcmd.setResourceType(ResourceType.VOLUME);
      }
      dcmd.setProxy(getHttpProxy());
      HostVO ssvm = _ssvmMgr.pickSsvmHost(sserver);
      if (ssvm == null) {
        s_logger.warn(
            "There is no secondary storage VM for secondary storage host " + sserver.getName());
        return;
      }
      DownloadListener dl =
          new DownloadListener(
              ssvm,
              sserver,
              volume,
              _timer,
              _volumeHostDao,
              volumeHost.getId(),
              this,
              dcmd,
              _volumeDao,
              _storageMgr,
              _resourceLimitMgr,
              _alertMgr,
              _accountMgr);

      if (downloadJobExists) {
        dl.setCurrState(volumeHost.getDownloadState());
      }
      DownloadListener old = null;
      synchronized (_listenerVolumeMap) {
        old = _listenerVolumeMap.put(volumeHost, dl);
      }
      if (old != null) {
        old.abandon();
      }

      try {
        send(ssvm.getId(), dcmd, dl);
      } catch (AgentUnavailableException e) {
        s_logger.warn(
            "Unable to start /resume download of volume "
                + volume.getName()
                + " to "
                + sserver.getName(),
            e);
        dl.setDisconnected();
        dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
      }
    }
  }
  protected Long restart(HaWorkVO work) {
    List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
    if (items.size() > 0) {
      StringBuilder str =
          new StringBuilder(
              "Cancelling this work item because newer ones have been scheduled.  Work Ids = [");
      for (HaWorkVO item : items) {
        str.append(item.getId()).append(", ");
      }
      str.delete(str.length() - 2, str.length()).append("]");
      s_logger.info(str.toString());
      return null;
    }

    items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
    if (items.size() > 0) {
      StringBuilder str =
          new StringBuilder(
              "Waiting because there's HA work being executed on an item currently.  Work Ids =[");
      for (HaWorkVO item : items) {
        str.append(item.getId()).append(", ");
      }
      str.delete(str.length() - 2, str.length()).append("]");
      s_logger.info(str.toString());
      return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
    }

    long vmId = work.getInstanceId();

    VMInstanceVO vm = _itMgr.findByIdAndType(work.getType(), work.getInstanceId());
    if (vm == null) {
      s_logger.info("Unable to find vm: " + vmId);
      return null;
    }

    s_logger.info("HA on " + vm);
    if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
      s_logger.info(
          "VM "
              + vm
              + " has been changed.  Current State = "
              + vm.getState()
              + " Previous State = "
              + work.getPreviousState()
              + " last updated = "
              + vm.getUpdated()
              + " previous updated = "
              + work.getUpdateTime());
      return null;
    }

    short alertType = AlertManager.ALERT_TYPE_USERVM;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
    } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_SSVM;
    }

    HostVO host = _hostDao.findById(work.getHostId());
    boolean isHostRemoved = false;
    if (host == null) {
      host = _hostDao.findByIdIncludingRemoved(work.getHostId());
      if (host != null) {
        s_logger.debug(
            "VM "
                + vm.toString()
                + " is now no longer on host "
                + work.getHostId()
                + " as the host is removed");
        isHostRemoved = true;
      }
    }

    DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
    HostPodVO podVO = _podDao.findById(host.getPodId());
    String hostDesc =
        "name: "
            + host.getName()
            + "(id:"
            + host.getId()
            + "), availability zone: "
            + dcVO.getName()
            + ", pod: "
            + podVO.getName();

    Boolean alive = null;
    if (work.getStep() == Step.Investigating) {
      if (!isHostRemoved) {
        if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
          s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
          return null;
        }

        Enumeration<Investigator> en = _investigators.enumeration();
        Investigator investigator = null;
        while (en.hasMoreElements()) {
          investigator = en.nextElement();
          alive = investigator.isVmAlive(vm, host);
          s_logger.info(investigator.getName() + " found " + vm + "to be alive? " + alive);
          if (alive != null) {
            break;
          }
        }
        boolean fenced = false;
        if (alive == null) {
          s_logger.debug("Fencing off VM that we don't know the state of");
          Enumeration<FenceBuilder> enfb = _fenceBuilders.enumeration();
          while (enfb.hasMoreElements()) {
            FenceBuilder fb = enfb.nextElement();
            Boolean result = fb.fenceOff(vm, host);
            s_logger.info("Fencer " + fb.getName() + " returned " + result);
            if (result != null && result) {
              fenced = true;
              break;
            }
          }
        } else if (!alive) {
          fenced = true;
        } else {
          s_logger.debug(
              "VM " + vm.getHostName() + " is found to be alive by " + investigator.getName());
          if (host.getStatus() == Status.Up) {
            s_logger.info(vm + " is alive and host is up. No need to restart it.");
            return null;
          } else {
            s_logger.debug("Rescheduling because the host is not up but the vm is alive");
            return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
          }
        }

        if (!fenced) {
          s_logger.debug("We were unable to fence off the VM " + vm);
          _alertMgr.sendAlert(
              alertType,
              vm.getDataCenterIdToDeployIn(),
              vm.getPodIdToDeployIn(),
              "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
              "Insufficient capacity to restart VM, name: "
                  + vm.getHostName()
                  + ", id: "
                  + vmId
                  + " which was running on host "
                  + hostDesc);
          return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
        }

        try {
          _itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
        } catch (ResourceUnavailableException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        } catch (OperationTimedoutException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        } catch (ConcurrentOperationException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        }

        work.setStep(Step.Scheduled);
        _haDao.update(work.getId(), work);
      } else {
        s_logger.debug(
            "How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
        try {
          _itMgr.advanceStop(vm, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
        } catch (ResourceUnavailableException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        } catch (OperationTimedoutException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        } catch (ConcurrentOperationException e) {
          assert false : "How do we hit this when force is true?";
          throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
        }
      }
    }

    vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());

    if (!_forceHA && !vm.isHaEnabled()) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("VM is not HA enabled so we're done.");
      }
      return null; // VM doesn't require HA
    }

    if (!_storageMgr.canVmRestartOnAnotherServer(vm.getId())) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("VM can not restart on another server.");
      }
      return null;
    }

    if (work.getTimesTried() > _maxRetries) {
      s_logger.warn("Retried to max times so deleting: " + vmId);
      return null;
    }

    try {
      VMInstanceVO started =
          _itMgr.advanceStart(
              vm,
              new HashMap<VirtualMachineProfile.Param, Object>(),
              _accountMgr.getSystemUser(),
              _accountMgr.getSystemAccount());
      if (started != null) {
        s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
        return null;
      }

      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
      }
    } catch (final InsufficientCapacityException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterIdToDeployIn(),
          vm.getPodIdToDeployIn(),
          "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
          "Insufficient capacity to restart VM, name: "
              + vm.getHostName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
    } catch (final ResourceUnavailableException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterIdToDeployIn(),
          vm.getPodIdToDeployIn(),
          "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getHostName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
    } catch (ConcurrentOperationException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterIdToDeployIn(),
          vm.getPodIdToDeployIn(),
          "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getHostName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
    } catch (OperationTimedoutException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterIdToDeployIn(),
          vm.getPodIdToDeployIn(),
          "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getHostName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
    }
    vm = _itMgr.findByIdAndType(vm.getType(), vm.getId());
    work.setUpdateTime(vm.getUpdated());
    work.setPreviousState(vm.getState());
    return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
  }
Пример #18
0
  @Override
  public void handleTemplateSync(HostVO ssHost) {
    if (ssHost == null) {
      s_logger.warn("Huh? ssHost is null");
      return;
    }
    long sserverId = ssHost.getId();
    long zoneId = ssHost.getDataCenterId();
    if (!(ssHost.getType() == Host.Type.SecondaryStorage
        || ssHost.getType() == Host.Type.LocalSecondaryStorage)) {
      s_logger.warn("Huh? Agent id " + sserverId + " is not secondary storage host");
      return;
    }

    Map<String, TemplateInfo> templateInfos = listTemplate(ssHost);
    if (templateInfos == null) {
      return;
    }

    Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
    List<VMTemplateVO> allTemplates = _templateDao.listAllInZone(zoneId);
    List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
    List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();

    if (rtngTmplts != null) {
      for (VMTemplateVO rtngTmplt : rtngTmplts) {
        if (!allTemplates.contains(rtngTmplt)) {
          allTemplates.add(rtngTmplt);
        }
      }
    }

    if (defaultBuiltin != null) {
      for (VMTemplateVO builtinTmplt : defaultBuiltin) {
        if (!allTemplates.contains(builtinTmplt)) {
          allTemplates.add(builtinTmplt);
        }
      }
    }

    toBeDownloaded.addAll(allTemplates);

    for (VMTemplateVO tmplt : allTemplates) {
      String uniqueName = tmplt.getUniqueName();
      VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(sserverId, tmplt.getId());
      if (templateInfos.containsKey(uniqueName)) {
        TemplateInfo tmpltInfo = templateInfos.remove(uniqueName);
        toBeDownloaded.remove(tmplt);
        if (tmpltHost != null) {
          s_logger.info(
              "Template Sync found " + tmplt.getName() + " already in the template host table");
          if (tmpltHost.getDownloadState() != Status.DOWNLOADED) {
            tmpltHost.setErrorString("");
          }
          if (tmpltInfo.isCorrupted()) {
            tmpltHost.setDownloadState(Status.DOWNLOAD_ERROR);
            String msg =
                "Template "
                    + tmplt.getName()
                    + ":"
                    + tmplt.getId()
                    + " is corrupted on secondary storage "
                    + tmpltHost.getId();
            tmpltHost.setErrorString(msg);
            s_logger.info("msg");
            if (tmplt.getUrl() == null) {
              msg =
                  "Private Template ("
                      + tmplt
                      + ") with install path "
                      + tmpltInfo.getInstallPath()
                      + "is corrupted, please check in secondary storage: "
                      + tmpltHost.getHostId();
              s_logger.warn(msg);
            } else {
              toBeDownloaded.add(tmplt);
            }

          } else {
            tmpltHost.setDownloadPercent(100);
            tmpltHost.setDownloadState(Status.DOWNLOADED);
            tmpltHost.setInstallPath(tmpltInfo.getInstallPath());
            tmpltHost.setSize(tmpltInfo.getSize());
            tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize());
            tmpltHost.setLastUpdated(new Date());

            // Skipping limit checks for SYSTEM Account and for the templates created from volumes
            // or snapshots
            // which already got checked and incremented during createTemplate API call.
            if (tmpltInfo.getSize() > 0
                && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
                && tmplt.getUrl() != null) {
              long accountId = tmplt.getAccountId();
              try {
                _resourceLimitMgr.checkResourceLimit(
                    _accountMgr.getAccount(accountId),
                    com.cloud.configuration.Resource.ResourceType.secondary_storage,
                    tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
              } catch (ResourceAllocationException e) {
                s_logger.warn(e.getMessage());
                _alertMgr.sendAlert(
                    _alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
                    ssHost.getDataCenterId(),
                    null,
                    e.getMessage(),
                    e.getMessage());
              } finally {
                _resourceLimitMgr.recalculateResourceCount(
                    accountId,
                    _accountMgr.getAccount(accountId).getDomainId(),
                    com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
              }
            }
          }
          _vmTemplateHostDao.update(tmpltHost.getId(), tmpltHost);
        } else {
          tmpltHost =
              new VMTemplateHostVO(
                  sserverId,
                  tmplt.getId(),
                  new Date(),
                  100,
                  Status.DOWNLOADED,
                  null,
                  null,
                  null,
                  tmpltInfo.getInstallPath(),
                  tmplt.getUrl());
          tmpltHost.setSize(tmpltInfo.getSize());
          tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize());
          _vmTemplateHostDao.persist(tmpltHost);
          VMTemplateZoneVO tmpltZoneVO =
              _vmTemplateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
          if (tmpltZoneVO == null) {
            tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date());
            _vmTemplateZoneDao.persist(tmpltZoneVO);
          } else {
            tmpltZoneVO.setLastUpdated(new Date());
            _vmTemplateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
          }
        }

        continue;
      }
      if (tmpltHost != null && tmpltHost.getDownloadState() != Status.DOWNLOADED) {
        s_logger.info(
            "Template Sync did not find "
                + tmplt.getName()
                + " ready on server "
                + sserverId
                + ", will request download to start/resume shortly");

      } else if (tmpltHost == null) {
        s_logger.info(
            "Template Sync did not find "
                + tmplt.getName()
                + " on the server "
                + sserverId
                + ", will request download shortly");
        VMTemplateHostVO templtHost =
            new VMTemplateHostVO(
                sserverId,
                tmplt.getId(),
                new Date(),
                0,
                Status.NOT_DOWNLOADED,
                null,
                null,
                null,
                null,
                tmplt.getUrl());
        _vmTemplateHostDao.persist(templtHost);
        VMTemplateZoneVO tmpltZoneVO = _vmTemplateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
        if (tmpltZoneVO == null) {
          tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date());
          _vmTemplateZoneDao.persist(tmpltZoneVO);
        } else {
          tmpltZoneVO.setLastUpdated(new Date());
          _vmTemplateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
        }
      }
    }

    if (toBeDownloaded.size() > 0) {
      /* Only download templates whose hypervirsor type is in the zone */
      List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
      if (availHypers.isEmpty()) {
        /*
         * This is for cloudzone, local secondary storage resource
         * started before cluster created
         */
        availHypers.add(HypervisorType.KVM);
      }
      /* Baremetal need not to download any template */
      availHypers.remove(HypervisorType.BareMetal);
      availHypers.add(HypervisorType.None); // bug 9809: resume ISO
      // download.
      for (VMTemplateVO tmplt : toBeDownloaded) {
        if (tmplt.getUrl() == null) { // If url is null we can't
          // initiate the download
          continue;
        }
        // if this is private template, and there is no record for this
        // template in this sHost, skip
        if (!tmplt.isPublicTemplate() && !tmplt.isFeatured()) {
          VMTemplateHostVO tmpltHost =
              _vmTemplateHostDao.findByHostTemplate(sserverId, tmplt.getId());
          if (tmpltHost == null) {
            continue;
          }
        }
        if (availHypers.contains(tmplt.getHypervisorType())) {
          if (_swiftMgr.isSwiftEnabled()) {
            if (_swiftMgr.isTemplateInstalled(tmplt.getId())) {
              continue;
            }
          }
          s_logger.debug(
              "Template " + tmplt.getName() + " needs to be downloaded to " + ssHost.getName());
          downloadTemplateToStorage(tmplt, ssHost);
        } else {
          s_logger.info(
              "Skipping download of template "
                  + tmplt.getName()
                  + " since we don't have any "
                  + tmplt.getHypervisorType()
                  + " hypervisors");
        }
      }
    }

    for (String uniqueName : templateInfos.keySet()) {
      TemplateInfo tInfo = templateInfos.get(uniqueName);
      List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(tInfo.getId());
      // check if there is any Vm using this ISO.
      if (userVmUsingIso == null || userVmUsingIso.isEmpty()) {
        DeleteTemplateCommand dtCommand =
            new DeleteTemplateCommand(ssHost.getStorageUrl(), tInfo.getInstallPath());
        try {
          _agentMgr.sendToSecStorage(ssHost, dtCommand, null);
        } catch (AgentUnavailableException e) {
          String err =
              "Failed to delete "
                  + tInfo.getTemplateName()
                  + " on secondary storage "
                  + sserverId
                  + " which isn't in the database";
          s_logger.error(err);
          return;
        }

        String description =
            "Deleted template "
                + tInfo.getTemplateName()
                + " on secondary storage "
                + sserverId
                + " since it isn't in the database";
        s_logger.info(description);
      }
    }
  }
    private boolean collectDirectNetworkUsage(HostVO host) {
      s_logger.debug("Direct Network Usage stats collector is running...");

      long zoneId = host.getDataCenterId();
      DetailVO lastCollectDetail = _detailsDao.findDetail(host.getId(), "last_collection");
      if (lastCollectDetail == null) {
        s_logger.warn(
            "Last collection time not available. Skipping direct usage collection for Traffic Monitor: "
                + host.getId());
        return false;
      }
      Date lastCollection = new Date(new Long(lastCollectDetail.getValue()));

      // Get list of IPs currently allocated
      List<IPAddressVO> allocatedIps = listAllocatedDirectIps(zoneId);
      Calendar rightNow = Calendar.getInstance();

      // Allow 2 hours for traffic sentinel to populate historical traffic
      // This coule be made configurable

      rightNow.add(Calendar.HOUR_OF_DAY, -2);
      Date now = rightNow.getTime();

      if (lastCollection.after(now)) {
        s_logger.debug(
            "Current time is less than 2 hours after last collection time : "
                + lastCollection.toString()
                + ". Skipping direct network usage collection");
        return false;
      }

      // Get IP Assign/Release events from lastCollection time till now
      List<UsageEventVO> IpEvents = _eventDao.listDirectIpEvents(lastCollection, now, zoneId);

      Map<String, Date> ipAssigment = new HashMap<String, Date>();
      List<UsageIPAddressVO> IpPartialUsage =
          new ArrayList<
              UsageIPAddressVO>(); // Ips which were allocated only for the part of collection
                                   // duration
      List<UsageIPAddressVO> fullDurationIpUsage =
          new ArrayList<
              UsageIPAddressVO>(); // Ips which were allocated only for the entire collection
                                   // duration

      // Use UsageEvents to track the IP assignment
      // Add them to IpUsage list with account_id , ip_address, alloc_date, release_date

      for (UsageEventVO IpEvent : IpEvents) {
        String address = IpEvent.getResourceName();
        if (EventTypes.EVENT_NET_IP_ASSIGN.equals(IpEvent.getType())) {
          ipAssigment.put(address, IpEvent.getCreateDate());
        } else if (EventTypes.EVENT_NET_IP_RELEASE.equals(IpEvent.getType())) {
          if (ipAssigment.containsKey(address)) {
            Date assigned = ipAssigment.get(address);
            ipAssigment.remove(address);
            IpPartialUsage.add(
                new UsageIPAddressVO(
                    IpEvent.getAccountId(), address, assigned, IpEvent.getCreateDate()));
          } else {
            // Ip was assigned prior to lastCollection Date
            IpPartialUsage.add(
                new UsageIPAddressVO(
                    IpEvent.getAccountId(), address, lastCollection, IpEvent.getCreateDate()));
          }
        }
      }

      List<String> IpList = new ArrayList<String>();
      for (IPAddressVO ip : allocatedIps) {
        if (ip.getAllocatedToAccountId() == AccountVO.ACCOUNT_ID_SYSTEM) {
          // Ignore usage for system account
          continue;
        }
        String address = (ip.getAddress()).toString();
        if (ipAssigment.containsKey(address)) {
          // Ip was assigned during the current period but not release till Date now
          IpPartialUsage.add(
              new UsageIPAddressVO(
                  ip.getAllocatedToAccountId(), address, ipAssigment.get(address), now));
        } else {
          // Ip was not assigned or released during current period. Consider entire duration for
          // usage calculation (lastCollection to now)
          fullDurationIpUsage.add(
              new UsageIPAddressVO(ip.getAllocatedToAccountId(), address, lastCollection, now));
          // Store just the Ips to send the list as part of DirectNetworkUsageCommand
          IpList.add(address);
        }
      }

      List<UserStatisticsVO> collectedStats = new ArrayList<UserStatisticsVO>();

      // Get usage for Ips which were assigned for the entire duration
      if (fullDurationIpUsage.size() > 0) {
        DirectNetworkUsageCommand cmd =
            new DirectNetworkUsageCommand(IpList, lastCollection, now, _TSinclZones, _TSexclZones);
        DirectNetworkUsageAnswer answer =
            (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
        if (answer == null || !answer.getResult()) {
          String details = (answer != null) ? answer.getDetails() : "details unavailable";
          String msg =
              "Unable to get network usage stats from "
                  + host.getId()
                  + " due to: "
                  + details
                  + ".";
          s_logger.error(msg);
          return false;
        } else {
          for (UsageIPAddressVO usageIp : fullDurationIpUsage) {
            String publicIp = usageIp.getAddress();
            long[] bytesSentRcvd = answer.get(publicIp);
            Long bytesSent = bytesSentRcvd[0];
            Long bytesRcvd = bytesSentRcvd[1];
            if (bytesSent == null || bytesRcvd == null) {
              s_logger.debug("Incorrect bytes for IP: " + publicIp);
              continue;
            }
            if (bytesSent == 0L && bytesRcvd == 0L) {
              s_logger.trace("Ignore zero bytes for IP: " + publicIp);
              continue;
            }
            UserStatisticsVO stats =
                new UserStatisticsVO(usageIp.getAccountId(), zoneId, null, null, null, null);
            stats.setCurrentBytesSent(bytesSent);
            stats.setCurrentBytesReceived(bytesRcvd);
            collectedStats.add(stats);
          }
        }
      }

      // Get usage for Ips which were assigned for part of the duration period
      for (UsageIPAddressVO usageIp : IpPartialUsage) {
        IpList = new ArrayList<String>();
        IpList.add(usageIp.getAddress());
        DirectNetworkUsageCommand cmd =
            new DirectNetworkUsageCommand(
                IpList, usageIp.getAssigned(), usageIp.getReleased(), _TSinclZones, _TSexclZones);
        DirectNetworkUsageAnswer answer =
            (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
        if (answer == null || !answer.getResult()) {
          String details = (answer != null) ? answer.getDetails() : "details unavailable";
          String msg =
              "Unable to get network usage stats from "
                  + host.getId()
                  + " due to: "
                  + details
                  + ".";
          s_logger.error(msg);
          return false;
        } else {
          String publicIp = usageIp.getAddress();
          long[] bytesSentRcvd = answer.get(publicIp);
          Long bytesSent = bytesSentRcvd[0];
          Long bytesRcvd = bytesSentRcvd[1];
          if (bytesSent == null || bytesRcvd == null) {
            s_logger.debug("Incorrect bytes for IP: " + publicIp);
            continue;
          }
          if (bytesSent == 0L && bytesRcvd == 0L) {
            s_logger.trace("Ignore zero bytes for IP: " + publicIp);
            continue;
          }
          UserStatisticsVO stats =
              new UserStatisticsVO(usageIp.getAccountId(), zoneId, null, null, null, null);
          stats.setCurrentBytesSent(bytesSent);
          stats.setCurrentBytesReceived(bytesRcvd);
          collectedStats.add(stats);
        }
      }

      if (collectedStats.size() == 0) {
        s_logger.debug("No new direct network stats. No need to persist");
        return false;
      }
      // Persist all the stats and last_collection time in a single transaction
      Transaction txn = Transaction.open(Transaction.CLOUD_DB);
      try {
        txn.start();
        for (UserStatisticsVO stat : collectedStats) {
          UserStatisticsVO stats =
              _statsDao.lock(
                  stat.getAccountId(),
                  stat.getDataCenterId(),
                  0L,
                  null,
                  host.getId(),
                  "DirectNetwork");
          if (stats == null) {
            stats =
                new UserStatisticsVO(
                    stat.getAccountId(), zoneId, null, host.getId(), "DirectNetwork", 0L);
            stats.setCurrentBytesSent(stat.getCurrentBytesSent());
            stats.setCurrentBytesReceived(stat.getCurrentBytesReceived());
            _statsDao.persist(stats);
          } else {
            stats.setCurrentBytesSent(stats.getCurrentBytesSent() + stat.getCurrentBytesSent());
            stats.setCurrentBytesReceived(
                stats.getCurrentBytesReceived() + stat.getCurrentBytesReceived());
            _statsDao.update(stats.getId(), stats);
          }
        }
        lastCollectDetail.setValue("" + now.getTime());
        _detailsDao.update(lastCollectDetail.getId(), lastCollectDetail);
        txn.commit();
      } finally {
        txn.close();
      }

      return true;
    }
  public Long migrate(final HaWorkVO work) {
    final long vmId = work.getInstanceId();

    final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());

    VMInstanceVO vm = mgr.get(vmId);
    if (vm == null || vm.getRemoved() != null) {
      s_logger.debug("Unable to find the vm " + vmId);
      return null;
    }

    s_logger.info("Migrating vm: " + vm.toString());
    if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
      s_logger.info("VM is not longer running on the current hostId");
      return null;
    }

    short alertType = AlertManager.ALERT_TYPE_USERVM_MIGRATE;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
    }

    HostVO fromHost = _hostDao.findById(vm.getHostId());
    String fromHostName = ((fromHost == null) ? "unknown" : fromHost.getName());
    HostVO toHost = null;
    if (work.getStep() == Step.Scheduled) {
      if (vm.getState() != State.Running) {
        s_logger.info(
            "VM's state is not ready for migration. "
                + vm.toString()
                + " State is "
                + vm.getState().toString());
        return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
      }

      DataCenterVO dcVO = _dcDao.findById(fromHost.getDataCenterId());
      HostPodVO podVO = _podDao.findById(fromHost.getPodId());

      try {
        toHost = mgr.prepareForMigration(vm);
        if (toHost == null) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Unable to find a host for migrating vm " + vmId);
          }
          _alertMgr.sendAlert(
              alertType,
              vm.getDataCenterId(),
              vm.getPodId(),
              "Unable to migrate vm "
                  + vm.getName()
                  + " from host "
                  + fromHostName
                  + " in zone "
                  + dcVO.getName()
                  + " and pod "
                  + podVO.getName(),
              "Unable to find a suitable host");
        }
      } catch (final InsufficientCapacityException e) {
        s_logger.warn("Unable to mgirate due to insufficient capacity " + vm.toString());
        _alertMgr.sendAlert(
            alertType,
            vm.getDataCenterId(),
            vm.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHostName
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Insufficient capacity");
      } catch (final StorageUnavailableException e) {
        s_logger.warn("Storage is unavailable: " + vm.toString());
        _alertMgr.sendAlert(
            alertType,
            vm.getDataCenterId(),
            vm.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHostName
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Storage is gone.");
      }

      if (toHost == null) {
        _agentMgr.maintenanceFailed(vm.getHostId());
        return null;
      }

      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Migrating from " + work.getHostId() + " to " + toHost.getId());
      }
      work.setStep(Step.Migrating);
      work.setHostId(toHost.getId());
      _haDao.update(work.getId(), work);
    }

    if (work.getStep() == Step.Migrating) {
      vm = mgr.get(vmId); // let's see if anything has changed.
      boolean migrated = false;
      if (vm == null
          || vm.getRemoved() != null
          || vm.getHostId() == null
          || !_itMgr.stateTransitTo(vm, Event.MigrationRequested, vm.getHostId())) {
        s_logger.info("Migration cancelled because state has changed: " + vm.toString());
      } else {
        try {
          boolean isWindows =
              _guestOSCategoryDao
                  .findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId())
                  .getName()
                  .equalsIgnoreCase("Windows");
          MigrateCommand cmd =
              new MigrateCommand(vm.getInstanceName(), toHost.getPrivateIpAddress(), isWindows);
          Answer answer = _agentMgr.send(fromHost.getId(), cmd);
          if (answer != null && answer.getResult()) {
            migrated = true;
            _storageMgr.unshare(vm, fromHost);
            work.setStep(Step.Investigating);
            _haDao.update(work.getId(), work);
          }
        } catch (final AgentUnavailableException e) {
          s_logger.debug("host became unavailable");
        } catch (final OperationTimedoutException e) {
          s_logger.debug("operation timed out");
          if (e.isActive()) {
            scheduleRestart(vm, true);
          }
        }
      }

      if (!migrated) {
        s_logger.info("Migration was unsuccessful.  Cleaning up: " + vm.toString());

        DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
        HostPodVO podVO = _podDao.findById(vm.getPodId());
        _alertMgr.sendAlert(
            alertType,
            fromHost.getDataCenterId(),
            fromHost.getPodId(),
            "Unable to migrate vm "
                + vm.getName()
                + " from host "
                + fromHost.getName()
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Migrate Command failed.  Please check logs.");

        _itMgr.stateTransitTo(vm, Event.MigrationFailedOnSource, toHost.getId());
        _agentMgr.maintenanceFailed(vm.getHostId());

        Command cleanup = mgr.cleanup(vm, null);
        _agentMgr.easySend(toHost.getId(), cleanup);
        _storageMgr.unshare(vm, toHost);

        return null;
      }
    }

    if (toHost == null) {
      toHost = _hostDao.findById(work.getHostId());
    }
    DataCenterVO dcVO = _dcDao.findById(toHost.getDataCenterId());
    HostPodVO podVO = _podDao.findById(toHost.getPodId());

    try {
      if (!mgr.completeMigration(vm, toHost)) {
        _alertMgr.sendAlert(
            alertType,
            toHost.getDataCenterId(),
            toHost.getPodId(),
            "Unable to migrate "
                + vmId
                + " to host "
                + toHost.getName()
                + " in zone "
                + dcVO.getName()
                + " and pod "
                + podVO.getName(),
            "Migration not completed");
        s_logger.warn("Unable to complete migration: " + vm.toString());
      } else {
        s_logger.info("Migration is complete: " + vm.toString());
      }
      return null;
    } catch (final AgentUnavailableException e) {
      s_logger.warn("Agent is unavailable for " + vm.toString());
    } catch (final OperationTimedoutException e) {
      s_logger.warn("Operation timed outfor " + vm.toString());
    }
    _itMgr.stateTransitTo(vm, Event.MigrationFailedOnDest, toHost.getId());
    return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
  }
  @Override
  public boolean generateSetupCommand(Long ssHostId) {
    HostVO cssHost = _hostDao.findById(ssHostId);
    Long zoneId = cssHost.getDataCenterId();
    if (cssHost.getType() == Host.Type.SecondaryStorageVM) {

      SecondaryStorageVmVO secStorageVm = _secStorageVmDao.findByInstanceName(cssHost.getName());
      if (secStorageVm == null) {
        s_logger.warn("secondary storage VM " + cssHost.getName() + " doesn't exist");
        return false;
      }

      List<HostVO> ssHosts = _hostDao.listSecondaryStorageHosts(zoneId);
      for (HostVO ssHost : ssHosts) {
        String secUrl = ssHost.getStorageUrl();
        SecStorageSetupCommand setupCmd = new SecStorageSetupCommand(secUrl);

        Answer answer = _agentMgr.easySend(ssHostId, setupCmd);
        if (answer != null && answer.getResult()) {
          SecStorageSetupAnswer an = (SecStorageSetupAnswer) answer;
          ssHost.setParent(an.get_dir());
          _hostDao.update(ssHost.getId(), ssHost);
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "Successfully programmed secondary storage "
                    + ssHost.getName()
                    + " in secondary storage VM "
                    + secStorageVm.getInstanceName());
          }
        } else {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "Successfully programmed secondary storage "
                    + ssHost.getName()
                    + " in secondary storage VM "
                    + secStorageVm.getInstanceName());
          }
          return false;
        }
      }
    } else if (cssHost.getType() == Host.Type.SecondaryStorage) {
      List<SecondaryStorageVmVO> alreadyRunning =
          _secStorageVmDao.getSecStorageVmListInStates(
              SecondaryStorageVm.Role.templateProcessor, zoneId, State.Running);
      String secUrl = cssHost.getStorageUrl();
      SecStorageSetupCommand setupCmd = new SecStorageSetupCommand(secUrl);
      for (SecondaryStorageVmVO ssVm : alreadyRunning) {
        HostVO host = _hostDao.findByName(ssVm.getInstanceName());
        Answer answer = _agentMgr.easySend(host.getId(), setupCmd);
        if (answer != null && answer.getResult()) {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "Successfully programmed secondary storage "
                    + host.getName()
                    + " in secondary storage VM "
                    + ssVm.getInstanceName());
          }
        } else {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(
                "Successfully programmed secondary storage "
                    + host.getName()
                    + " in secondary storage VM "
                    + ssVm.getInstanceName());
          }
          return false;
        }
      }
    }
    return true;
  }
  protected void loadResource(Long hostId) {
    HostVO host = hostDao.findById(hostId);
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("guid", host.getGuid());
    params.put("ipaddress", host.getPrivateIpAddress());
    params.put("username", "root");
    params.put("password", "password");
    params.put("zone", String.valueOf(host.getDataCenterId()));
    params.put("pod", String.valueOf(host.getPodId()));

    ServerResource resource = null;
    if (host.getHypervisorType() == HypervisorType.XenServer) {
      resource = new XcpOssResource();
      try {
        resource.configure(host.getName(), params);

      } catch (ConfigurationException e) {
        logger.debug("Failed to load resource:" + e.toString());
      }
    } else if (host.getHypervisorType() == HypervisorType.KVM) {
      resource = new LibvirtComputingResource();
      try {
        params.put("public.network.device", "cloudbr0");
        params.put("private.network.device", "cloudbr0");
        resource.configure(host.getName(), params);
      } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    } else if (host.getHypervisorType() == HypervisorType.VMware) {
      ClusterVO cluster = clusterDao.findById(host.getClusterId());
      String url = clusterDetailsDao.findDetail(cluster.getId(), "url").getValue();
      URI uri;
      try {
        uri = new URI(url);
        String userName = clusterDetailsDao.findDetail(cluster.getId(), "username").getValue();
        String password = clusterDetailsDao.findDetail(cluster.getId(), "password").getValue();
        VmwareServerDiscoverer discover = new VmwareServerDiscoverer();

        Map<? extends ServerResource, Map<String, String>> resources =
            discover.find(
                host.getDataCenterId(),
                host.getPodId(),
                host.getClusterId(),
                uri,
                userName,
                password,
                null);
        for (Map.Entry<? extends ServerResource, Map<String, String>> entry :
            resources.entrySet()) {
          resource = entry.getKey();
        }
        if (resource == null) {
          throw new CloudRuntimeException("can't find resource");
        }
      } catch (DiscoveryException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (URISyntaxException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }

    hostResourcesMap.put(hostId, resource);
    HostEnvironment env = new HostEnvironment();
    SetupCommand cmd = new SetupCommand(env);
    cmd.setNeedSetup(true);

    resource.executeRequest(cmd);
  }
Пример #23
0
  @DB
  @Override
  public void updateCapacityForHost(HostVO host) {
    // prepare the service offerings
    List<ServiceOfferingVO> offerings = _offeringsDao.listAllIncludingRemoved();
    Map<Long, ServiceOfferingVO> offeringsMap = new HashMap<Long, ServiceOfferingVO>();
    for (ServiceOfferingVO offering : offerings) {
      offeringsMap.put(offering.getId(), offering);
    }

    long usedCpu = 0;
    long usedMemory = 0;
    long reservedMemory = 0;
    long reservedCpu = 0;

    List<VMInstanceVO> vms = _vmDao.listUpByHostId(host.getId());
    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Found " + vms.size() + " VMs on host " + host.getId());
    }

    for (VMInstanceVO vm : vms) {
      ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
      usedMemory += so.getRamSize() * 1024L * 1024L;
      usedCpu += so.getCpu() * so.getSpeed();
    }

    List<VMInstanceVO> vmsByLastHostId = _vmDao.listByLastHostId(host.getId());
    if (s_logger.isDebugEnabled()) {
      s_logger.debug(
          "Found " + vmsByLastHostId.size() + " VM, not running on host " + host.getId());
    }
    for (VMInstanceVO vm : vmsByLastHostId) {
      long secondsSinceLastUpdate =
          (DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000;
      if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
        ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId());
        reservedMemory += so.getRamSize() * 1024L * 1024L;
        reservedCpu += so.getCpu() * so.getSpeed();
      }
    }

    CapacityVO cpuCap = _capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_CPU);
    CapacityVO memCap =
        _capacityDao.findByHostIdType(host.getId(), CapacityVO.CAPACITY_TYPE_MEMORY);

    if (cpuCap != null && memCap != null) {
      if (cpuCap.getUsedCapacity() == usedCpu && cpuCap.getReservedCapacity() == reservedCpu) {
        s_logger.debug(
            "No need to calibrate cpu capacity, host:"
                + host.getId()
                + " usedCpu: "
                + cpuCap.getUsedCapacity()
                + " reservedCpu: "
                + cpuCap.getReservedCapacity());
      } else if (cpuCap.getReservedCapacity() != reservedCpu) {
        s_logger.debug(
            "Calibrate reserved cpu for host: "
                + host.getId()
                + " old reservedCpu:"
                + cpuCap.getReservedCapacity()
                + " new reservedCpu:"
                + reservedCpu);
        cpuCap.setReservedCapacity(reservedCpu);
      } else if (cpuCap.getUsedCapacity() != usedCpu) {
        s_logger.debug(
            "Calibrate used cpu for host: "
                + host.getId()
                + " old usedCpu:"
                + cpuCap.getUsedCapacity()
                + " new usedCpu:"
                + usedCpu);
        cpuCap.setUsedCapacity(usedCpu);
      }

      if (memCap.getUsedCapacity() == usedMemory
          && memCap.getReservedCapacity() == reservedMemory) {
        s_logger.debug(
            "No need to calibrate memory capacity, host:"
                + host.getId()
                + " usedMem: "
                + memCap.getUsedCapacity()
                + " reservedMem: "
                + memCap.getReservedCapacity());
      } else if (memCap.getReservedCapacity() != reservedMemory) {
        s_logger.debug(
            "Calibrate reserved memory for host: "
                + host.getId()
                + " old reservedMem:"
                + memCap.getReservedCapacity()
                + " new reservedMem:"
                + reservedMemory);
        memCap.setReservedCapacity(reservedMemory);
      } else if (memCap.getUsedCapacity() != usedMemory) {
        /*
         * Didn't calibrate for used memory, because VMs can be in state(starting/migrating) that I don't know on which host they are
         * allocated
         */
        s_logger.debug(
            "Calibrate used memory for host: "
                + host.getId()
                + " old usedMem: "
                + memCap.getUsedCapacity()
                + " new usedMem: "
                + usedMemory);
        memCap.setUsedCapacity(usedMemory);
      }

      try {
        _capacityDao.update(cpuCap.getId(), cpuCap);
        _capacityDao.update(memCap.getId(), memCap);
      } catch (Exception e) {
        s_logger.error(
            "Caught exception while updating cpu/memory capacity for the host " + host.getId(), e);
      }
    } else {
      Transaction txn = Transaction.currentTxn();
      CapacityState capacityState =
          _configMgr.findClusterAllocationState(ApiDBUtils.findClusterById(host.getClusterId()))
                  == AllocationState.Disabled
              ? CapacityState.Disabled
              : CapacityState.Enabled;
      txn.start();
      CapacityVO capacity =
          new CapacityVO(
              host.getId(),
              host.getDataCenterId(),
              host.getPodId(),
              host.getClusterId(),
              usedMemory,
              host.getTotalMemory(),
              CapacityVO.CAPACITY_TYPE_MEMORY);
      capacity.setReservedCapacity(reservedMemory);
      capacity.setCapacityState(capacityState);
      _capacityDao.persist(capacity);

      capacity =
          new CapacityVO(
              host.getId(),
              host.getDataCenterId(),
              host.getPodId(),
              host.getClusterId(),
              usedCpu,
              (long) (host.getCpus().longValue() * host.getSpeed().longValue()),
              CapacityVO.CAPACITY_TYPE_CPU);
      capacity.setReservedCapacity(reservedCpu);
      capacity.setCapacityState(capacityState);
      _capacityDao.persist(capacity);
      txn.commit();
    }
  }
Пример #24
0
  @Test(priority = -1)
  public void setUp() {
    ComponentContext.initComponentsLifeCycle();

    host = hostDao.findByGuid(this.getHostGuid());
    if (host != null) {
      dcId = host.getDataCenterId();
      clusterId = host.getClusterId();
      podId = host.getPodId();
      return;
    }
    // create data center
    DataCenterVO dc =
        new DataCenterVO(
            UUID.randomUUID().toString(),
            "test",
            "8.8.8.8",
            null,
            "10.0.0.1",
            null,
            "10.0.0.1/24",
            null,
            null,
            NetworkType.Basic,
            null,
            null,
            true,
            true,
            null,
            null);
    dc = dcDao.persist(dc);
    dcId = dc.getId();
    // create pod

    HostPodVO pod =
        new HostPodVO(
            UUID.randomUUID().toString(),
            dc.getId(),
            this.getHostGateway(),
            this.getHostCidr(),
            8,
            "test");
    pod = podDao.persist(pod);
    podId = pod.getId();
    // create xen cluster
    ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
    cluster.setHypervisorType(HypervisorType.XenServer.toString());
    cluster.setClusterType(ClusterType.CloudManaged);
    cluster.setManagedState(ManagedState.Managed);
    cluster = clusterDao.persist(cluster);
    clusterId = cluster.getId();
    // create xen host

    host = new HostVO(this.getHostGuid());
    host.setName("devcloud xen host");
    host.setType(Host.Type.Routing);
    host.setPrivateIpAddress(this.getHostIp());
    host.setDataCenterId(dc.getId());
    host.setVersion("6.0.1");
    host.setAvailable(true);
    host.setSetup(true);
    host.setPodId(podId);
    host.setLastPinged(0);
    host.setResourceState(ResourceState.Enabled);
    host.setHypervisorType(HypervisorType.XenServer);
    host.setClusterId(cluster.getId());

    host = hostDao.persist(host);

    imageStore = new ImageStoreVO();
    imageStore.setName("test");
    imageStore.setDataCenterId(dcId);
    imageStore.setProviderName("CloudStack ImageStore Provider");
    imageStore.setRole(DataStoreRole.Image);
    imageStore.setUrl(this.getSecondaryStorage());
    imageStore.setUuid(UUID.randomUUID().toString());
    imageStore = imageStoreDao.persist(imageStore);
  }
Пример #25
0
  @Test(priority = -1)
  public void setUp() {
    ComponentContext.initComponentsLifeCycle();

    host = hostDao.findByGuid(this.getHostGuid());
    if (host != null) {
      dcId = host.getDataCenterId();
      clusterId = host.getClusterId();
      podId = host.getPodId();
      imageStore = this.imageStoreDao.findByName(imageStoreName);
    } else {
      // create data center
      DataCenterVO dc =
          new DataCenterVO(
              UUID.randomUUID().toString(),
              "test",
              "8.8.8.8",
              null,
              "10.0.0.1",
              null,
              "10.0.0.1/24",
              null,
              null,
              NetworkType.Basic,
              null,
              null,
              true,
              true,
              null,
              null);
      dc = dcDao.persist(dc);
      dcId = dc.getId();
      // create pod

      HostPodVO pod =
          new HostPodVO(
              UUID.randomUUID().toString(),
              dc.getId(),
              this.getHostGateway(),
              this.getHostCidr(),
              8,
              "test");
      pod = podDao.persist(pod);
      podId = pod.getId();
      // create xen cluster
      ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
      cluster.setHypervisorType(HypervisorType.VMware.toString());
      cluster.setClusterType(ClusterType.ExternalManaged);
      cluster.setManagedState(ManagedState.Managed);
      cluster = clusterDao.persist(cluster);
      clusterId = cluster.getId();

      // setup vcenter
      ClusterDetailsVO clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "url", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "username", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "password", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      // create xen host

      host = new HostVO(this.getHostGuid());
      host.setName("devcloud vmware host");
      host.setType(Host.Type.Routing);
      host.setPrivateIpAddress(this.getHostIp());
      host.setDataCenterId(dc.getId());
      host.setVersion("6.0.1");
      host.setAvailable(true);
      host.setSetup(true);
      host.setPodId(podId);
      host.setLastPinged(0);
      host.setResourceState(ResourceState.Enabled);
      host.setHypervisorType(HypervisorType.VMware);
      host.setClusterId(cluster.getId());

      host = hostDao.persist(host);

      imageStore = new ImageStoreVO();
      imageStore.setName(imageStoreName);
      imageStore.setDataCenterId(dcId);
      imageStore.setProviderName("CloudStack ImageStore Provider");
      imageStore.setRole(DataStoreRole.Image);
      imageStore.setUrl(this.getSecondaryStorage());
      imageStore.setUuid(UUID.randomUUID().toString());
      imageStore.setProtocol("nfs");
      imageStore = imageStoreDao.persist(imageStore);
    }

    image = new VMTemplateVO();
    image.setTemplateType(TemplateType.USER);
    image.setUrl(this.getTemplateUrl());
    image.setUniqueName(UUID.randomUUID().toString());
    image.setName(UUID.randomUUID().toString());
    image.setPublicTemplate(true);
    image.setFeatured(true);
    image.setRequiresHvm(true);
    image.setBits(64);
    image.setFormat(Storage.ImageFormat.VHD);
    image.setEnablePassword(true);
    image.setEnableSshKey(true);
    image.setGuestOSId(1);
    image.setBootable(true);
    image.setPrepopulate(true);
    image.setCrossZones(true);
    image.setExtractable(true);

    image = imageDataDao.persist(image);

    /*
     * TemplateDataStoreVO templateStore = new TemplateDataStoreVO();
     *
     * templateStore.setDataStoreId(imageStore.getId());
     * templateStore.setDownloadPercent(100);
     * templateStore.setDownloadState(Status.DOWNLOADED);
     * templateStore.setDownloadUrl(imageStore.getUrl());
     * templateStore.setInstallPath(this.getImageInstallPath());
     * templateStore.setTemplateId(image.getId());
     * templateStoreDao.persist(templateStore);
     */

    DataStore store = this.dataStoreMgr.getDataStore(imageStore.getId(), DataStoreRole.Image);
    TemplateInfo template = templateFactory.getTemplate(image.getId(), DataStoreRole.Image);
    DataObject templateOnStore = store.create(template);
    TemplateObjectTO to = new TemplateObjectTO();
    to.setPath(this.getImageInstallPath());
    CopyCmdAnswer answer = new CopyCmdAnswer(to);
    templateOnStore.processEvent(Event.CreateOnlyRequested);
    templateOnStore.processEvent(Event.OperationSuccessed, answer);
  }
  @Override
  public boolean generateFirewallConfiguration(Long ssAHostId) {
    if (ssAHostId == null) {
      return true;
    }
    HostVO ssAHost = _hostDao.findById(ssAHostId);
    Long zoneId = ssAHost.getDataCenterId();
    SecondaryStorageVmVO thisSecStorageVm = _secStorageVmDao.findByInstanceName(ssAHost.getName());

    if (thisSecStorageVm == null) {
      s_logger.warn("secondary storage VM " + ssAHost.getName() + " doesn't exist");
      return false;
    }

    List<SecondaryStorageVmVO> alreadyRunning =
        _secStorageVmDao.getSecStorageVmListInStates(
            SecondaryStorageVm.Role.templateProcessor,
            State.Running,
            State.Migrating,
            State.Starting);

    String copyPort =
        _useSSlCopy ? "443" : Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
    SecStorageFirewallCfgCommand cpc = new SecStorageFirewallCfgCommand();
    SecStorageFirewallCfgCommand thiscpc = new SecStorageFirewallCfgCommand();
    thiscpc.addPortConfig(
        thisSecStorageVm.getPublicIpAddress(),
        copyPort,
        true,
        TemplateConstants.DEFAULT_TMPLT_COPY_INTF);
    for (SecondaryStorageVmVO ssVm : alreadyRunning) {
      if (ssVm.getDataCenterIdToDeployIn() == zoneId) {
        continue;
      }
      if (ssVm.getPublicIpAddress() != null) {
        cpc.addPortConfig(
            ssVm.getPublicIpAddress(), copyPort, true, TemplateConstants.DEFAULT_TMPLT_COPY_INTF);
      }
      if (ssVm.getState() != State.Running) {
        continue;
      }
      String instanceName = ssVm.getInstanceName();
      HostVO host = _hostDao.findByName(instanceName);
      if (host == null) {
        continue;
      }
      Answer answer = _agentMgr.easySend(host.getId(), thiscpc);
      if (answer != null && answer.getResult()) {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug("Successfully programmed firewall rules into " + ssVm.getHostName());
        }
      } else {
        if (s_logger.isDebugEnabled()) {
          s_logger.debug(
              "failed to program firewall rules into secondary storage vm : " + ssVm.getHostName());
        }
        return false;
      }
    }

    Answer answer = _agentMgr.easySend(ssAHostId, cpc);
    if (answer != null && answer.getResult()) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "Successfully programmed firewall rules into " + thisSecStorageVm.getHostName());
      }
    } else {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "failed to program firewall rules into secondary storage vm : "
                + thisSecStorageVm.getHostName());
      }
      return false;
    }

    return true;
  }
Пример #27
0
  // TODO: Get rid of this case once we've determined that the capacity listeners above have all the
  // changes
  // create capacity entries if none exist for this server
  private void createCapacityEntry(StartupCommand startup, HostVO server) {
    SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
    capacitySC.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
    capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
    capacitySC.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());

    if (startup instanceof StartupRoutingCommand) {
      SearchCriteria<CapacityVO> capacityCPU = _capacityDao.createSearchCriteria();
      capacityCPU.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
      capacityCPU.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
      capacityCPU.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
      capacityCPU.addAnd("capacityType", SearchCriteria.Op.EQ, CapacityVO.CAPACITY_TYPE_CPU);
      List<CapacityVO> capacityVOCpus = _capacityDao.search(capacitySC, null);
      Float cpuovercommitratio =
          Float.parseFloat(
              _clusterDetailsDao
                  .findDetail(server.getClusterId(), "cpuOvercommitRatio")
                  .getValue());
      Float memoryOvercommitRatio =
          Float.parseFloat(
              _clusterDetailsDao
                  .findDetail(server.getClusterId(), "memoryOvercommitRatio")
                  .getValue());

      if (capacityVOCpus != null && !capacityVOCpus.isEmpty()) {
        CapacityVO CapacityVOCpu = capacityVOCpus.get(0);
        long newTotalCpu =
            (long)
                (server.getCpus().longValue() * server.getSpeed().longValue() * cpuovercommitratio);
        if ((CapacityVOCpu.getTotalCapacity() <= newTotalCpu)
            || ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity())
                <= newTotalCpu)) {
          CapacityVOCpu.setTotalCapacity(newTotalCpu);
        } else if ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity()
                > newTotalCpu)
            && (CapacityVOCpu.getUsedCapacity() < newTotalCpu)) {
          CapacityVOCpu.setReservedCapacity(0);
          CapacityVOCpu.setTotalCapacity(newTotalCpu);
        } else {
          s_logger.debug(
              "What? new cpu is :"
                  + newTotalCpu
                  + ", old one is "
                  + CapacityVOCpu.getUsedCapacity()
                  + ","
                  + CapacityVOCpu.getReservedCapacity()
                  + ","
                  + CapacityVOCpu.getTotalCapacity());
        }
        _capacityDao.update(CapacityVOCpu.getId(), CapacityVOCpu);
      } else {
        CapacityVO capacity =
            new CapacityVO(
                server.getId(),
                server.getDataCenterId(),
                server.getPodId(),
                server.getClusterId(),
                0L,
                (long) (server.getCpus().longValue() * server.getSpeed().longValue()),
                CapacityVO.CAPACITY_TYPE_CPU);
        _capacityDao.persist(capacity);
      }

      SearchCriteria<CapacityVO> capacityMem = _capacityDao.createSearchCriteria();
      capacityMem.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId());
      capacityMem.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId());
      capacityMem.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId());
      capacityMem.addAnd("capacityType", SearchCriteria.Op.EQ, CapacityVO.CAPACITY_TYPE_MEMORY);
      List<CapacityVO> capacityVOMems = _capacityDao.search(capacityMem, null);

      if (capacityVOMems != null && !capacityVOMems.isEmpty()) {
        CapacityVO CapacityVOMem = capacityVOMems.get(0);
        long newTotalMem = (long) ((server.getTotalMemory()) * memoryOvercommitRatio);
        if (CapacityVOMem.getTotalCapacity() <= newTotalMem
            || (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity()
                <= newTotalMem)) {
          CapacityVOMem.setTotalCapacity(newTotalMem);
        } else if (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity()
                > newTotalMem
            && CapacityVOMem.getUsedCapacity() < newTotalMem) {
          CapacityVOMem.setReservedCapacity(0);
          CapacityVOMem.setTotalCapacity(newTotalMem);
        } else {
          s_logger.debug(
              "What? new cpu is :"
                  + newTotalMem
                  + ", old one is "
                  + CapacityVOMem.getUsedCapacity()
                  + ","
                  + CapacityVOMem.getReservedCapacity()
                  + ","
                  + CapacityVOMem.getTotalCapacity());
        }
        _capacityDao.update(CapacityVOMem.getId(), CapacityVOMem);
      } else {
        CapacityVO capacity =
            new CapacityVO(
                server.getId(),
                server.getDataCenterId(),
                server.getPodId(),
                server.getClusterId(),
                0L,
                server.getTotalMemory(),
                CapacityVO.CAPACITY_TYPE_MEMORY);
        _capacityDao.persist(capacity);
      }
    }
  }
  protected Long restart(final HaWorkVO work) {
    final long vmId = work.getInstanceId();

    final VirtualMachineGuru<VMInstanceVO> mgr = findManager(work.getType());
    if (mgr == null) {
      s_logger.warn(
          "Unable to find a handler for " + work.getType().toString() + ", throwing out " + vmId);
      return null;
    }

    VMInstanceVO vm = mgr.get(vmId);
    if (vm == null) {
      s_logger.info("Unable to find vm: " + vmId);
      return null;
    }

    s_logger.info("HA on " + vm.toString());
    if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
      s_logger.info(
          "VM "
              + vm.toString()
              + " has been changed.  Current State = "
              + vm.getState()
              + " Previous State = "
              + work.getPreviousState()
              + " last updated = "
              + vm.getUpdated()
              + " previous updated = "
              + work.getUpdateTime());
      return null;
    }

    final HostVO host = _hostDao.findById(work.getHostId());

    DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
    HostPodVO podVO = _podDao.findById(host.getPodId());
    String hostDesc =
        "name: "
            + host.getName()
            + "(id:"
            + host.getId()
            + "), availability zone: "
            + dcVO.getName()
            + ", pod: "
            + podVO.getName();

    short alertType = AlertManager.ALERT_TYPE_USERVM;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_DOMAIN_ROUTER;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
      alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
    }

    Boolean alive = null;
    if (work.getStep() == Step.Investigating) {
      if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
        s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
        if (vm.getState() == State.Starting && vm.getUpdated() == work.getUpdateTime()) {
          _itMgr.stateTransitTo(vm, Event.AgentReportStopped, null);
        }
        return null;
      }

      Enumeration<Investigator> en = _investigators.enumeration();
      Investigator investigator = null;
      while (en.hasMoreElements()) {
        investigator = en.nextElement();
        alive = investigator.isVmAlive(vm, host);
        if (alive != null) {
          s_logger.debug(
              investigator.getName() + " found VM " + vm.getName() + "to be alive? " + alive);
          break;
        }
      }
      if (alive != null && alive) {
        s_logger.debug("VM " + vm.getName() + " is found to be alive by " + investigator.getName());
        if (host.getStatus() == Status.Up) {
          compareState(vm, new AgentVmInfo(vm.getInstanceName(), mgr, State.Running), false);
          return null;
        } else {
          s_logger.debug("Rescheduling because the host is not up but the vm is alive");
          return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
        }
      }

      boolean fenced = false;
      if (alive == null || !alive) {
        fenced = true;
        s_logger.debug("Fencing off VM that we don't know the state of");
        Enumeration<FenceBuilder> enfb = _fenceBuilders.enumeration();
        while (enfb.hasMoreElements()) {
          final FenceBuilder fb = enfb.nextElement();
          Boolean result = fb.fenceOff(vm, host);
          if (result != null && !result) {
            fenced = false;
          }
        }
      }

      if (alive == null && !fenced) {
        s_logger.debug("We were unable to fence off the VM " + vm.toString());
        _alertMgr.sendAlert(
            alertType,
            vm.getDataCenterId(),
            vm.getPodId(),
            "Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
            "Insufficient capacity to restart VM, name: "
                + vm.getName()
                + ", id: "
                + vmId
                + " which was running on host "
                + hostDesc);
        return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
      }

      mgr.completeStopCommand(vm);

      work.setStep(Step.Scheduled);
      _haDao.update(work.getId(), work);
    }

    // send an alert for VMs that stop unexpectedly
    _alertMgr.sendAlert(
        alertType,
        vm.getDataCenterId(),
        vm.getPodId(),
        "VM (name: "
            + vm.getName()
            + ", id: "
            + vmId
            + ") stopped unexpectedly on host "
            + hostDesc,
        "Virtual Machine "
            + vm.getName()
            + " (id: "
            + vm.getId()
            + ") running on host ["
            + hostDesc
            + "] stopped unexpectedly.");

    vm = mgr.get(vm.getId());

    if (!_forceHA && !vm.isHaEnabled()) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("VM is not HA enabled so we're done.");
      }
      return null; // VM doesn't require HA
    }

    if (!_storageMgr.canVmRestartOnAnotherServer(vm.getId())) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("VM can not restart on another server.");
      }
      return null;
    }

    if (work.getTimesTried() > _maxRetries) {
      s_logger.warn("Retried to max times so deleting: " + vmId);
      return null;
    }

    try {
      VMInstanceVO started = mgr.start(vm.getId(), 0);
      if (started != null) {
        s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
        return null;
      }

      if (s_logger.isDebugEnabled()) {
        s_logger.debug(
            "Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
      }
      vm = mgr.get(vm.getId());
      work.setUpdateTime(vm.getUpdated());
      work.setPreviousState(vm.getState());
      return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
    } catch (final InsufficientCapacityException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterId(),
          vm.getPodId(),
          "Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
          "Insufficient capacity to restart VM, name: "
              + vm.getName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
      return null;
    } catch (final StorageUnavailableException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterId(),
          vm.getPodId(),
          "Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
      return null;
    } catch (ConcurrentOperationException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterId(),
          vm.getPodId(),
          "Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
      return null;
    } catch (ExecutionException e) {
      s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
      _alertMgr.sendAlert(
          alertType,
          vm.getDataCenterId(),
          vm.getPodId(),
          "Unable to restart " + vm.getName() + " which was running on host " + hostDesc,
          "The Storage is unavailable for trying to restart VM, name: "
              + vm.getName()
              + ", id: "
              + vmId
              + " which was running on host "
              + hostDesc);
      return null;
    }
  }