示例#1
0
  public void testDownload() {
    s_logger.info("Testing Download answer");
    VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
    Mockito.when(template.getId()).thenReturn(1L);
    Mockito.when(template.getFormat()).thenReturn(ImageFormat.QCOW2);
    Mockito.when(template.getName()).thenReturn("templatename");
    Mockito.when(template.getTemplateType()).thenReturn(TemplateType.USER);
    Mockito.when(template.getDisplayText()).thenReturn("displayText");
    Mockito.when(template.getHypervisorType()).thenReturn(HypervisorType.KVM);
    Mockito.when(template.getUrl()).thenReturn("url");

    NfsTO nfs = new NfsTO("secUrl", DataStoreRole.Image);
    TemplateObjectTO to = new TemplateObjectTO(template);
    to.setImageDataStore(nfs);
    DownloadCommand cmd = new DownloadCommand(to, 30000000l);
    Request req = new Request(1, 1, cmd, true);

    req.logD("Debug for Download");

    DownloadAnswer answer =
        new DownloadAnswer(
            "jobId",
            50,
            "errorString",
            Status.ABANDONED,
            "filesystempath",
            "installpath",
            10000000,
            20000000,
            "chksum");
    Response resp = new Response(req, answer);
    resp.logD("Debug for Download");
  }
  public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {

    super(template.getName(), template.getOrigUrl(), template.getFormat(), template.getAccountId());
    _store = template.getDataStore();
    installPath = template.getPath();
    hvm = template.isRequiresHvm();
    checksum = template.getChecksum();
    id = template.getId();
    description = template.getDescription();
    if (_store instanceof NfsTO) {
      setSecUrl(((NfsTO) _store).getUrl());
    }
    this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
  }
  @Override
  protected Answer execute(CopyCommand cmd) {
    DataTO srcData = cmd.getSrcTO();
    DataTO destData = cmd.getDestTO();
    DataStoreTO srcDataStore = srcData.getDataStore();
    DataStoreTO destDataStore = destData.getDataStore();
    // if copied between s3 and nfs cache, go to resource
    boolean needDelegation = false;
    if (destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache) {
      if (srcDataStore instanceof S3TO || srcDataStore instanceof SwiftTO) {
        needDelegation = true;
      }
    }

    if (srcDataStore.getRole() == DataStoreRole.ImageCache
        && destDataStore.getRole() == DataStoreRole.Image) {
      // need to take extra processing for vmware, such as packing to ova, before sending to S3
      if (srcData.getObjectType() == DataObjectType.VOLUME) {
        NfsTO cacheStore = (NfsTO) srcDataStore;
        String parentPath = storageResource.getRootDir(cacheStore.getUrl(), _nfsVersion);
        VolumeObjectTO vol = (VolumeObjectTO) srcData;
        String path = vol.getPath();
        int index = path.lastIndexOf(File.separator);
        String name = path.substring(index + 1);
        storageManager.createOva(parentPath + File.separator + path, name);
        vol.setPath(path + File.separator + name + ".ova");
      } else if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
        // sync template from NFS cache to S3 in NFS migration to S3 case
        storageManager.createOvaForTemplate((TemplateObjectTO) srcData);
      } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT) {
        // pack ova first
        // sync snapshot from NFS cache to S3 in NFS migration to S3 case
        String parentPath = storageResource.getRootDir(srcDataStore.getUrl(), _nfsVersion);
        SnapshotObjectTO snap = (SnapshotObjectTO) srcData;
        String path = snap.getPath();
        int index = path.lastIndexOf(File.separator);
        String name = path.substring(index + 1);
        String snapDir = path.substring(0, index);
        storageManager.createOva(parentPath + File.separator + snapDir, name);
        if (destData.getObjectType() == DataObjectType.TEMPLATE) {
          // create template from snapshot on src at first, then copy it to s3
          TemplateObjectTO cacheTemplate = (TemplateObjectTO) destData;
          cacheTemplate.setDataStore(srcDataStore);
          CopyCmdAnswer answer = (CopyCmdAnswer) processor.createTemplateFromSnapshot(cmd);
          if (!answer.getResult()) {
            return answer;
          }
          cacheTemplate.setDataStore(destDataStore);
          TemplateObjectTO template = (TemplateObjectTO) answer.getNewData();
          template.setDataStore(srcDataStore);
          CopyCommand newCmd =
              new CopyCommand(template, destData, cmd.getWait(), cmd.executeInSequence());
          Answer result = storageResource.defaultAction(newCmd);
          // clean up template data on staging area
          try {
            DeleteCommand deleteCommand = new DeleteCommand(template);
            storageResource.defaultAction(deleteCommand);
          } catch (Exception e) {
            s_logger.debug("Failed to clean up staging area:", e);
          }
          return result;
        }
      }
      needDelegation = true;
    }

    if (srcData.getObjectType() == DataObjectType.SNAPSHOT
        && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
      // for back up snapshot, we need to do backup to cache, then to object store if object store
      // is used.
      if (cmd.getCacheTO() != null) {
        cmd.setDestTO(cmd.getCacheTO());

        CopyCmdAnswer answer = (CopyCmdAnswer) processor.backupSnapshot(cmd);
        if (!answer.getResult()) {
          return answer;
        }
        NfsTO cacheStore = (NfsTO) cmd.getCacheTO().getDataStore();
        String parentPath = storageResource.getRootDir(cacheStore.getUrl(), _nfsVersion);
        SnapshotObjectTO newSnapshot = (SnapshotObjectTO) answer.getNewData();
        String path = newSnapshot.getPath();
        int index = path.lastIndexOf(File.separator);
        String name = path.substring(index + 1);
        String dir = path.substring(0, index);
        storageManager.createOva(parentPath + File.separator + dir, name);
        newSnapshot.setPath(newSnapshot.getPath() + ".ova");
        newSnapshot.setDataStore(cmd.getCacheTO().getDataStore());
        CopyCommand newCmd =
            new CopyCommand(newSnapshot, destData, cmd.getWait(), cmd.executeInSequence());
        Answer result = storageResource.defaultAction(newCmd);

        // clean up data on staging area
        try {
          newSnapshot.setPath(path);
          DeleteCommand deleteCommand = new DeleteCommand(newSnapshot);
          storageResource.defaultAction(deleteCommand);
        } catch (Exception e) {
          s_logger.debug("Failed to clean up staging area:", e);
        }
        return result;
      }
    }

    if (needDelegation) {
      return storageResource.defaultAction(cmd);
    } else {
      return super.execute(cmd);
    }
  }
  @Override
  public void handleTemplateSync(DataStore store) {
    if (store == null) {
      s_logger.warn("Huh? image store is null");
      return;
    }
    long storeId = store.getId();

    // add lock to make template sync for a data store only be done once
    String lockString = "templatesync.storeId:" + storeId;
    GlobalLock syncLock = GlobalLock.getInternLock(lockString);
    try {
      if (syncLock.lock(3)) {
        try {
          Long zoneId = store.getScope().getScopeId();

          Map<String, TemplateProp> templateInfos = listTemplate(store);
          if (templateInfos == null) {
            return;
          }

          Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
          List<VMTemplateVO> allTemplates = null;
          if (zoneId == null) {
            // region wide store
            allTemplates = _templateDao.listAllActive();
          } else {
            // zone wide store
            allTemplates = _templateDao.listAllInZone(zoneId);
          }
          List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
          List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();

          if (rtngTmplts != null) {
            for (VMTemplateVO rtngTmplt : rtngTmplts) {
              if (!allTemplates.contains(rtngTmplt)) {
                allTemplates.add(rtngTmplt);
              }
            }
          }

          if (defaultBuiltin != null) {
            for (VMTemplateVO builtinTmplt : defaultBuiltin) {
              if (!allTemplates.contains(builtinTmplt)) {
                allTemplates.add(builtinTmplt);
              }
            }
          }

          toBeDownloaded.addAll(allTemplates);

          for (VMTemplateVO tmplt : allTemplates) {
            String uniqueName = tmplt.getUniqueName();
            TemplateDataStoreVO tmpltStore =
                _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
            if (templateInfos.containsKey(uniqueName)) {
              TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
              toBeDownloaded.remove(tmplt);
              if (tmpltStore != null) {
                s_logger.info("Template Sync found " + uniqueName + " already in the image store");
                if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
                  tmpltStore.setErrorString("");
                }
                if (tmpltInfo.isCorrupted()) {
                  tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
                  String msg =
                      "Template "
                          + tmplt.getName()
                          + ":"
                          + tmplt.getId()
                          + " is corrupted on secondary storage "
                          + tmpltStore.getId();
                  tmpltStore.setErrorString(msg);
                  s_logger.info("msg");
                  if (tmplt.getUrl() == null) {
                    msg =
                        "Private Template ("
                            + tmplt
                            + ") with install path "
                            + tmpltInfo.getInstallPath()
                            + "is corrupted, please check in image store: "
                            + tmpltStore.getDataStoreId();
                    s_logger.warn(msg);
                  } else {
                    s_logger.info(
                        "Removing template_store_ref entry for corrupted template "
                            + tmplt.getName());
                    _vmTemplateStoreDao.remove(tmpltStore.getId());
                    toBeDownloaded.add(tmplt);
                  }

                } else {
                  tmpltStore.setDownloadPercent(100);
                  tmpltStore.setDownloadState(Status.DOWNLOADED);
                  tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
                  tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
                  tmpltStore.setSize(tmpltInfo.getSize());
                  tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
                  tmpltStore.setLastUpdated(new Date());
                  // update size in vm_template table
                  VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
                  tmlpt.setSize(tmpltInfo.getSize());
                  _templateDao.update(tmplt.getId(), tmlpt);

                  // Skipping limit checks for SYSTEM Account and for the templates created from
                  // volumes or snapshots
                  // which already got checked and incremented during createTemplate API call.
                  if (tmpltInfo.getSize() > 0
                      && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
                      && tmplt.getUrl() != null) {
                    long accountId = tmplt.getAccountId();
                    try {
                      _resourceLimitMgr.checkResourceLimit(
                          _accountMgr.getAccount(accountId),
                          com.cloud.configuration.Resource.ResourceType.secondary_storage,
                          tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
                    } catch (ResourceAllocationException e) {
                      s_logger.warn(e.getMessage());
                      _alertMgr.sendAlert(
                          AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
                          zoneId,
                          null,
                          e.getMessage(),
                          e.getMessage());
                    } finally {
                      _resourceLimitMgr.recalculateResourceCount(
                          accountId,
                          _accountMgr.getAccount(accountId).getDomainId(),
                          com.cloud.configuration.Resource.ResourceType.secondary_storage
                              .getOrdinal());
                    }
                  }
                }
                _vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
              } else {
                tmpltStore =
                    new TemplateDataStoreVO(
                        storeId,
                        tmplt.getId(),
                        new Date(),
                        100,
                        Status.DOWNLOADED,
                        null,
                        null,
                        null,
                        tmpltInfo.getInstallPath(),
                        tmplt.getUrl());
                tmpltStore.setSize(tmpltInfo.getSize());
                tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
                tmpltStore.setDataStoreRole(store.getRole());
                _vmTemplateStoreDao.persist(tmpltStore);

                // update size in vm_template table
                VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
                tmlpt.setSize(tmpltInfo.getSize());
                _templateDao.update(tmplt.getId(), tmlpt);
                associateTemplateToZone(tmplt.getId(), zoneId);
              }
            } else {
              s_logger.info(
                  "Template Sync did not find "
                      + uniqueName
                      + " on image store "
                      + storeId
                      + ", may request download based on available hypervisor types");
              if (tmpltStore != null) {
                if (isRegionStore(store)
                    && tmpltStore.getDownloadState()
                        == VMTemplateStorageResourceAssoc.Status.DOWNLOADED
                    && tmpltStore.getState() == State.Ready
                    && tmpltStore.getInstallPath() == null) {
                  s_logger.info(
                      "Keep fake entry in template store table for migration of previous NFS to object store");
                } else {
                  s_logger.info(
                      "Removing leftover template "
                          + uniqueName
                          + " entry from template store table");
                  // remove those leftover entries
                  _vmTemplateStoreDao.remove(tmpltStore.getId());
                }
              }
            }
          }

          if (toBeDownloaded.size() > 0) {
            /* Only download templates whose hypervirsor type is in the zone */
            List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
            if (availHypers.isEmpty()) {
              /*
               * This is for cloudzone, local secondary storage resource
               * started before cluster created
               */
              availHypers.add(HypervisorType.KVM);
            }
            /* Baremetal need not to download any template */
            availHypers.remove(HypervisorType.BareMetal);
            availHypers.add(HypervisorType.None); // bug 9809: resume ISO
            // download.
            for (VMTemplateVO tmplt : toBeDownloaded) {
              if (tmplt.getUrl() == null) { // If url is null we can't
                s_logger.info(
                    "Skip downloading template "
                        + tmplt.getUniqueName()
                        + " since no url is specified.");
                continue;
              }
              // if this is private template, skip sync to a new image store
              if (!tmplt.isPublicTemplate()
                  && !tmplt.isFeatured()
                  && tmplt.getTemplateType() != TemplateType.SYSTEM) {
                s_logger.info(
                    "Skip sync downloading private template "
                        + tmplt.getUniqueName()
                        + " to a new image store");
                continue;
              }

              // if this is a region store, and there is already an DOWNLOADED entry there without
              // install_path information, which
              // means that this is a duplicate entry from migration of previous NFS to staging.
              if (isRegionStore(store)) {
                TemplateDataStoreVO tmpltStore =
                    _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
                if (tmpltStore != null
                    && tmpltStore.getDownloadState()
                        == VMTemplateStorageResourceAssoc.Status.DOWNLOADED
                    && tmpltStore.getState() == State.Ready
                    && tmpltStore.getInstallPath() == null) {
                  s_logger.info("Skip sync template for migration of previous NFS to object store");
                  continue;
                }
              }

              if (availHypers.contains(tmplt.getHypervisorType())) {
                s_logger.info(
                    "Downloading template "
                        + tmplt.getUniqueName()
                        + " to image store "
                        + store.getName());
                associateTemplateToZone(tmplt.getId(), zoneId);
                TemplateInfo tmpl =
                    _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
                createTemplateAsync(tmpl, store, null);
              } else {
                s_logger.info(
                    "Skip downloading template "
                        + tmplt.getUniqueName()
                        + " since current data center does not have hypervisor "
                        + tmplt.getHypervisorType().toString());
              }
            }
          }

          for (String uniqueName : templateInfos.keySet()) {
            TemplateProp tInfo = templateInfos.get(uniqueName);
            if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
              // we cannot directly call deleteTemplateSync here to
              // reuse delete logic since in this case, our db does not have
              // this template at all.
              TemplateObjectTO tmplTO = new TemplateObjectTO();
              tmplTO.setDataStore(store.getTO());
              tmplTO.setPath(tInfo.getInstallPath());
              tmplTO.setId(tInfo.getId());
              DeleteCommand dtCommand = new DeleteCommand(tmplTO);
              EndPoint ep = _epSelector.select(store);
              Answer answer = null;
              if (ep == null) {
                String errMsg =
                    "No remote endpoint to send command, check if host or ssvm is down?";
                s_logger.error(errMsg);
                answer = new Answer(dtCommand, false, errMsg);
              } else {
                answer = ep.sendMessage(dtCommand);
              }
              if (answer == null || !answer.getResult()) {
                s_logger.info("Failed to deleted template at store: " + store.getName());

              } else {
                String description =
                    "Deleted template "
                        + tInfo.getTemplateName()
                        + " on secondary storage "
                        + storeId;
                s_logger.info(description);
              }
            }
          }
        } finally {
          syncLock.unlock();
        }
      } else {
        s_logger.info(
            "Couldn't get global lock on "
                + lockString
                + ", another thread may be doing template sync on data store "
                + storeId
                + " now.");
      }
    } finally {
      syncLock.releaseRef();
    }
  }
  @Test(priority = -1)
  public void setUp() {
    ComponentContext.initComponentsLifeCycle();

    host = hostDao.findByGuid(this.getHostGuid());
    if (host != null) {
      dcId = host.getDataCenterId();
      clusterId = host.getClusterId();
      podId = host.getPodId();
      imageStore = this.imageStoreDao.findByName(imageStoreName);
    } else {
      // create data center
      DataCenterVO dc =
          new DataCenterVO(
              UUID.randomUUID().toString(),
              "test",
              "8.8.8.8",
              null,
              "10.0.0.1",
              null,
              "10.0.0.1/24",
              null,
              null,
              NetworkType.Basic,
              null,
              null,
              true,
              true,
              null,
              null);
      dc = dcDao.persist(dc);
      dcId = dc.getId();
      // create pod

      HostPodVO pod =
          new HostPodVO(
              UUID.randomUUID().toString(),
              dc.getId(),
              this.getHostGateway(),
              this.getHostCidr(),
              8,
              "test");
      pod = podDao.persist(pod);
      podId = pod.getId();
      // create xen cluster
      ClusterVO cluster = new ClusterVO(dc.getId(), pod.getId(), "devcloud cluster");
      cluster.setHypervisorType(HypervisorType.VMware.toString());
      cluster.setClusterType(ClusterType.ExternalManaged);
      cluster.setManagedState(ManagedState.Managed);
      cluster = clusterDao.persist(cluster);
      clusterId = cluster.getId();

      // setup vcenter
      ClusterDetailsVO clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "url", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "username", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      clusterDetailVO = new ClusterDetailsVO(cluster.getId(), "password", null);
      this.clusterDetailsDao.persist(clusterDetailVO);
      // create xen host

      host = new HostVO(this.getHostGuid());
      host.setName("devcloud vmware host");
      host.setType(Host.Type.Routing);
      host.setPrivateIpAddress(this.getHostIp());
      host.setDataCenterId(dc.getId());
      host.setVersion("6.0.1");
      host.setAvailable(true);
      host.setSetup(true);
      host.setPodId(podId);
      host.setLastPinged(0);
      host.setResourceState(ResourceState.Enabled);
      host.setHypervisorType(HypervisorType.VMware);
      host.setClusterId(cluster.getId());

      host = hostDao.persist(host);

      imageStore = new ImageStoreVO();
      imageStore.setName(imageStoreName);
      imageStore.setDataCenterId(dcId);
      imageStore.setProviderName("CloudStack ImageStore Provider");
      imageStore.setRole(DataStoreRole.Image);
      imageStore.setUrl(this.getSecondaryStorage());
      imageStore.setUuid(UUID.randomUUID().toString());
      imageStore.setProtocol("nfs");
      imageStore = imageStoreDao.persist(imageStore);
    }

    image = new VMTemplateVO();
    image.setTemplateType(TemplateType.USER);
    image.setUrl(this.getTemplateUrl());
    image.setUniqueName(UUID.randomUUID().toString());
    image.setName(UUID.randomUUID().toString());
    image.setPublicTemplate(true);
    image.setFeatured(true);
    image.setRequiresHvm(true);
    image.setBits(64);
    image.setFormat(Storage.ImageFormat.VHD);
    image.setEnablePassword(true);
    image.setEnableSshKey(true);
    image.setGuestOSId(1);
    image.setBootable(true);
    image.setPrepopulate(true);
    image.setCrossZones(true);
    image.setExtractable(true);

    image = imageDataDao.persist(image);

    /*
     * TemplateDataStoreVO templateStore = new TemplateDataStoreVO();
     *
     * templateStore.setDataStoreId(imageStore.getId());
     * templateStore.setDownloadPercent(100);
     * templateStore.setDownloadState(Status.DOWNLOADED);
     * templateStore.setDownloadUrl(imageStore.getUrl());
     * templateStore.setInstallPath(this.getImageInstallPath());
     * templateStore.setTemplateId(image.getId());
     * templateStoreDao.persist(templateStore);
     */

    DataStore store = this.dataStoreMgr.getDataStore(imageStore.getId(), DataStoreRole.Image);
    TemplateInfo template = templateFactory.getTemplate(image.getId(), DataStoreRole.Image);
    DataObject templateOnStore = store.create(template);
    TemplateObjectTO to = new TemplateObjectTO();
    to.setPath(this.getImageInstallPath());
    CopyCmdAnswer answer = new CopyCmdAnswer(to);
    templateOnStore.processEvent(Event.CreateOnlyRequested);
    templateOnStore.processEvent(Event.OperationSuccessed, answer);
  }