private void handleTemplateStatusResponse(
        final UploadStatusAnswer answer,
        final VMTemplateVO template,
        final TemplateDataStoreVO templateDataStore) {
      final StateMachine2<
              VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate>
          stateMachine = VirtualMachineTemplate.State.getStateMachine();
      Transaction.execute(
          new TransactionCallbackNoReturn() {
            @Override
            public void doInTransactionWithoutResult(TransactionStatus status) {
              VMTemplateVO tmpTemplate = _templateDao.findById(template.getId());
              TemplateDataStoreVO tmpTemplateDataStore =
                  _templateDataStoreDao.findById(templateDataStore.getId());
              try {
                switch (answer.getStatus()) {
                  case COMPLETED:
                    tmpTemplateDataStore.setDownloadState(
                        VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
                    tmpTemplateDataStore.setState(State.Ready);
                    tmpTemplateDataStore.setInstallPath(answer.getInstallPath());
                    tmpTemplateDataStore.setPhysicalSize(answer.getPhysicalSize());
                    tmpTemplateDataStore.setSize(answer.getVirtualSize());
                    tmpTemplateDataStore.setDownloadPercent(100);
                    tmpTemplateDataStore.setExtractUrl(null);

                    VMTemplateVO templateUpdate = _templateDao.createForUpdate();
                    templateUpdate.setSize(answer.getVirtualSize());
                    _templateDao.update(tmpTemplate.getId(), templateUpdate);
                    stateMachine.transitTo(
                        tmpTemplate,
                        VirtualMachineTemplate.Event.OperationSucceeded,
                        null,
                        _templateDao);
                    _resourceLimitMgr.incrementResourceCount(
                        template.getAccountId(),
                        Resource.ResourceType.secondary_storage,
                        answer.getVirtualSize());

                    if (s_logger.isDebugEnabled()) {
                      s_logger.debug(
                          "Template " + tmpTemplate.getUuid() + " uploaded successfully");
                    }
                    break;
                  case IN_PROGRESS:
                    if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) {
                      tmpTemplateDataStore.setDownloadState(
                          VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
                      stateMachine.transitTo(
                          tmpTemplate,
                          VirtualMachineTemplate.Event.UploadRequested,
                          null,
                          _templateDao);
                      tmpTemplateDataStore.setDownloadPercent(answer.getDownloadPercent());
                    } else if (tmpTemplate.getState()
                        == VirtualMachineTemplate.State.UploadInProgress) { // check for timeout
                      if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime()
                          > _uploadOperationTimeout) {
                        tmpTemplateDataStore.setDownloadState(
                            VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
                        tmpTemplateDataStore.setState(State.Failed);
                        stateMachine.transitTo(
                            tmpTemplate,
                            VirtualMachineTemplate.Event.OperationFailed,
                            null,
                            _templateDao);
                        if (s_logger.isDebugEnabled()) {
                          s_logger.debug(
                              "Template "
                                  + tmpTemplate.getUuid()
                                  + " failed to upload due to operation timed out");
                        }
                      } else {
                        tmpTemplateDataStore.setDownloadPercent(answer.getDownloadPercent());
                      }
                    }
                    break;
                  case ERROR:
                    tmpTemplateDataStore.setDownloadState(
                        VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
                    tmpTemplateDataStore.setState(State.Failed);
                    stateMachine.transitTo(
                        tmpTemplate,
                        VirtualMachineTemplate.Event.OperationFailed,
                        null,
                        _templateDao);
                    if (s_logger.isDebugEnabled()) {
                      s_logger.debug(
                          "Template "
                              + tmpTemplate.getUuid()
                              + " failed to upload. Error details: "
                              + answer.getDetails());
                    }
                    break;
                  case UNKNOWN:
                    if (tmpTemplate.getState()
                        == VirtualMachineTemplate.State.NotUploaded) { // check for timeout
                      if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime()
                          > _uploadOperationTimeout) {
                        tmpTemplateDataStore.setDownloadState(
                            VMTemplateStorageResourceAssoc.Status.ABANDONED);
                        tmpTemplateDataStore.setState(State.Failed);
                        stateMachine.transitTo(
                            tmpTemplate,
                            VirtualMachineTemplate.Event.OperationTimeout,
                            null,
                            _templateDao);
                        if (s_logger.isDebugEnabled()) {
                          s_logger.debug(
                              "Template "
                                  + tmpTemplate.getUuid()
                                  + " failed to upload due to operation timed out");
                        }
                      }
                    }
                    break;
                }
                _templateDataStoreDao.update(tmpTemplateDataStore.getId(), tmpTemplateDataStore);
              } catch (NoTransitionException e) {
                s_logger.error("Unexpected error " + e.getMessage());
              }
            }
          });
    }
  @Override
  public void handleTemplateSync(DataStore store) {
    if (store == null) {
      s_logger.warn("Huh? image store is null");
      return;
    }
    long storeId = store.getId();

    // add lock to make template sync for a data store only be done once
    String lockString = "templatesync.storeId:" + storeId;
    GlobalLock syncLock = GlobalLock.getInternLock(lockString);
    try {
      if (syncLock.lock(3)) {
        try {
          Long zoneId = store.getScope().getScopeId();

          Map<String, TemplateProp> templateInfos = listTemplate(store);
          if (templateInfos == null) {
            return;
          }

          Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
          List<VMTemplateVO> allTemplates = null;
          if (zoneId == null) {
            // region wide store
            allTemplates =
                _templateDao.listByState(
                    VirtualMachineTemplate.State.Active,
                    VirtualMachineTemplate.State.NotUploaded,
                    VirtualMachineTemplate.State.UploadInProgress);
          } else {
            // zone wide store
            allTemplates =
                _templateDao.listInZoneByState(
                    zoneId,
                    VirtualMachineTemplate.State.Active,
                    VirtualMachineTemplate.State.NotUploaded,
                    VirtualMachineTemplate.State.UploadInProgress);
          }
          List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
          List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();

          if (rtngTmplts != null) {
            for (VMTemplateVO rtngTmplt : rtngTmplts) {
              if (!allTemplates.contains(rtngTmplt)) {
                allTemplates.add(rtngTmplt);
              }
            }
          }

          if (defaultBuiltin != null) {
            for (VMTemplateVO builtinTmplt : defaultBuiltin) {
              if (!allTemplates.contains(builtinTmplt)) {
                allTemplates.add(builtinTmplt);
              }
            }
          }

          toBeDownloaded.addAll(allTemplates);

          final StateMachine2<
                  VirtualMachineTemplate.State,
                  VirtualMachineTemplate.Event,
                  VirtualMachineTemplate>
              stateMachine = VirtualMachineTemplate.State.getStateMachine();
          for (VMTemplateVO tmplt : allTemplates) {
            String uniqueName = tmplt.getUniqueName();
            TemplateDataStoreVO tmpltStore =
                _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
            if (templateInfos.containsKey(uniqueName)) {
              TemplateProp tmpltInfo = templateInfos.remove(uniqueName);
              toBeDownloaded.remove(tmplt);
              if (tmpltStore != null) {
                s_logger.info("Template Sync found " + uniqueName + " already in the image store");
                if (tmpltStore.getDownloadState() != Status.DOWNLOADED) {
                  tmpltStore.setErrorString("");
                }
                if (tmpltInfo.isCorrupted()) {
                  tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
                  String msg =
                      "Template "
                          + tmplt.getName()
                          + ":"
                          + tmplt.getId()
                          + " is corrupted on secondary storage "
                          + tmpltStore.getId();
                  tmpltStore.setErrorString(msg);
                  s_logger.info(msg);
                  _alertMgr.sendAlert(
                      AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, zoneId, null, msg, msg);
                  if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
                      || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
                    s_logger.info(
                        "Template Sync found "
                            + uniqueName
                            + " on image store "
                            + storeId
                            + " uploaded using SSVM as corrupted, marking it as failed");
                    tmpltStore.setState(State.Failed);
                    try {
                      stateMachine.transitTo(
                          tmplt, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
                    } catch (NoTransitionException e) {
                      s_logger.error(
                          "Unexpected state transition exception for template "
                              + tmplt.getName()
                              + ". Details: "
                              + e.getMessage());
                    }
                  } else if (tmplt.getUrl() == null) {
                    msg =
                        "Private template ("
                            + tmplt
                            + ") with install path "
                            + tmpltInfo.getInstallPath()
                            + " is corrupted, please check in image store: "
                            + tmpltStore.getDataStoreId();
                    s_logger.warn(msg);
                  } else {
                    s_logger.info(
                        "Removing template_store_ref entry for corrupted template "
                            + tmplt.getName());
                    _vmTemplateStoreDao.remove(tmpltStore.getId());
                    toBeDownloaded.add(tmplt);
                  }
                } else {
                  tmpltStore.setDownloadPercent(100);
                  tmpltStore.setDownloadState(Status.DOWNLOADED);
                  tmpltStore.setState(ObjectInDataStoreStateMachine.State.Ready);
                  tmpltStore.setInstallPath(tmpltInfo.getInstallPath());
                  tmpltStore.setSize(tmpltInfo.getSize());
                  tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
                  tmpltStore.setLastUpdated(new Date());
                  // update size in vm_template table
                  VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
                  tmlpt.setSize(tmpltInfo.getSize());
                  _templateDao.update(tmplt.getId(), tmlpt);

                  if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
                      || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
                    try {
                      stateMachine.transitTo(
                          tmplt,
                          VirtualMachineTemplate.Event.OperationSucceeded,
                          null,
                          _templateDao);
                    } catch (NoTransitionException e) {
                      s_logger.error(
                          "Unexpected state transition exception for template "
                              + tmplt.getName()
                              + ". Details: "
                              + e.getMessage());
                    }
                  }

                  // Skipping limit checks for SYSTEM Account and for the templates created from
                  // volumes or snapshots
                  // which already got checked and incremented during createTemplate API call.
                  if (tmpltInfo.getSize() > 0
                      && tmplt.getAccountId() != Account.ACCOUNT_ID_SYSTEM
                      && tmplt.getUrl() != null) {
                    long accountId = tmplt.getAccountId();
                    try {
                      _resourceLimitMgr.checkResourceLimit(
                          _accountMgr.getAccount(accountId),
                          com.cloud.configuration.Resource.ResourceType.secondary_storage,
                          tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
                    } catch (ResourceAllocationException e) {
                      s_logger.warn(e.getMessage());
                      _alertMgr.sendAlert(
                          AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED,
                          zoneId,
                          null,
                          e.getMessage(),
                          e.getMessage());
                    } finally {
                      _resourceLimitMgr.recalculateResourceCount(
                          accountId,
                          _accountMgr.getAccount(accountId).getDomainId(),
                          com.cloud.configuration.Resource.ResourceType.secondary_storage
                              .getOrdinal());
                    }
                  }
                }
                _vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
              } else {
                tmpltStore =
                    new TemplateDataStoreVO(
                        storeId,
                        tmplt.getId(),
                        new Date(),
                        100,
                        Status.DOWNLOADED,
                        null,
                        null,
                        null,
                        tmpltInfo.getInstallPath(),
                        tmplt.getUrl());
                tmpltStore.setSize(tmpltInfo.getSize());
                tmpltStore.setPhysicalSize(tmpltInfo.getPhysicalSize());
                tmpltStore.setDataStoreRole(store.getRole());
                _vmTemplateStoreDao.persist(tmpltStore);

                // update size in vm_template table
                VMTemplateVO tmlpt = _templateDao.findById(tmplt.getId());
                tmlpt.setSize(tmpltInfo.getSize());
                _templateDao.update(tmplt.getId(), tmlpt);
                associateTemplateToZone(tmplt.getId(), zoneId);
              }
            } else if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded
                || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
              s_logger.info(
                  "Template Sync did not find "
                      + uniqueName
                      + " on image store "
                      + storeId
                      + " uploaded using SSVM, marking it as failed");
              toBeDownloaded.remove(tmplt);
              tmpltStore.setDownloadState(Status.DOWNLOAD_ERROR);
              String msg =
                  "Template "
                      + tmplt.getName()
                      + ":"
                      + tmplt.getId()
                      + " is corrupted on secondary storage "
                      + tmpltStore.getId();
              tmpltStore.setErrorString(msg);
              tmpltStore.setState(State.Failed);
              _vmTemplateStoreDao.update(tmpltStore.getId(), tmpltStore);
              try {
                stateMachine.transitTo(
                    tmplt, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
              } catch (NoTransitionException e) {
                s_logger.error(
                    "Unexpected state transition exception for template "
                        + tmplt.getName()
                        + ". Details: "
                        + e.getMessage());
              }
            } else {
              s_logger.info(
                  "Template Sync did not find "
                      + uniqueName
                      + " on image store "
                      + storeId
                      + ", may request download based on available hypervisor types");
              if (tmpltStore != null) {
                if (_storeMgr.isRegionStore(store)
                    && tmpltStore.getDownloadState()
                        == VMTemplateStorageResourceAssoc.Status.DOWNLOADED
                    && tmpltStore.getState() == State.Ready
                    && tmpltStore.getInstallPath() == null) {
                  s_logger.info(
                      "Keep fake entry in template store table for migration of previous NFS to object store");
                } else {
                  s_logger.info(
                      "Removing leftover template "
                          + uniqueName
                          + " entry from template store table");
                  // remove those leftover entries
                  _vmTemplateStoreDao.remove(tmpltStore.getId());
                }
              }
            }
          }

          if (toBeDownloaded.size() > 0) {
            /* Only download templates whose hypervirsor type is in the zone */
            List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(zoneId);
            if (availHypers.isEmpty()) {
              /*
               * This is for cloudzone, local secondary storage resource
               * started before cluster created
               */
              availHypers.add(HypervisorType.KVM);
            }
            /* Baremetal need not to download any template */
            availHypers.remove(HypervisorType.BareMetal);
            availHypers.add(HypervisorType.None); // bug 9809: resume ISO
            // download.
            for (VMTemplateVO tmplt : toBeDownloaded) {
              if (tmplt.getUrl() == null) { // If url is null, skip downloading
                s_logger.info(
                    "Skip downloading template "
                        + tmplt.getUniqueName()
                        + " since no url is specified.");
                continue;
              }
              // if this is private template, skip sync to a new image store
              if (!tmplt.isPublicTemplate()
                  && !tmplt.isFeatured()
                  && tmplt.getTemplateType() != TemplateType.SYSTEM) {
                s_logger.info(
                    "Skip sync downloading private template "
                        + tmplt.getUniqueName()
                        + " to a new image store");
                continue;
              }

              // if this is a region store, and there is already an DOWNLOADED entry there without
              // install_path information, which
              // means that this is a duplicate entry from migration of previous NFS to staging.
              if (_storeMgr.isRegionStore(store)) {
                TemplateDataStoreVO tmpltStore =
                    _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
                if (tmpltStore != null
                    && tmpltStore.getDownloadState()
                        == VMTemplateStorageResourceAssoc.Status.DOWNLOADED
                    && tmpltStore.getState() == State.Ready
                    && tmpltStore.getInstallPath() == null) {
                  s_logger.info("Skip sync template for migration of previous NFS to object store");
                  continue;
                }
              }

              if (availHypers.contains(tmplt.getHypervisorType())) {
                s_logger.info(
                    "Downloading template "
                        + tmplt.getUniqueName()
                        + " to image store "
                        + store.getName());
                associateTemplateToZone(tmplt.getId(), zoneId);
                TemplateInfo tmpl =
                    _templateFactory.getTemplate(tmplt.getId(), DataStoreRole.Image);
                createTemplateAsync(tmpl, store, null);
              } else {
                s_logger.info(
                    "Skip downloading template "
                        + tmplt.getUniqueName()
                        + " since current data center does not have hypervisor "
                        + tmplt.getHypervisorType().toString());
              }
            }
          }

          for (String uniqueName : templateInfos.keySet()) {
            TemplateProp tInfo = templateInfos.get(uniqueName);
            if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
              // we cannot directly call deleteTemplateSync here to reuse delete logic since in this
              // case db does not have this template at all.
              TemplateObjectTO tmplTO = new TemplateObjectTO();
              tmplTO.setDataStore(store.getTO());
              tmplTO.setPath(tInfo.getInstallPath());
              tmplTO.setId(tInfo.getId());
              DeleteCommand dtCommand = new DeleteCommand(tmplTO);
              EndPoint ep = _epSelector.select(store);
              Answer answer = null;
              if (ep == null) {
                String errMsg =
                    "No remote endpoint to send command, check if host or ssvm is down?";
                s_logger.error(errMsg);
                answer = new Answer(dtCommand, false, errMsg);
              } else {
                answer = ep.sendMessage(dtCommand);
              }
              if (answer == null || !answer.getResult()) {
                s_logger.info("Failed to deleted template at store: " + store.getName());

              } else {
                String description =
                    "Deleted template "
                        + tInfo.getTemplateName()
                        + " on secondary storage "
                        + storeId;
                s_logger.info(description);
              }
            }
          }
        } finally {
          syncLock.unlock();
        }
      } else {
        s_logger.info(
            "Couldn't get global lock on "
                + lockString
                + ", another thread may be doing template sync on data store "
                + storeId
                + " now.");
      }
    } finally {
      syncLock.releaseRef();
    }
  }