@Override
  @Transactional
  public HDocument saveDocument(
      String projectSlug,
      String iterationSlug,
      Resource sourceDoc,
      Set<String> extensions,
      boolean copyTrans) {
    // Only active iterations allow the addition of a document
    HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);

    // Check permission
    identity.checkPermission(hProjectIteration, "import-template");

    String docId = sourceDoc.getName();

    HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, docId);
    HLocale hLocale = this.localeServiceImpl.validateSourceLocale(sourceDoc.getLang());

    boolean changed = false;
    int nextDocRev;
    if (document == null) { // must be a create operation
      nextDocRev = 1;
      changed = true;
      // TODO check that entity name matches id parameter
      document = new HDocument(sourceDoc.getName(), sourceDoc.getContentType(), hLocale);
      document.setProjectIteration(hProjectIteration);
      hProjectIteration.getDocuments().put(docId, document);
      document = documentDAO.makePersistent(document);
    } else if (document.isObsolete()) { // must also be a create operation
      nextDocRev = document.getRevision() + 1;
      changed = true;
      document.setObsolete(false);
      // not sure if this is needed
      hProjectIteration.getDocuments().put(docId, document);
    } else { // must be an update operation
      nextDocRev = document.getRevision() + 1;
    }

    changed |=
        resourceUtils.transferFromResource(sourceDoc, document, extensions, hLocale, nextDocRev);
    documentDAO.flush();

    long actorId = authenticatedAccount.getPerson().getId();
    if (changed) {
      documentUploadedEvent.fireAfterSuccess(
          new DocumentUploadedEvent(actorId, document.getId(), true, hLocale.getLocaleId()));
      clearStatsCacheForUpdatedDocument(document);
    }

    if (copyTrans && nextDocRev == 1) {
      copyTranslations(document);
    }

    return document;
  }
  /**
   * for a given locale, we can filter it by content state or search in source and target.
   *
   * @param documentId document id (NOT the String type docId)
   * @param hLocale locale
   * @param constraints filter constraints
   * @param firstResult start index
   * @param maxResult max result
   * @return a list of HTextFlow that matches the constraint.
   * @see
   *     org.zanata.service.impl.TextFlowSearchServiceImpl#findTextFlows(org.zanata.webtrans.shared.model.WorkspaceId,
   *     org.zanata.search.FilterConstraints)
   */
  public List<HTextFlow> getTextFlowByDocumentIdWithConstraints(
      DocumentId documentId,
      HLocale hLocale,
      FilterConstraints constraints,
      int firstResult,
      int maxResult) {
    FilterConstraintToQuery constraintToQuery =
        FilterConstraintToQuery.filterInSingleDocument(constraints, documentId);
    String queryString = constraintToQuery.toEntityQuery();
    log.debug("\n query {}\n", queryString);

    Query textFlowQuery = getSession().createQuery(queryString);
    constraintToQuery.setQueryParameters(textFlowQuery, hLocale);
    textFlowQuery.setFirstResult(firstResult).setMaxResults(maxResult);
    textFlowQuery
        .setCacheable(true)
        .setComment("TextFlowDAO.getTextFlowByDocumentIdWithConstraint");

    @SuppressWarnings("unchecked")
    List<HTextFlow> result = textFlowQuery.list();
    log.debug(
        "{} textFlow for locale {} filter by {}",
        result.size(),
        hLocale.getLocaleId(),
        constraints);
    return result;
  }
  /**
   * Indicates if a given text flow should have a match found for a given target locale, or if it is
   * already good enough.
   */
  private boolean shouldFindMatch(
      HTextFlow textFlow, HLocale locale, boolean requireTranslationReview) {
    // TODO getTargets will fill up ehcache for large textflows and locales. Check which one is more
    // efficient
    HTextFlowTarget targetForLocale = textFlow.getTargets().get(locale.getId());
    //        HTextFlowTarget targetForLocale = textFlowTargetDAO.getTextFlowTarget(
    //                textFlow, locale);

    if (targetForLocale == null || targetForLocale.getState() == ContentState.NeedReview) {
      return true;
    } else if (requireTranslationReview && targetForLocale.getState() != ContentState.Approved) {
      return true;
    } else if (!requireTranslationReview && targetForLocale.getState() != ContentState.Translated) {
      return true;
    } else {
      return false;
    }
  }
  public Integer runCopyTrans(
      HLocale targetLocale,
      HCopyTransOptions options,
      HDocument document,
      boolean requireTranslationReview,
      List<HTextFlow> copyTargets) {
    int numCopied = 0;
    boolean checkContext = false, checkProject = false, checkDocument = false;

    // Only outright reject copies if the options say so
    if (options.getDocIdMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
      checkDocument = true;
    }
    if (options.getProjectMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
      checkProject = true;
    }
    if (options.getContextMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
      checkContext = true;
    }

    Long actorId = authenticatedAccount.getPerson().getId();
    for (HTextFlow textFlow : copyTargets) {
      if (shouldFindMatch(textFlow, targetLocale, requireTranslationReview)) {

        Optional<HTextFlowTarget> bestMatch =
            translationFinder.searchBestMatchTransMemory(
                textFlow,
                targetLocale.getLocaleId(),
                document.getLocale().getLocaleId(),
                checkContext,
                checkDocument,
                checkProject);
        if (bestMatch.isPresent()) {
          numCopied++;

          saveCopyTransMatch(actorId, bestMatch.get(), textFlow, options, requireTranslationReview);
        }
      }
    }
    return numCopied;
  }
  @Override
  public String call() throws Exception {
    // Needed Components
    DocumentDAO documentDAO = (DocumentDAO) Component.getInstance(DocumentDAO.class);
    LocaleDAO localeDAO = (LocaleDAO) Component.getInstance(LocaleDAO.class);
    ResourceUtils resourceUtils = (ResourceUtils) Component.getInstance(ResourceUtils.class);
    TextFlowTargetDAO textFlowTargetDAO =
        (TextFlowTargetDAO) Component.getInstance(TextFlowTargetDAO.class);
    FileSystemService fileSystemService =
        (FileSystemService) Component.getInstance(FileSystemServiceImpl.class);
    ConfigurationService configurationService =
        (ConfigurationService) Component.getInstance(ConfigurationServiceImpl.class);

    final String projectDirectory = projectSlug + "-" + iterationSlug + "/";
    final HLocale hLocale = localeDAO.findByLocaleId(new LocaleId(localeId));
    final String mappedLocale = hLocale.getLocaleId().getId();
    final String localeDirectory = projectDirectory + mappedLocale + "/";

    final File downloadFile = fileSystemService.createDownloadStagingFile("zip");
    final FileOutputStream output = new FileOutputStream(downloadFile);
    final ZipOutputStream zipOutput = new ZipOutputStream(output);
    zipOutput.setMethod(ZipOutputStream.DEFLATED);
    final PoWriter2 poWriter = new PoWriter2(false, !isPoProject);
    final Set<String> extensions = new HashSet<String>();

    extensions.add("gettext");
    extensions.add("comment");

    // Generate the download descriptor file
    String downloadId =
        fileSystemService.createDownloadDescriptorFile(
            downloadFile, projectSlug + "_" + iterationSlug + "_" + localeId + ".zip", userName);

    // Add the config file at the root of the project directory
    String configFilename = projectDirectory + configurationService.getConfigurationFileName();
    zipOutput.putNextEntry(new ZipEntry(configFilename));
    zipOutput.write(
        configurationService
            .getConfigForOfflineTranslation(projectSlug, iterationSlug, hLocale)
            .getBytes());
    zipOutput.closeEntry();
    getHandle().increaseProgress(1);

    final List<HDocument> allIterationDocs =
        documentDAO.getAllByProjectIteration(projectSlug, iterationSlug);
    for (HDocument document : allIterationDocs) {
      // Stop the process if signaled to do so
      if (getHandle().isCancelled()) {
        zipOutput.close();
        downloadFile.delete();
        fileSystemService.deleteDownloadDescriptorFile(downloadId);
        return null;
      }

      TranslationsResource translationResource = new TranslationsResource();
      List<HTextFlowTarget> hTargets = textFlowTargetDAO.findTranslations(document, hLocale);
      resourceUtils.transferToTranslationsResource(
          translationResource, document, hLocale, extensions, hTargets, Optional.<String>absent());

      Resource res = resourceUtils.buildResource(document);

      String filename = localeDirectory + document.getDocId() + ".po";
      zipOutput.putNextEntry(new ZipEntry(filename));
      poWriter.writePo(zipOutput, "UTF-8", res, translationResource);
      zipOutput.closeEntry();

      getHandle().increaseProgress(1);
    }

    zipOutput.flush();
    zipOutput.close();

    return downloadId;
  }
  private Integer mergeTranslations(
      final Long sourceVersionId,
      final Long targetVersionId,
      final int batchStart,
      final int batchLength,
      final boolean useNewerTranslation,
      final List<HLocale> supportedLocales)
      throws Exception {

    final Stopwatch stopwatch = Stopwatch.createUnstarted();
    stopwatch.start();

    List<HTextFlow[]> matches =
        textFlowDAO.getSourceByMatchedContext(
            sourceVersionId, targetVersionId, batchStart, batchLength);

    Multimap<DocumentLocaleKey, TextFlowTargetStateChange> eventMap = HashMultimap.create();

    Map<DocumentLocaleKey, Map<ContentState, Long>> docStatsMap = Maps.newHashMap();

    Map<DocumentLocaleKey, Long> lastUpdatedTargetId = Maps.newHashMap();
    ;

    for (HTextFlow[] results : matches) {
      HTextFlow sourceTf = results[0];
      HTextFlow targetTf = results[1];
      boolean foundChange = false;
      Map<Long, ContentState> localeContentStateMap = Maps.newHashMap();

      for (HLocale hLocale : supportedLocales) {
        HTextFlowTarget sourceTft = sourceTf.getTargets().get(hLocale.getId());
        // only process translated state
        if (sourceTft == null || !sourceTft.getState().isTranslated()) {
          continue;
        }

        HTextFlowTarget targetTft = targetTf.getTargets().get(hLocale.getId());
        if (targetTft == null) {
          targetTft = new HTextFlowTarget(targetTf, hLocale);
          targetTft.setVersionNum(0);
          targetTf.getTargets().put(hLocale.getId(), targetTft);
        }

        if (MergeTranslationsServiceImpl.shouldMerge(sourceTft, targetTft, useNewerTranslation)) {
          foundChange = true;

          ContentState oldState = targetTft.getState();
          localeContentStateMap.put(hLocale.getId(), oldState);
          mergeTextFlowTarget(sourceTft, targetTft);
        }
      }
      if (foundChange) {
        translationStateCacheImpl.clearDocumentStatistics(targetTf.getDocument().getId());
        textFlowDAO.makePersistent(targetTf);
        textFlowDAO.flush();

        for (Map.Entry<Long, ContentState> entry : localeContentStateMap.entrySet()) {
          HTextFlowTarget updatedTarget = targetTf.getTargets().get(entry.getKey());

          DocumentLocaleKey key =
              new DocumentLocaleKey(
                  targetTf.getDocument().getId(), updatedTarget.getLocale().getLocaleId());

          eventMap.put(
              key,
              new TextFlowTargetStateEvent.TextFlowTargetStateChange(
                  targetTf.getId(),
                  updatedTarget.getId(),
                  updatedTarget.getState(),
                  entry.getValue()));

          lastUpdatedTargetId.put(key, updatedTarget.getId());

          Map<ContentState, Long> contentStateDeltas =
              docStatsMap.get(key) == null ? Maps.newHashMap() : docStatsMap.get(key);

          DocStatsEvent.updateContentStateDeltas(
              contentStateDeltas,
              updatedTarget.getState(),
              entry.getValue(),
              targetTf.getWordCount());

          docStatsMap.put(key, contentStateDeltas);
        }
      }
    }
    Long actorId = authenticatedAccount.getPerson().getId();
    for (Map.Entry<DocumentLocaleKey, Collection<TextFlowTargetStateChange>> entry :
        eventMap.asMap().entrySet()) {
      TextFlowTargetStateEvent tftUpdatedEvent =
          new TextFlowTargetStateEvent(
              entry.getKey(), targetVersionId, actorId, ImmutableList.copyOf(entry.getValue()));
      textFlowTargetStateEvent.fire(tftUpdatedEvent);
    }
    for (Map.Entry<DocumentLocaleKey, Map<ContentState, Long>> entry : docStatsMap.entrySet()) {
      DocStatsEvent docEvent =
          new DocStatsEvent(
              entry.getKey(),
              targetVersionId,
              entry.getValue(),
              lastUpdatedTargetId.get(entry.getKey()));
      docStatsEvent.fire(docEvent);
    }
    stopwatch.stop();
    log.info(
        "Complete merge translations of {} in {}",
        matches.size() * supportedLocales.size(),
        stopwatch);
    return matches.size() * supportedLocales.size();
  }
  //    @Test(enabled = true, description = "this should only be executed manually in IDE")
  @Ignore
  @Test
  @PerformanceProfiling
  public void pushTranslation() {
    EntityMaker entityMaker =
        EntityMakerBuilder.builder()
            .addFieldOrPropertyMaker(
                HProject.class, "sourceViewURL", FixedValueMaker.EMPTY_STRING_MAKER)
            .build();
    HProjectIteration iteration = entityMaker.makeAndPersist(getEm(), HProjectIteration.class);
    HLocale srcLocale = createAndPersistLocale(LocaleId.EN_US, getEm());
    HLocale transLocale = createAndPersistLocale(LocaleId.DE, getEm());

    String versionSlug = iteration.getSlug();
    String projectSlug = iteration.getProject().getSlug();

    HDocument document = new HDocument("message", ContentType.PO, srcLocale);
    document.setProjectIteration(iteration);
    getEm().persist(document);
    getEm().flush();

    // adjust this number to suit testing purpose
    int numOfTextFlows = 50;
    int numOfTextFlowsHavingTarget =
        createSourceAndSomeTargets(document, transLocale, numOfTextFlows);
    getEm().getTransaction().commit();
    getEm().getTransaction().begin();

    Long targetsCountBefore =
        getEm()
            .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
            .setParameter("locale", transLocale)
            .getSingleResult();
    Assertions.assertThat(targetsCountBefore).isEqualTo(numOfTextFlowsHavingTarget);

    // ============ add targets =========
    TranslationsResource translations = new TranslationsResource();
    translations.setRevision(1);
    for (int i = 0; i < numOfTextFlows; i++) {
      addSampleTranslation(translations, "res" + i);
    }
    Monitor mon = MonitorFactory.start("");
    log.info("==== start translateAllInDoc");
    service.translateAllInDoc(
        projectSlug,
        versionSlug,
        document.getDocId(),
        transLocale.getLocaleId(),
        translations,
        extensions,
        MergeType.AUTO,
        false,
        TranslationSourceType.API_UPLOAD);
    log.info("==== stop translateAllInDoc: {}", mon.stop());
    getEm().getTransaction().commit();
    getEm().getTransaction().begin();

    Long targetsCount =
        getEm()
            .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
            .setParameter("locale", transLocale)
            .getSingleResult();
    Assertions.assertThat(targetsCount).isEqualTo(numOfTextFlows);

    List<HTextFlowTargetHistory> histories =
        getEm()
            .createQuery("from HTextFlowTargetHistory", HTextFlowTargetHistory.class)
            .getResultList();
    Assertions.assertThat(histories).hasSize(numOfTextFlowsHavingTarget);
  }
  // TODO Need to refactor this method to get Message statistic by default.
  // This is to be consistance with UI which uses message stats, and for
  // calculating remaining hours.
  @Override
  public ContainerTranslationStatistics getStatistics(
      String projectSlug,
      String iterationSlug,
      boolean includeDetails,
      boolean includeWordStats,
      String[] locales) {
    LocaleId[] localeIds;

    // if no locales are specified, search in all locales
    if (locales.length == 0) {
      List<HLocale> iterationLocales =
          localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug);
      localeIds = new LocaleId[iterationLocales.size()];
      for (int i = 0, iterationLocalesSize = iterationLocales.size();
          i < iterationLocalesSize;
          i++) {
        HLocale loc = iterationLocales.get(i);
        localeIds[i] = loc.getLocaleId();
      }
    } else {
      localeIds = new LocaleId[locales.length];
      for (int i = 0; i < locales.length; i++) {
        localeIds[i] = new LocaleId(locales[i]);
      }
    }

    HProjectIteration iteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);

    if (iteration == null) {
      throw new NoSuchEntityException(projectSlug + "/" + iterationSlug);
    }

    Map<String, TransUnitCount> transUnitIterationStats =
        projectIterationDAO.getAllStatisticsForContainer(iteration.getId());
    Map<String, TransUnitWords> wordIterationStats =
        projectIterationDAO.getAllWordStatsStatistics(iteration.getId());

    ContainerTranslationStatistics iterationStats = new ContainerTranslationStatistics();
    iterationStats.setId(iterationSlug);
    iterationStats.addRef(
        new Link(
            URI.create(zPathService.generatePathForProjectIteration(iteration)),
            "statSource",
            "PROJ_ITER"));
    long iterationTotalMssgs =
        projectIterationDAO.getTotalMessageCountForIteration(iteration.getId());
    long iterationTotalWords = projectIterationDAO.getTotalWordCountForIteration(iteration.getId());

    for (LocaleId locId : localeIds) {
      // trans unit level stats
      TransUnitCount count = transUnitIterationStats.get(locId.getId());
      // Stats might not return anything if nothing is translated
      if (count == null) {
        count = new TransUnitCount(0, 0, (int) iterationTotalMssgs);
      }

      HTextFlowTarget target =
          localeServiceImpl.getLastTranslated(projectSlug, iterationSlug, locId);

      String lastModifiedBy = "";
      Date lastModifiedDate = null;

      if (target != null) {
        lastModifiedDate = target.getLastChanged();
        if (target.getLastModifiedBy() != null) {
          lastModifiedBy = target.getLastModifiedBy().getAccount().getUsername();
        }
      }

      TransUnitWords wordCount = wordIterationStats.get(locId.getId());
      if (wordCount == null) {
        wordCount = new TransUnitWords(0, 0, (int) iterationTotalWords);
      }

      TranslationStatistics transUnitStats =
          getMessageStats(count, locId, lastModifiedDate, lastModifiedBy);
      transUnitStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
      iterationStats.addStats(transUnitStats);

      // word level stats
      if (includeWordStats) {
        TranslationStatistics wordsStats =
            getWordsStats(wordCount, locId, lastModifiedDate, lastModifiedBy);
        wordsStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
        iterationStats.addStats(wordsStats);
      }
    }

    // TODO Do in a single query
    if (includeDetails) {
      for (String docId : iteration.getDocuments().keySet()) {
        iterationStats.addDetailedStats(
            this.getStatistics(projectSlug, iterationSlug, docId, includeWordStats, locales));
      }
    }

    return iterationStats;
  }
  @Override
  public ContainerTranslationStatistics getStatistics(
      String projectSlug,
      String iterationSlug,
      String docId,
      boolean includeWordStats,
      String[] locales) {
    LocaleId[] localeIds;

    // if no locales are specified, search in all locales
    if (locales.length == 0) {
      List<HLocale> iterationLocales =
          localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug);
      localeIds = new LocaleId[iterationLocales.size()];
      for (int i = 0, iterationLocalesSize = iterationLocales.size();
          i < iterationLocalesSize;
          i++) {
        HLocale loc = iterationLocales.get(i);
        localeIds[i] = loc.getLocaleId();
      }
    } else {
      localeIds = new LocaleId[locales.length];
      for (int i = 0; i < locales.length; i++) {
        localeIds[i] = new LocaleId(locales[i]);
      }
    }

    HDocument document =
        documentDAO.getByProjectIterationAndDocId(projectSlug, iterationSlug, docId);

    if (document == null) {
      throw new NoSuchEntityException(projectSlug + "/" + iterationSlug + "/" + docId);
    }

    ContainerTranslationStatistics docStatistics = new ContainerTranslationStatistics();
    docStatistics.setId(docId);
    docStatistics.addRef(
        new Link(URI.create(zPathService.generatePathForDocument(document)), "statSource", "DOC"));

    for (LocaleId localeId : localeIds) {
      ContainerTranslationStatistics docStats = getDocStatistics(document.getId(), localeId);

      DocumentStatus docStatus =
          translationStateCacheImpl.getDocumentStatus(document.getId(), localeId);

      TranslationStatistics docWordStatistic = docStats.getStats(localeId.getId(), StatUnit.WORD);
      TranslationStatistics docMsgStatistic = docStats.getStats(localeId.getId(), StatUnit.MESSAGE);

      docMsgStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy());
      docMsgStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate());
      docMsgStatistic.setLastTranslated(
          getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy()));
      docStatistics.addStats(docMsgStatistic);

      if (includeWordStats) {
        docWordStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy());
        docWordStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate());
        docWordStatistic.setLastTranslated(
            getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy()));
        docStatistics.addStats(docWordStatistic);
      }
    }
    return docStatistics;
  }