@Override @Transactional public HDocument saveDocument( String projectSlug, String iterationSlug, Resource sourceDoc, Set<String> extensions, boolean copyTrans) { // Only active iterations allow the addition of a document HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug); // Check permission identity.checkPermission(hProjectIteration, "import-template"); String docId = sourceDoc.getName(); HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, docId); HLocale hLocale = this.localeServiceImpl.validateSourceLocale(sourceDoc.getLang()); boolean changed = false; int nextDocRev; if (document == null) { // must be a create operation nextDocRev = 1; changed = true; // TODO check that entity name matches id parameter document = new HDocument(sourceDoc.getName(), sourceDoc.getContentType(), hLocale); document.setProjectIteration(hProjectIteration); hProjectIteration.getDocuments().put(docId, document); document = documentDAO.makePersistent(document); } else if (document.isObsolete()) { // must also be a create operation nextDocRev = document.getRevision() + 1; changed = true; document.setObsolete(false); // not sure if this is needed hProjectIteration.getDocuments().put(docId, document); } else { // must be an update operation nextDocRev = document.getRevision() + 1; } changed |= resourceUtils.transferFromResource(sourceDoc, document, extensions, hLocale, nextDocRev); documentDAO.flush(); long actorId = authenticatedAccount.getPerson().getId(); if (changed) { documentUploadedEvent.fireAfterSuccess( new DocumentUploadedEvent(actorId, document.getId(), true, hLocale.getLocaleId())); clearStatsCacheForUpdatedDocument(document); } if (copyTrans && nextDocRev == 1) { copyTranslations(document); } return document; }
/** * for a given locale, we can filter it by content state or search in source and target. * * @param documentId document id (NOT the String type docId) * @param hLocale locale * @param constraints filter constraints * @param firstResult start index * @param maxResult max result * @return a list of HTextFlow that matches the constraint. * @see * org.zanata.service.impl.TextFlowSearchServiceImpl#findTextFlows(org.zanata.webtrans.shared.model.WorkspaceId, * org.zanata.search.FilterConstraints) */ public List<HTextFlow> getTextFlowByDocumentIdWithConstraints( DocumentId documentId, HLocale hLocale, FilterConstraints constraints, int firstResult, int maxResult) { FilterConstraintToQuery constraintToQuery = FilterConstraintToQuery.filterInSingleDocument(constraints, documentId); String queryString = constraintToQuery.toEntityQuery(); log.debug("\n query {}\n", queryString); Query textFlowQuery = getSession().createQuery(queryString); constraintToQuery.setQueryParameters(textFlowQuery, hLocale); textFlowQuery.setFirstResult(firstResult).setMaxResults(maxResult); textFlowQuery .setCacheable(true) .setComment("TextFlowDAO.getTextFlowByDocumentIdWithConstraint"); @SuppressWarnings("unchecked") List<HTextFlow> result = textFlowQuery.list(); log.debug( "{} textFlow for locale {} filter by {}", result.size(), hLocale.getLocaleId(), constraints); return result; }
/** * Indicates if a given text flow should have a match found for a given target locale, or if it is * already good enough. */ private boolean shouldFindMatch( HTextFlow textFlow, HLocale locale, boolean requireTranslationReview) { // TODO getTargets will fill up ehcache for large textflows and locales. Check which one is more // efficient HTextFlowTarget targetForLocale = textFlow.getTargets().get(locale.getId()); // HTextFlowTarget targetForLocale = textFlowTargetDAO.getTextFlowTarget( // textFlow, locale); if (targetForLocale == null || targetForLocale.getState() == ContentState.NeedReview) { return true; } else if (requireTranslationReview && targetForLocale.getState() != ContentState.Approved) { return true; } else if (!requireTranslationReview && targetForLocale.getState() != ContentState.Translated) { return true; } else { return false; } }
public Integer runCopyTrans( HLocale targetLocale, HCopyTransOptions options, HDocument document, boolean requireTranslationReview, List<HTextFlow> copyTargets) { int numCopied = 0; boolean checkContext = false, checkProject = false, checkDocument = false; // Only outright reject copies if the options say so if (options.getDocIdMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkDocument = true; } if (options.getProjectMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkProject = true; } if (options.getContextMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkContext = true; } Long actorId = authenticatedAccount.getPerson().getId(); for (HTextFlow textFlow : copyTargets) { if (shouldFindMatch(textFlow, targetLocale, requireTranslationReview)) { Optional<HTextFlowTarget> bestMatch = translationFinder.searchBestMatchTransMemory( textFlow, targetLocale.getLocaleId(), document.getLocale().getLocaleId(), checkContext, checkDocument, checkProject); if (bestMatch.isPresent()) { numCopied++; saveCopyTransMatch(actorId, bestMatch.get(), textFlow, options, requireTranslationReview); } } } return numCopied; }
@Override public String call() throws Exception { // Needed Components DocumentDAO documentDAO = (DocumentDAO) Component.getInstance(DocumentDAO.class); LocaleDAO localeDAO = (LocaleDAO) Component.getInstance(LocaleDAO.class); ResourceUtils resourceUtils = (ResourceUtils) Component.getInstance(ResourceUtils.class); TextFlowTargetDAO textFlowTargetDAO = (TextFlowTargetDAO) Component.getInstance(TextFlowTargetDAO.class); FileSystemService fileSystemService = (FileSystemService) Component.getInstance(FileSystemServiceImpl.class); ConfigurationService configurationService = (ConfigurationService) Component.getInstance(ConfigurationServiceImpl.class); final String projectDirectory = projectSlug + "-" + iterationSlug + "/"; final HLocale hLocale = localeDAO.findByLocaleId(new LocaleId(localeId)); final String mappedLocale = hLocale.getLocaleId().getId(); final String localeDirectory = projectDirectory + mappedLocale + "/"; final File downloadFile = fileSystemService.createDownloadStagingFile("zip"); final FileOutputStream output = new FileOutputStream(downloadFile); final ZipOutputStream zipOutput = new ZipOutputStream(output); zipOutput.setMethod(ZipOutputStream.DEFLATED); final PoWriter2 poWriter = new PoWriter2(false, !isPoProject); final Set<String> extensions = new HashSet<String>(); extensions.add("gettext"); extensions.add("comment"); // Generate the download descriptor file String downloadId = fileSystemService.createDownloadDescriptorFile( downloadFile, projectSlug + "_" + iterationSlug + "_" + localeId + ".zip", userName); // Add the config file at the root of the project directory String configFilename = projectDirectory + configurationService.getConfigurationFileName(); zipOutput.putNextEntry(new ZipEntry(configFilename)); zipOutput.write( configurationService .getConfigForOfflineTranslation(projectSlug, iterationSlug, hLocale) .getBytes()); zipOutput.closeEntry(); getHandle().increaseProgress(1); final List<HDocument> allIterationDocs = documentDAO.getAllByProjectIteration(projectSlug, iterationSlug); for (HDocument document : allIterationDocs) { // Stop the process if signaled to do so if (getHandle().isCancelled()) { zipOutput.close(); downloadFile.delete(); fileSystemService.deleteDownloadDescriptorFile(downloadId); return null; } TranslationsResource translationResource = new TranslationsResource(); List<HTextFlowTarget> hTargets = textFlowTargetDAO.findTranslations(document, hLocale); resourceUtils.transferToTranslationsResource( translationResource, document, hLocale, extensions, hTargets, Optional.<String>absent()); Resource res = resourceUtils.buildResource(document); String filename = localeDirectory + document.getDocId() + ".po"; zipOutput.putNextEntry(new ZipEntry(filename)); poWriter.writePo(zipOutput, "UTF-8", res, translationResource); zipOutput.closeEntry(); getHandle().increaseProgress(1); } zipOutput.flush(); zipOutput.close(); return downloadId; }
private Integer mergeTranslations( final Long sourceVersionId, final Long targetVersionId, final int batchStart, final int batchLength, final boolean useNewerTranslation, final List<HLocale> supportedLocales) throws Exception { final Stopwatch stopwatch = Stopwatch.createUnstarted(); stopwatch.start(); List<HTextFlow[]> matches = textFlowDAO.getSourceByMatchedContext( sourceVersionId, targetVersionId, batchStart, batchLength); Multimap<DocumentLocaleKey, TextFlowTargetStateChange> eventMap = HashMultimap.create(); Map<DocumentLocaleKey, Map<ContentState, Long>> docStatsMap = Maps.newHashMap(); Map<DocumentLocaleKey, Long> lastUpdatedTargetId = Maps.newHashMap(); ; for (HTextFlow[] results : matches) { HTextFlow sourceTf = results[0]; HTextFlow targetTf = results[1]; boolean foundChange = false; Map<Long, ContentState> localeContentStateMap = Maps.newHashMap(); for (HLocale hLocale : supportedLocales) { HTextFlowTarget sourceTft = sourceTf.getTargets().get(hLocale.getId()); // only process translated state if (sourceTft == null || !sourceTft.getState().isTranslated()) { continue; } HTextFlowTarget targetTft = targetTf.getTargets().get(hLocale.getId()); if (targetTft == null) { targetTft = new HTextFlowTarget(targetTf, hLocale); targetTft.setVersionNum(0); targetTf.getTargets().put(hLocale.getId(), targetTft); } if (MergeTranslationsServiceImpl.shouldMerge(sourceTft, targetTft, useNewerTranslation)) { foundChange = true; ContentState oldState = targetTft.getState(); localeContentStateMap.put(hLocale.getId(), oldState); mergeTextFlowTarget(sourceTft, targetTft); } } if (foundChange) { translationStateCacheImpl.clearDocumentStatistics(targetTf.getDocument().getId()); textFlowDAO.makePersistent(targetTf); textFlowDAO.flush(); for (Map.Entry<Long, ContentState> entry : localeContentStateMap.entrySet()) { HTextFlowTarget updatedTarget = targetTf.getTargets().get(entry.getKey()); DocumentLocaleKey key = new DocumentLocaleKey( targetTf.getDocument().getId(), updatedTarget.getLocale().getLocaleId()); eventMap.put( key, new TextFlowTargetStateEvent.TextFlowTargetStateChange( targetTf.getId(), updatedTarget.getId(), updatedTarget.getState(), entry.getValue())); lastUpdatedTargetId.put(key, updatedTarget.getId()); Map<ContentState, Long> contentStateDeltas = docStatsMap.get(key) == null ? Maps.newHashMap() : docStatsMap.get(key); DocStatsEvent.updateContentStateDeltas( contentStateDeltas, updatedTarget.getState(), entry.getValue(), targetTf.getWordCount()); docStatsMap.put(key, contentStateDeltas); } } } Long actorId = authenticatedAccount.getPerson().getId(); for (Map.Entry<DocumentLocaleKey, Collection<TextFlowTargetStateChange>> entry : eventMap.asMap().entrySet()) { TextFlowTargetStateEvent tftUpdatedEvent = new TextFlowTargetStateEvent( entry.getKey(), targetVersionId, actorId, ImmutableList.copyOf(entry.getValue())); textFlowTargetStateEvent.fire(tftUpdatedEvent); } for (Map.Entry<DocumentLocaleKey, Map<ContentState, Long>> entry : docStatsMap.entrySet()) { DocStatsEvent docEvent = new DocStatsEvent( entry.getKey(), targetVersionId, entry.getValue(), lastUpdatedTargetId.get(entry.getKey())); docStatsEvent.fire(docEvent); } stopwatch.stop(); log.info( "Complete merge translations of {} in {}", matches.size() * supportedLocales.size(), stopwatch); return matches.size() * supportedLocales.size(); }
// @Test(enabled = true, description = "this should only be executed manually in IDE") @Ignore @Test @PerformanceProfiling public void pushTranslation() { EntityMaker entityMaker = EntityMakerBuilder.builder() .addFieldOrPropertyMaker( HProject.class, "sourceViewURL", FixedValueMaker.EMPTY_STRING_MAKER) .build(); HProjectIteration iteration = entityMaker.makeAndPersist(getEm(), HProjectIteration.class); HLocale srcLocale = createAndPersistLocale(LocaleId.EN_US, getEm()); HLocale transLocale = createAndPersistLocale(LocaleId.DE, getEm()); String versionSlug = iteration.getSlug(); String projectSlug = iteration.getProject().getSlug(); HDocument document = new HDocument("message", ContentType.PO, srcLocale); document.setProjectIteration(iteration); getEm().persist(document); getEm().flush(); // adjust this number to suit testing purpose int numOfTextFlows = 50; int numOfTextFlowsHavingTarget = createSourceAndSomeTargets(document, transLocale, numOfTextFlows); getEm().getTransaction().commit(); getEm().getTransaction().begin(); Long targetsCountBefore = getEm() .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class) .setParameter("locale", transLocale) .getSingleResult(); Assertions.assertThat(targetsCountBefore).isEqualTo(numOfTextFlowsHavingTarget); // ============ add targets ========= TranslationsResource translations = new TranslationsResource(); translations.setRevision(1); for (int i = 0; i < numOfTextFlows; i++) { addSampleTranslation(translations, "res" + i); } Monitor mon = MonitorFactory.start(""); log.info("==== start translateAllInDoc"); service.translateAllInDoc( projectSlug, versionSlug, document.getDocId(), transLocale.getLocaleId(), translations, extensions, MergeType.AUTO, false, TranslationSourceType.API_UPLOAD); log.info("==== stop translateAllInDoc: {}", mon.stop()); getEm().getTransaction().commit(); getEm().getTransaction().begin(); Long targetsCount = getEm() .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class) .setParameter("locale", transLocale) .getSingleResult(); Assertions.assertThat(targetsCount).isEqualTo(numOfTextFlows); List<HTextFlowTargetHistory> histories = getEm() .createQuery("from HTextFlowTargetHistory", HTextFlowTargetHistory.class) .getResultList(); Assertions.assertThat(histories).hasSize(numOfTextFlowsHavingTarget); }
// TODO Need to refactor this method to get Message statistic by default. // This is to be consistance with UI which uses message stats, and for // calculating remaining hours. @Override public ContainerTranslationStatistics getStatistics( String projectSlug, String iterationSlug, boolean includeDetails, boolean includeWordStats, String[] locales) { LocaleId[] localeIds; // if no locales are specified, search in all locales if (locales.length == 0) { List<HLocale> iterationLocales = localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug); localeIds = new LocaleId[iterationLocales.size()]; for (int i = 0, iterationLocalesSize = iterationLocales.size(); i < iterationLocalesSize; i++) { HLocale loc = iterationLocales.get(i); localeIds[i] = loc.getLocaleId(); } } else { localeIds = new LocaleId[locales.length]; for (int i = 0; i < locales.length; i++) { localeIds[i] = new LocaleId(locales[i]); } } HProjectIteration iteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug); if (iteration == null) { throw new NoSuchEntityException(projectSlug + "/" + iterationSlug); } Map<String, TransUnitCount> transUnitIterationStats = projectIterationDAO.getAllStatisticsForContainer(iteration.getId()); Map<String, TransUnitWords> wordIterationStats = projectIterationDAO.getAllWordStatsStatistics(iteration.getId()); ContainerTranslationStatistics iterationStats = new ContainerTranslationStatistics(); iterationStats.setId(iterationSlug); iterationStats.addRef( new Link( URI.create(zPathService.generatePathForProjectIteration(iteration)), "statSource", "PROJ_ITER")); long iterationTotalMssgs = projectIterationDAO.getTotalMessageCountForIteration(iteration.getId()); long iterationTotalWords = projectIterationDAO.getTotalWordCountForIteration(iteration.getId()); for (LocaleId locId : localeIds) { // trans unit level stats TransUnitCount count = transUnitIterationStats.get(locId.getId()); // Stats might not return anything if nothing is translated if (count == null) { count = new TransUnitCount(0, 0, (int) iterationTotalMssgs); } HTextFlowTarget target = localeServiceImpl.getLastTranslated(projectSlug, iterationSlug, locId); String lastModifiedBy = ""; Date lastModifiedDate = null; if (target != null) { lastModifiedDate = target.getLastChanged(); if (target.getLastModifiedBy() != null) { lastModifiedBy = target.getLastModifiedBy().getAccount().getUsername(); } } TransUnitWords wordCount = wordIterationStats.get(locId.getId()); if (wordCount == null) { wordCount = new TransUnitWords(0, 0, (int) iterationTotalWords); } TranslationStatistics transUnitStats = getMessageStats(count, locId, lastModifiedDate, lastModifiedBy); transUnitStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount)); iterationStats.addStats(transUnitStats); // word level stats if (includeWordStats) { TranslationStatistics wordsStats = getWordsStats(wordCount, locId, lastModifiedDate, lastModifiedBy); wordsStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount)); iterationStats.addStats(wordsStats); } } // TODO Do in a single query if (includeDetails) { for (String docId : iteration.getDocuments().keySet()) { iterationStats.addDetailedStats( this.getStatistics(projectSlug, iterationSlug, docId, includeWordStats, locales)); } } return iterationStats; }
@Override public ContainerTranslationStatistics getStatistics( String projectSlug, String iterationSlug, String docId, boolean includeWordStats, String[] locales) { LocaleId[] localeIds; // if no locales are specified, search in all locales if (locales.length == 0) { List<HLocale> iterationLocales = localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug); localeIds = new LocaleId[iterationLocales.size()]; for (int i = 0, iterationLocalesSize = iterationLocales.size(); i < iterationLocalesSize; i++) { HLocale loc = iterationLocales.get(i); localeIds[i] = loc.getLocaleId(); } } else { localeIds = new LocaleId[locales.length]; for (int i = 0; i < locales.length; i++) { localeIds[i] = new LocaleId(locales[i]); } } HDocument document = documentDAO.getByProjectIterationAndDocId(projectSlug, iterationSlug, docId); if (document == null) { throw new NoSuchEntityException(projectSlug + "/" + iterationSlug + "/" + docId); } ContainerTranslationStatistics docStatistics = new ContainerTranslationStatistics(); docStatistics.setId(docId); docStatistics.addRef( new Link(URI.create(zPathService.generatePathForDocument(document)), "statSource", "DOC")); for (LocaleId localeId : localeIds) { ContainerTranslationStatistics docStats = getDocStatistics(document.getId(), localeId); DocumentStatus docStatus = translationStateCacheImpl.getDocumentStatus(document.getId(), localeId); TranslationStatistics docWordStatistic = docStats.getStats(localeId.getId(), StatUnit.WORD); TranslationStatistics docMsgStatistic = docStats.getStats(localeId.getId(), StatUnit.MESSAGE); docMsgStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy()); docMsgStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate()); docMsgStatistic.setLastTranslated( getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy())); docStatistics.addStats(docMsgStatistic); if (includeWordStats) { docWordStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy()); docWordStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate()); docWordStatistic.setLastTranslated( getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy())); docStatistics.addStats(docWordStatistic); } } return docStatistics; }