@Override
@Transactional
public HDocument saveDocument(
String projectSlug,
String iterationSlug,
Resource sourceDoc,
Set<String> extensions,
boolean copyTrans) {
// Only active iterations allow the addition of a document
HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);
// Check permission
identity.checkPermission(hProjectIteration, "import-template");
String docId = sourceDoc.getName();
HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, docId);
HLocale hLocale = this.localeServiceImpl.validateSourceLocale(sourceDoc.getLang());
boolean changed = false;
int nextDocRev;
if (document == null) { // must be a create operation
nextDocRev = 1;
changed = true;
// TODO check that entity name matches id parameter
document = new HDocument(sourceDoc.getName(), sourceDoc.getContentType(), hLocale);
document.setProjectIteration(hProjectIteration);
hProjectIteration.getDocuments().put(docId, document);
document = documentDAO.makePersistent(document);
} else if (document.isObsolete()) { // must also be a create operation
nextDocRev = document.getRevision() + 1;
changed = true;
document.setObsolete(false);
// not sure if this is needed
hProjectIteration.getDocuments().put(docId, document);
} else { // must be an update operation
nextDocRev = document.getRevision() + 1;
}
changed |=
resourceUtils.transferFromResource(sourceDoc, document, extensions, hLocale, nextDocRev);
documentDAO.flush();
long actorId = authenticatedAccount.getPerson().getId();
if (changed) {
documentUploadedEvent.fireAfterSuccess(
new DocumentUploadedEvent(actorId, document.getId(), true, hLocale.getLocaleId()));
clearStatsCacheForUpdatedDocument(document);
}
if (copyTrans && nextDocRev == 1) {
copyTranslations(document);
}
return document;
}
/**
* for a given locale, we can filter it by content state or search in source and target.
*
* @param documentId document id (NOT the String type docId)
* @param hLocale locale
* @param constraints filter constraints
* @param firstResult start index
* @param maxResult max result
* @return a list of HTextFlow that matches the constraint.
* @see
* org.zanata.service.impl.TextFlowSearchServiceImpl#findTextFlows(org.zanata.webtrans.shared.model.WorkspaceId,
* org.zanata.search.FilterConstraints)
*/
public List<HTextFlow> getTextFlowByDocumentIdWithConstraints(
DocumentId documentId,
HLocale hLocale,
FilterConstraints constraints,
int firstResult,
int maxResult) {
FilterConstraintToQuery constraintToQuery =
FilterConstraintToQuery.filterInSingleDocument(constraints, documentId);
String queryString = constraintToQuery.toEntityQuery();
log.debug("\n query {}\n", queryString);
Query textFlowQuery = getSession().createQuery(queryString);
constraintToQuery.setQueryParameters(textFlowQuery, hLocale);
textFlowQuery.setFirstResult(firstResult).setMaxResults(maxResult);
textFlowQuery
.setCacheable(true)
.setComment("TextFlowDAO.getTextFlowByDocumentIdWithConstraint");
@SuppressWarnings("unchecked")
List<HTextFlow> result = textFlowQuery.list();
log.debug(
"{} textFlow for locale {} filter by {}",
result.size(),
hLocale.getLocaleId(),
constraints);
return result;
}
/**
* Indicates if a given text flow should have a match found for a given target locale, or if it is
* already good enough.
*/
private boolean shouldFindMatch(
HTextFlow textFlow, HLocale locale, boolean requireTranslationReview) {
// TODO getTargets will fill up ehcache for large textflows and locales. Check which one is more
// efficient
HTextFlowTarget targetForLocale = textFlow.getTargets().get(locale.getId());
// HTextFlowTarget targetForLocale = textFlowTargetDAO.getTextFlowTarget(
// textFlow, locale);
if (targetForLocale == null || targetForLocale.getState() == ContentState.NeedReview) {
return true;
} else if (requireTranslationReview && targetForLocale.getState() != ContentState.Approved) {
return true;
} else if (!requireTranslationReview && targetForLocale.getState() != ContentState.Translated) {
return true;
} else {
return false;
}
}
public Integer runCopyTrans(
HLocale targetLocale,
HCopyTransOptions options,
HDocument document,
boolean requireTranslationReview,
List<HTextFlow> copyTargets) {
int numCopied = 0;
boolean checkContext = false, checkProject = false, checkDocument = false;
// Only outright reject copies if the options say so
if (options.getDocIdMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
checkDocument = true;
}
if (options.getProjectMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
checkProject = true;
}
if (options.getContextMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) {
checkContext = true;
}
Long actorId = authenticatedAccount.getPerson().getId();
for (HTextFlow textFlow : copyTargets) {
if (shouldFindMatch(textFlow, targetLocale, requireTranslationReview)) {
Optional<HTextFlowTarget> bestMatch =
translationFinder.searchBestMatchTransMemory(
textFlow,
targetLocale.getLocaleId(),
document.getLocale().getLocaleId(),
checkContext,
checkDocument,
checkProject);
if (bestMatch.isPresent()) {
numCopied++;
saveCopyTransMatch(actorId, bestMatch.get(), textFlow, options, requireTranslationReview);
}
}
}
return numCopied;
}
@Override
public String call() throws Exception {
// Needed Components
DocumentDAO documentDAO = (DocumentDAO) Component.getInstance(DocumentDAO.class);
LocaleDAO localeDAO = (LocaleDAO) Component.getInstance(LocaleDAO.class);
ResourceUtils resourceUtils = (ResourceUtils) Component.getInstance(ResourceUtils.class);
TextFlowTargetDAO textFlowTargetDAO =
(TextFlowTargetDAO) Component.getInstance(TextFlowTargetDAO.class);
FileSystemService fileSystemService =
(FileSystemService) Component.getInstance(FileSystemServiceImpl.class);
ConfigurationService configurationService =
(ConfigurationService) Component.getInstance(ConfigurationServiceImpl.class);
final String projectDirectory = projectSlug + "-" + iterationSlug + "/";
final HLocale hLocale = localeDAO.findByLocaleId(new LocaleId(localeId));
final String mappedLocale = hLocale.getLocaleId().getId();
final String localeDirectory = projectDirectory + mappedLocale + "/";
final File downloadFile = fileSystemService.createDownloadStagingFile("zip");
final FileOutputStream output = new FileOutputStream(downloadFile);
final ZipOutputStream zipOutput = new ZipOutputStream(output);
zipOutput.setMethod(ZipOutputStream.DEFLATED);
final PoWriter2 poWriter = new PoWriter2(false, !isPoProject);
final Set<String> extensions = new HashSet<String>();
extensions.add("gettext");
extensions.add("comment");
// Generate the download descriptor file
String downloadId =
fileSystemService.createDownloadDescriptorFile(
downloadFile, projectSlug + "_" + iterationSlug + "_" + localeId + ".zip", userName);
// Add the config file at the root of the project directory
String configFilename = projectDirectory + configurationService.getConfigurationFileName();
zipOutput.putNextEntry(new ZipEntry(configFilename));
zipOutput.write(
configurationService
.getConfigForOfflineTranslation(projectSlug, iterationSlug, hLocale)
.getBytes());
zipOutput.closeEntry();
getHandle().increaseProgress(1);
final List<HDocument> allIterationDocs =
documentDAO.getAllByProjectIteration(projectSlug, iterationSlug);
for (HDocument document : allIterationDocs) {
// Stop the process if signaled to do so
if (getHandle().isCancelled()) {
zipOutput.close();
downloadFile.delete();
fileSystemService.deleteDownloadDescriptorFile(downloadId);
return null;
}
TranslationsResource translationResource = new TranslationsResource();
List<HTextFlowTarget> hTargets = textFlowTargetDAO.findTranslations(document, hLocale);
resourceUtils.transferToTranslationsResource(
translationResource, document, hLocale, extensions, hTargets, Optional.<String>absent());
Resource res = resourceUtils.buildResource(document);
String filename = localeDirectory + document.getDocId() + ".po";
zipOutput.putNextEntry(new ZipEntry(filename));
poWriter.writePo(zipOutput, "UTF-8", res, translationResource);
zipOutput.closeEntry();
getHandle().increaseProgress(1);
}
zipOutput.flush();
zipOutput.close();
return downloadId;
}
private Integer mergeTranslations(
final Long sourceVersionId,
final Long targetVersionId,
final int batchStart,
final int batchLength,
final boolean useNewerTranslation,
final List<HLocale> supportedLocales)
throws Exception {
final Stopwatch stopwatch = Stopwatch.createUnstarted();
stopwatch.start();
List<HTextFlow[]> matches =
textFlowDAO.getSourceByMatchedContext(
sourceVersionId, targetVersionId, batchStart, batchLength);
Multimap<DocumentLocaleKey, TextFlowTargetStateChange> eventMap = HashMultimap.create();
Map<DocumentLocaleKey, Map<ContentState, Long>> docStatsMap = Maps.newHashMap();
Map<DocumentLocaleKey, Long> lastUpdatedTargetId = Maps.newHashMap();
;
for (HTextFlow[] results : matches) {
HTextFlow sourceTf = results[0];
HTextFlow targetTf = results[1];
boolean foundChange = false;
Map<Long, ContentState> localeContentStateMap = Maps.newHashMap();
for (HLocale hLocale : supportedLocales) {
HTextFlowTarget sourceTft = sourceTf.getTargets().get(hLocale.getId());
// only process translated state
if (sourceTft == null || !sourceTft.getState().isTranslated()) {
continue;
}
HTextFlowTarget targetTft = targetTf.getTargets().get(hLocale.getId());
if (targetTft == null) {
targetTft = new HTextFlowTarget(targetTf, hLocale);
targetTft.setVersionNum(0);
targetTf.getTargets().put(hLocale.getId(), targetTft);
}
if (MergeTranslationsServiceImpl.shouldMerge(sourceTft, targetTft, useNewerTranslation)) {
foundChange = true;
ContentState oldState = targetTft.getState();
localeContentStateMap.put(hLocale.getId(), oldState);
mergeTextFlowTarget(sourceTft, targetTft);
}
}
if (foundChange) {
translationStateCacheImpl.clearDocumentStatistics(targetTf.getDocument().getId());
textFlowDAO.makePersistent(targetTf);
textFlowDAO.flush();
for (Map.Entry<Long, ContentState> entry : localeContentStateMap.entrySet()) {
HTextFlowTarget updatedTarget = targetTf.getTargets().get(entry.getKey());
DocumentLocaleKey key =
new DocumentLocaleKey(
targetTf.getDocument().getId(), updatedTarget.getLocale().getLocaleId());
eventMap.put(
key,
new TextFlowTargetStateEvent.TextFlowTargetStateChange(
targetTf.getId(),
updatedTarget.getId(),
updatedTarget.getState(),
entry.getValue()));
lastUpdatedTargetId.put(key, updatedTarget.getId());
Map<ContentState, Long> contentStateDeltas =
docStatsMap.get(key) == null ? Maps.newHashMap() : docStatsMap.get(key);
DocStatsEvent.updateContentStateDeltas(
contentStateDeltas,
updatedTarget.getState(),
entry.getValue(),
targetTf.getWordCount());
docStatsMap.put(key, contentStateDeltas);
}
}
}
Long actorId = authenticatedAccount.getPerson().getId();
for (Map.Entry<DocumentLocaleKey, Collection<TextFlowTargetStateChange>> entry :
eventMap.asMap().entrySet()) {
TextFlowTargetStateEvent tftUpdatedEvent =
new TextFlowTargetStateEvent(
entry.getKey(), targetVersionId, actorId, ImmutableList.copyOf(entry.getValue()));
textFlowTargetStateEvent.fire(tftUpdatedEvent);
}
for (Map.Entry<DocumentLocaleKey, Map<ContentState, Long>> entry : docStatsMap.entrySet()) {
DocStatsEvent docEvent =
new DocStatsEvent(
entry.getKey(),
targetVersionId,
entry.getValue(),
lastUpdatedTargetId.get(entry.getKey()));
docStatsEvent.fire(docEvent);
}
stopwatch.stop();
log.info(
"Complete merge translations of {} in {}",
matches.size() * supportedLocales.size(),
stopwatch);
return matches.size() * supportedLocales.size();
}
// @Test(enabled = true, description = "this should only be executed manually in IDE")
@Ignore
@Test
@PerformanceProfiling
public void pushTranslation() {
EntityMaker entityMaker =
EntityMakerBuilder.builder()
.addFieldOrPropertyMaker(
HProject.class, "sourceViewURL", FixedValueMaker.EMPTY_STRING_MAKER)
.build();
HProjectIteration iteration = entityMaker.makeAndPersist(getEm(), HProjectIteration.class);
HLocale srcLocale = createAndPersistLocale(LocaleId.EN_US, getEm());
HLocale transLocale = createAndPersistLocale(LocaleId.DE, getEm());
String versionSlug = iteration.getSlug();
String projectSlug = iteration.getProject().getSlug();
HDocument document = new HDocument("message", ContentType.PO, srcLocale);
document.setProjectIteration(iteration);
getEm().persist(document);
getEm().flush();
// adjust this number to suit testing purpose
int numOfTextFlows = 50;
int numOfTextFlowsHavingTarget =
createSourceAndSomeTargets(document, transLocale, numOfTextFlows);
getEm().getTransaction().commit();
getEm().getTransaction().begin();
Long targetsCountBefore =
getEm()
.createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
.setParameter("locale", transLocale)
.getSingleResult();
Assertions.assertThat(targetsCountBefore).isEqualTo(numOfTextFlowsHavingTarget);
// ============ add targets =========
TranslationsResource translations = new TranslationsResource();
translations.setRevision(1);
for (int i = 0; i < numOfTextFlows; i++) {
addSampleTranslation(translations, "res" + i);
}
Monitor mon = MonitorFactory.start("");
log.info("==== start translateAllInDoc");
service.translateAllInDoc(
projectSlug,
versionSlug,
document.getDocId(),
transLocale.getLocaleId(),
translations,
extensions,
MergeType.AUTO,
false,
TranslationSourceType.API_UPLOAD);
log.info("==== stop translateAllInDoc: {}", mon.stop());
getEm().getTransaction().commit();
getEm().getTransaction().begin();
Long targetsCount =
getEm()
.createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
.setParameter("locale", transLocale)
.getSingleResult();
Assertions.assertThat(targetsCount).isEqualTo(numOfTextFlows);
List<HTextFlowTargetHistory> histories =
getEm()
.createQuery("from HTextFlowTargetHistory", HTextFlowTargetHistory.class)
.getResultList();
Assertions.assertThat(histories).hasSize(numOfTextFlowsHavingTarget);
}
// TODO Need to refactor this method to get Message statistic by default.
// This is to be consistance with UI which uses message stats, and for
// calculating remaining hours.
@Override
public ContainerTranslationStatistics getStatistics(
String projectSlug,
String iterationSlug,
boolean includeDetails,
boolean includeWordStats,
String[] locales) {
LocaleId[] localeIds;
// if no locales are specified, search in all locales
if (locales.length == 0) {
List<HLocale> iterationLocales =
localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug);
localeIds = new LocaleId[iterationLocales.size()];
for (int i = 0, iterationLocalesSize = iterationLocales.size();
i < iterationLocalesSize;
i++) {
HLocale loc = iterationLocales.get(i);
localeIds[i] = loc.getLocaleId();
}
} else {
localeIds = new LocaleId[locales.length];
for (int i = 0; i < locales.length; i++) {
localeIds[i] = new LocaleId(locales[i]);
}
}
HProjectIteration iteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);
if (iteration == null) {
throw new NoSuchEntityException(projectSlug + "/" + iterationSlug);
}
Map<String, TransUnitCount> transUnitIterationStats =
projectIterationDAO.getAllStatisticsForContainer(iteration.getId());
Map<String, TransUnitWords> wordIterationStats =
projectIterationDAO.getAllWordStatsStatistics(iteration.getId());
ContainerTranslationStatistics iterationStats = new ContainerTranslationStatistics();
iterationStats.setId(iterationSlug);
iterationStats.addRef(
new Link(
URI.create(zPathService.generatePathForProjectIteration(iteration)),
"statSource",
"PROJ_ITER"));
long iterationTotalMssgs =
projectIterationDAO.getTotalMessageCountForIteration(iteration.getId());
long iterationTotalWords = projectIterationDAO.getTotalWordCountForIteration(iteration.getId());
for (LocaleId locId : localeIds) {
// trans unit level stats
TransUnitCount count = transUnitIterationStats.get(locId.getId());
// Stats might not return anything if nothing is translated
if (count == null) {
count = new TransUnitCount(0, 0, (int) iterationTotalMssgs);
}
HTextFlowTarget target =
localeServiceImpl.getLastTranslated(projectSlug, iterationSlug, locId);
String lastModifiedBy = "";
Date lastModifiedDate = null;
if (target != null) {
lastModifiedDate = target.getLastChanged();
if (target.getLastModifiedBy() != null) {
lastModifiedBy = target.getLastModifiedBy().getAccount().getUsername();
}
}
TransUnitWords wordCount = wordIterationStats.get(locId.getId());
if (wordCount == null) {
wordCount = new TransUnitWords(0, 0, (int) iterationTotalWords);
}
TranslationStatistics transUnitStats =
getMessageStats(count, locId, lastModifiedDate, lastModifiedBy);
transUnitStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
iterationStats.addStats(transUnitStats);
// word level stats
if (includeWordStats) {
TranslationStatistics wordsStats =
getWordsStats(wordCount, locId, lastModifiedDate, lastModifiedBy);
wordsStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
iterationStats.addStats(wordsStats);
}
}
// TODO Do in a single query
if (includeDetails) {
for (String docId : iteration.getDocuments().keySet()) {
iterationStats.addDetailedStats(
this.getStatistics(projectSlug, iterationSlug, docId, includeWordStats, locales));
}
}
return iterationStats;
}
@Override
public ContainerTranslationStatistics getStatistics(
String projectSlug,
String iterationSlug,
String docId,
boolean includeWordStats,
String[] locales) {
LocaleId[] localeIds;
// if no locales are specified, search in all locales
if (locales.length == 0) {
List<HLocale> iterationLocales =
localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug);
localeIds = new LocaleId[iterationLocales.size()];
for (int i = 0, iterationLocalesSize = iterationLocales.size();
i < iterationLocalesSize;
i++) {
HLocale loc = iterationLocales.get(i);
localeIds[i] = loc.getLocaleId();
}
} else {
localeIds = new LocaleId[locales.length];
for (int i = 0; i < locales.length; i++) {
localeIds[i] = new LocaleId(locales[i]);
}
}
HDocument document =
documentDAO.getByProjectIterationAndDocId(projectSlug, iterationSlug, docId);
if (document == null) {
throw new NoSuchEntityException(projectSlug + "/" + iterationSlug + "/" + docId);
}
ContainerTranslationStatistics docStatistics = new ContainerTranslationStatistics();
docStatistics.setId(docId);
docStatistics.addRef(
new Link(URI.create(zPathService.generatePathForDocument(document)), "statSource", "DOC"));
for (LocaleId localeId : localeIds) {
ContainerTranslationStatistics docStats = getDocStatistics(document.getId(), localeId);
DocumentStatus docStatus =
translationStateCacheImpl.getDocumentStatus(document.getId(), localeId);
TranslationStatistics docWordStatistic = docStats.getStats(localeId.getId(), StatUnit.WORD);
TranslationStatistics docMsgStatistic = docStats.getStats(localeId.getId(), StatUnit.MESSAGE);
docMsgStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy());
docMsgStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate());
docMsgStatistic.setLastTranslated(
getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy()));
docStatistics.addStats(docMsgStatistic);
if (includeWordStats) {
docWordStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy());
docWordStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate());
docWordStatistic.setLastTranslated(
getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy()));
docStatistics.addStats(docWordStatistic);
}
}
return docStatistics;
}
