@Override
protected int getMaxProgress() {
LocaleService localeService = (LocaleService) Component.getInstance(LocaleServiceImpl.class);
List<HLocale> localeList =
localeService.getSupportedLanguageByProjectIteration(
projectIteration.getProject().getSlug(), projectIteration.getSlug());
return projectIteration.getDocuments().size() * localeList.size();
}
@Override
@Transactional
public HDocument saveDocument(
String projectSlug,
String iterationSlug,
Resource sourceDoc,
Set<String> extensions,
boolean copyTrans) {
// Only active iterations allow the addition of a document
HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);
// Check permission
identity.checkPermission(hProjectIteration, "import-template");
String docId = sourceDoc.getName();
HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, docId);
HLocale hLocale = this.localeServiceImpl.validateSourceLocale(sourceDoc.getLang());
boolean changed = false;
int nextDocRev;
if (document == null) { // must be a create operation
nextDocRev = 1;
changed = true;
// TODO check that entity name matches id parameter
document = new HDocument(sourceDoc.getName(), sourceDoc.getContentType(), hLocale);
document.setProjectIteration(hProjectIteration);
hProjectIteration.getDocuments().put(docId, document);
document = documentDAO.makePersistent(document);
} else if (document.isObsolete()) { // must also be a create operation
nextDocRev = document.getRevision() + 1;
changed = true;
document.setObsolete(false);
// not sure if this is needed
hProjectIteration.getDocuments().put(docId, document);
} else { // must be an update operation
nextDocRev = document.getRevision() + 1;
}
changed |=
resourceUtils.transferFromResource(sourceDoc, document, extensions, hLocale, nextDocRev);
documentDAO.flush();
long actorId = authenticatedAccount.getPerson().getId();
if (changed) {
documentUploadedEvent.fireAfterSuccess(
new DocumentUploadedEvent(actorId, document.getId(), true, hLocale.getLocaleId()));
clearStatsCacheForUpdatedDocument(document);
}
if (copyTrans && nextDocRev == 1) {
copyTranslations(document);
}
return document;
}
private void prepareTransUnitUpdatedEvent(
int previousVersionNum, ContentState previousState, HTextFlowTarget target) {
LocaleId localeId = target.getLocaleId();
HTextFlow textFlow = target.getTextFlow();
HDocument document = textFlow.getDocument();
HProjectIteration projectIteration = document.getProjectIteration();
String iterationSlug = projectIteration.getSlug();
String projectSlug = projectIteration.getProject().getSlug();
ProjectType projectType = projectIteration.getProjectType();
WorkspaceId workspaceId =
new WorkspaceId(new ProjectIterationId(projectSlug, iterationSlug, projectType), localeId);
Optional<TranslationWorkspace> workspaceOptional =
translationWorkspaceManager.tryGetWorkspace(workspaceId);
if (!workspaceOptional.isPresent()) {
return;
}
TransUnitTransformer transUnitTransformer =
serviceLocator.getInstance(TransUnitTransformer.class);
TransUnit transUnit = transUnitTransformer.transform(textFlow, target, target.getLocale());
DocumentId documentId = new DocumentId(document.getId(), document.getDocId());
int wordCount = textFlow.getWordCount().intValue();
TransUnitUpdateInfo updateInfo =
createTransUnitUpdateInfo(
transUnit, documentId, wordCount, previousVersionNum, previousState);
CacheValue context =
updateContext.getIfPresent(new CacheKey(transUnit.getId(), transUnit.getLocaleId()));
TransUnitUpdated updated;
if (context != null) {
updated = new TransUnitUpdated(updateInfo, context.editorClientId, context.updateType);
log.debug("about to publish trans unit updated event {}", updated);
} else if (ServletContexts.instance().getRequest() != null) {
String sessionId = ServletContexts.instance().getRequest().getSession().getId();
EditorClientId editorClientId = new EditorClientId(sessionId, -1);
updated =
new TransUnitUpdated(
updateInfo, editorClientId, TransUnitUpdated.UpdateType.NonEditorSave);
} else {
updated =
new TransUnitUpdated(
updateInfo,
new EditorClientId("unknown", -1),
TransUnitUpdated.UpdateType.NonEditorSave);
}
if (Events.exists()) {
Events.instance()
.raiseTransactionSuccessEvent(
TextFlowTargetUpdatedEvent.EVENT_NAME,
new TextFlowTargetUpdatedEvent(workspaceOptional.get(), target.getId(), updated));
}
}
@Override
public int getTotalProgressCount(
HProjectIteration sourceVersion, HProjectIteration targetVersion) {
int matchCount = getTotalMatchCount(sourceVersion.getId(), targetVersion.getId());
List<HLocale> locales =
getSupportedLocales(targetVersion.getProject().getSlug(), targetVersion.getSlug());
return matchCount * locales.size();
}
@Override
public GetDocumentListResult execute(GetDocumentList action, ExecutionContext context)
throws ActionException {
identity.checkLoggedIn();
LocaleId localeId = action.getWorkspaceId().getLocaleId();
ProjectIterationId iterationId = action.getProjectIterationId();
ArrayList<DocumentInfo> docs = new ArrayList<DocumentInfo>();
HProjectIteration hProjectIteration =
projectIterationDAO.getBySlug(iterationId.getProjectSlug(), iterationId.getIterationSlug());
Collection<HDocument> hDocs = hProjectIteration.getDocuments().values();
for (HDocument hDoc : hDocs) {
if (action.getFilters() == null
|| action.getFilters().isEmpty()
|| action.getFilters().contains(hDoc.getPath() + hDoc.getName())) {
DocumentId docId = new DocumentId(hDoc.getId(), hDoc.getDocId());
TranslationStats stats = documentDAO.getStatistics(hDoc.getId(), localeId);
String lastModifiedBy = "";
HPerson person = hDoc.getLastModifiedBy();
if (person != null) {
lastModifiedBy = person.getAccount().getUsername();
}
Map<String, String> downloadExtensions = new HashMap<String, String>();
downloadExtensions.put(".po", "po?docId=" + hDoc.getDocId());
if (translationFileServiceImpl.hasPersistedDocument(
iterationId.getProjectSlug(),
iterationId.getIterationSlug(),
hDoc.getPath(),
hDoc.getName())) {
String extension =
"."
+ translationFileServiceImpl.getFileExtension(
iterationId.getProjectSlug(),
iterationId.getIterationSlug(),
hDoc.getPath(),
hDoc.getName());
downloadExtensions.put(extension, "baked?docId=" + hDoc.getDocId());
}
DocumentInfo doc =
new DocumentInfo(
docId,
hDoc.getName(),
hDoc.getPath(),
hDoc.getLocale().getLocaleId(),
stats,
lastModifiedBy,
hDoc.getLastChanged(),
downloadExtensions);
docs.add(doc);
}
}
return new GetDocumentListResult(iterationId, docs);
}
@Test
public void traverseProjectGraph() throws Exception {
EntityManager em = getEm();
HIterationProject project = em.find(HIterationProject.class, 1l);
assertThat(project, notNullValue());
List<HProjectIteration> projectTargets = project.getProjectIterations();
assertThat("Project should have 2 targets", projectTargets.size(), is(2));
HProjectIteration target = projectTargets.get(0);
assertThat("Expect target with id 1", target.getId(), is(1l));
}
private void processWebHookDocumentMilestoneEvent(
DocumentStatisticUpdatedEvent event,
Collection<ContentState> contentStates,
String message,
int percentMilestone) {
HProjectIteration version = projectIterationDAO.findById(event.getProjectIterationId());
HProject project = version.getProject();
if (!project.getWebHooks().isEmpty()) {
WordStatistic stats =
translationStateCacheImpl.getDocumentStatistics(
event.getDocumentId(), event.getLocaleId());
WordStatistic oldStats = StatisticsUtil.copyWordStatistic(stats);
if (oldStats != null) {
oldStats.decrement(event.getNewState(), event.getWordCount());
oldStats.increment(event.getPreviousState(), event.getWordCount());
boolean shouldPublish =
hasContentStateReachedMilestone(oldStats, stats, contentStates, percentMilestone);
if (shouldPublish) {
HDocument document = documentDAO.getById(event.getDocumentId());
String editorUrl =
urlUtil.fullEditorDocumentUrl(
project.getSlug(),
version.getSlug(),
event.getLocaleId(),
LocaleId.EN_US,
document.getDocId());
DocumentMilestoneEvent milestoneEvent =
new DocumentMilestoneEvent(
project.getSlug(),
version.getSlug(),
document.getDocId(),
event.getLocaleId(),
message,
editorUrl);
for (WebHook webHook : project.getWebHooks()) {
publishDocumentMilestoneEvent(webHook, milestoneEvent);
}
}
}
}
}
@Override
public GetDocumentListResult execute(GetDocumentList action, ExecutionContext context)
throws ActionException {
ZanataIdentity.instance().checkLoggedIn();
ProjectIterationId iterationId = action.getProjectIterationId();
ArrayList<DocumentInfo> docs = new ArrayList<DocumentInfo>();
HProjectIteration hProjectIteration =
projectIterationDAO.getBySlug(iterationId.getProjectSlug(), iterationId.getIterationSlug());
Collection<HDocument> hDocs = hProjectIteration.getDocuments().values();
for (HDocument hDoc : hDocs) {
DocumentId docId = new DocumentId(hDoc.getId());
DocumentInfo doc = new DocumentInfo(docId, hDoc.getName(), hDoc.getPath());
docs.add(doc);
}
return new GetDocumentListResult(iterationId, docs);
}
protected int mergeTranslationBatch(
HProjectIteration sourceVersion,
HProjectIteration targetVersion,
List<HLocale> supportedLocales,
boolean useNewerTranslation,
int offset,
int batchSize) {
try {
return runInTransaction(
() ->
this.mergeTranslations(
sourceVersion.getId(),
targetVersion.getId(),
offset,
batchSize,
useNewerTranslation,
supportedLocales));
} catch (Exception e) {
log.warn("exception during copy text flow target", e);
return 0;
}
}
private HProjectIteration retrieveAndCheckIteration(
String projectSlug, String iterationSlug, boolean writeOperation) {
HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);
HProject hProject = hProjectIteration == null ? null : hProjectIteration.getProject();
if (hProjectIteration == null) {
throw new NoSuchEntityException(
"Project Iteration '" + projectSlug + ":" + iterationSlug + "' not found.");
} else if (hProjectIteration.getStatus().equals(EntityStatus.OBSOLETE)
|| hProject.getStatus().equals(EntityStatus.OBSOLETE)) {
throw new NoSuchEntityException(
"Project Iteration '" + projectSlug + ":" + iterationSlug + "' not found.");
} else if (writeOperation) {
if (hProjectIteration.getStatus().equals(EntityStatus.READONLY)
|| hProject.getStatus().equals(EntityStatus.READONLY)) {
throw new ReadOnlyEntityException(
"Project Iteration '" + projectSlug + ":" + iterationSlug + "' is read-only.");
} else {
return hProjectIteration;
}
} else {
return hProjectIteration;
}
}
/**
* Check if sourceVersion or targetVersion has source document.
*
* @param sourceVersion
* @param targetVersion
*/
private boolean isVersionsEmpty(
HProjectIteration sourceVersion, HProjectIteration targetVersion) {
if (sourceVersion.getDocuments().isEmpty()) {
log.error(
"No documents in source version {}:{}",
sourceVersion.getProject().getSlug(),
sourceVersion.getSlug());
return true;
}
if (targetVersion.getDocuments().isEmpty()) {
log.error(
"No documents in target version {}:{}",
targetVersion.getProject().getSlug(),
targetVersion.getSlug());
return true;
}
return false;
}
@Override
@Async
public Future<Void> startMergeTranslations(
String sourceProjectSlug,
String sourceVersionSlug,
String targetProjectSlug,
String targetVersionSlug,
boolean useNewerTranslation,
MergeTranslationsTaskHandle handle) {
HProjectIteration sourceVersion =
projectIterationDAO.getBySlug(sourceProjectSlug, sourceVersionSlug);
if (sourceVersion == null) {
log.error("Cannot find source version of {}:{}", sourceProjectSlug, sourceVersionSlug);
return AsyncTaskResult.taskResult();
}
HProjectIteration targetVersion =
projectIterationDAO.getBySlug(targetProjectSlug, targetVersionSlug);
if (targetVersion == null) {
log.error("Cannot find target version of {}:{}", targetProjectSlug, targetVersionSlug);
return AsyncTaskResult.taskResult();
}
if (isVersionsEmpty(sourceVersion, targetVersion)) {
return AsyncTaskResult.taskResult();
}
if (getSupportedLocales(targetProjectSlug, targetVersionSlug).isEmpty()) {
log.error(
"No locales enabled in target version of {} [{}]", targetProjectSlug, targetVersionSlug);
return AsyncTaskResult.taskResult();
}
Optional<MergeTranslationsTaskHandle> taskHandleOpt = Optional.fromNullable(handle);
if (taskHandleOpt.isPresent()) {
prepareMergeTranslationsHandle(sourceVersion, targetVersion, taskHandleOpt.get());
}
Stopwatch overallStopwatch = Stopwatch.createStarted();
log.info(
"merge translations start: from {} to {}",
sourceProjectSlug + ":" + sourceVersionSlug,
targetProjectSlug + ":" + targetVersionSlug);
int startCount = 0;
int totalCount = getTotalMatchCount(sourceVersion.getId(), targetVersion.getId());
List<HLocale> supportedLocales =
getSupportedLocales(targetVersion.getProject().getSlug(), targetVersion.getSlug());
while (startCount < totalCount) {
int processedCount =
mergeTranslationBatch(
sourceVersion,
targetVersion,
supportedLocales,
useNewerTranslation,
startCount,
TEXTFLOWS_PER_BATCH);
if (taskHandleOpt.isPresent()) {
taskHandleOpt.get().increaseProgress(processedCount);
}
startCount += TEXTFLOWS_PER_BATCH;
textFlowDAO.clear();
}
versionStateCacheImpl.clearVersionStatsCache(targetVersion.getId());
log.info(
"merge translation end: from {} to {}, {}",
sourceProjectSlug + ":" + sourceVersionSlug,
targetProjectSlug + ":" + targetVersionSlug,
overallStopwatch);
return AsyncTaskResult.taskResult();
}
public IterationCopyTransTask(HProjectIteration projectIteration, HCopyTransOptions options) {
super(options, "IterationCopyTransTask: " + projectIteration.getSlug());
this.projectIteration = projectIteration;
}
// @Test(enabled = true, description = "this should only be executed manually in IDE")
@Ignore
@Test
@PerformanceProfiling
public void pushTranslation() {
EntityMaker entityMaker =
EntityMakerBuilder.builder()
.addFieldOrPropertyMaker(
HProject.class, "sourceViewURL", FixedValueMaker.EMPTY_STRING_MAKER)
.build();
HProjectIteration iteration = entityMaker.makeAndPersist(getEm(), HProjectIteration.class);
HLocale srcLocale = createAndPersistLocale(LocaleId.EN_US, getEm());
HLocale transLocale = createAndPersistLocale(LocaleId.DE, getEm());
String versionSlug = iteration.getSlug();
String projectSlug = iteration.getProject().getSlug();
HDocument document = new HDocument("message", ContentType.PO, srcLocale);
document.setProjectIteration(iteration);
getEm().persist(document);
getEm().flush();
// adjust this number to suit testing purpose
int numOfTextFlows = 50;
int numOfTextFlowsHavingTarget =
createSourceAndSomeTargets(document, transLocale, numOfTextFlows);
getEm().getTransaction().commit();
getEm().getTransaction().begin();
Long targetsCountBefore =
getEm()
.createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
.setParameter("locale", transLocale)
.getSingleResult();
Assertions.assertThat(targetsCountBefore).isEqualTo(numOfTextFlowsHavingTarget);
// ============ add targets =========
TranslationsResource translations = new TranslationsResource();
translations.setRevision(1);
for (int i = 0; i < numOfTextFlows; i++) {
addSampleTranslation(translations, "res" + i);
}
Monitor mon = MonitorFactory.start("");
log.info("==== start translateAllInDoc");
service.translateAllInDoc(
projectSlug,
versionSlug,
document.getDocId(),
transLocale.getLocaleId(),
translations,
extensions,
MergeType.AUTO,
false,
TranslationSourceType.API_UPLOAD);
log.info("==== stop translateAllInDoc: {}", mon.stop());
getEm().getTransaction().commit();
getEm().getTransaction().begin();
Long targetsCount =
getEm()
.createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class)
.setParameter("locale", transLocale)
.getSingleResult();
Assertions.assertThat(targetsCount).isEqualTo(numOfTextFlows);
List<HTextFlowTargetHistory> histories =
getEm()
.createQuery("from HTextFlowTargetHistory", HTextFlowTargetHistory.class)
.getResultList();
Assertions.assertThat(histories).hasSize(numOfTextFlowsHavingTarget);
}
// TODO Need to refactor this method to get Message statistic by default.
// This is to be consistance with UI which uses message stats, and for
// calculating remaining hours.
@Override
public ContainerTranslationStatistics getStatistics(
String projectSlug,
String iterationSlug,
boolean includeDetails,
boolean includeWordStats,
String[] locales) {
LocaleId[] localeIds;
// if no locales are specified, search in all locales
if (locales.length == 0) {
List<HLocale> iterationLocales =
localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug);
localeIds = new LocaleId[iterationLocales.size()];
for (int i = 0, iterationLocalesSize = iterationLocales.size();
i < iterationLocalesSize;
i++) {
HLocale loc = iterationLocales.get(i);
localeIds[i] = loc.getLocaleId();
}
} else {
localeIds = new LocaleId[locales.length];
for (int i = 0; i < locales.length; i++) {
localeIds[i] = new LocaleId(locales[i]);
}
}
HProjectIteration iteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug);
if (iteration == null) {
throw new NoSuchEntityException(projectSlug + "/" + iterationSlug);
}
Map<String, TransUnitCount> transUnitIterationStats =
projectIterationDAO.getAllStatisticsForContainer(iteration.getId());
Map<String, TransUnitWords> wordIterationStats =
projectIterationDAO.getAllWordStatsStatistics(iteration.getId());
ContainerTranslationStatistics iterationStats = new ContainerTranslationStatistics();
iterationStats.setId(iterationSlug);
iterationStats.addRef(
new Link(
URI.create(zPathService.generatePathForProjectIteration(iteration)),
"statSource",
"PROJ_ITER"));
long iterationTotalMssgs =
projectIterationDAO.getTotalMessageCountForIteration(iteration.getId());
long iterationTotalWords = projectIterationDAO.getTotalWordCountForIteration(iteration.getId());
for (LocaleId locId : localeIds) {
// trans unit level stats
TransUnitCount count = transUnitIterationStats.get(locId.getId());
// Stats might not return anything if nothing is translated
if (count == null) {
count = new TransUnitCount(0, 0, (int) iterationTotalMssgs);
}
HTextFlowTarget target =
localeServiceImpl.getLastTranslated(projectSlug, iterationSlug, locId);
String lastModifiedBy = "";
Date lastModifiedDate = null;
if (target != null) {
lastModifiedDate = target.getLastChanged();
if (target.getLastModifiedBy() != null) {
lastModifiedBy = target.getLastModifiedBy().getAccount().getUsername();
}
}
TransUnitWords wordCount = wordIterationStats.get(locId.getId());
if (wordCount == null) {
wordCount = new TransUnitWords(0, 0, (int) iterationTotalWords);
}
TranslationStatistics transUnitStats =
getMessageStats(count, locId, lastModifiedDate, lastModifiedBy);
transUnitStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
iterationStats.addStats(transUnitStats);
// word level stats
if (includeWordStats) {
TranslationStatistics wordsStats =
getWordsStats(wordCount, locId, lastModifiedDate, lastModifiedBy);
wordsStats.setRemainingHours(StatisticsUtil.getRemainingHours(wordCount));
iterationStats.addStats(wordsStats);
}
}
// TODO Do in a single query
if (includeDetails) {
for (String docId : iteration.getDocuments().keySet()) {
iterationStats.addDetailedStats(
this.getStatistics(projectSlug, iterationSlug, docId, includeWordStats, locales));
}
}
return iterationStats;
}