private void prepareTransUnitUpdatedEvent( int previousVersionNum, ContentState previousState, HTextFlowTarget target) { LocaleId localeId = target.getLocaleId(); HTextFlow textFlow = target.getTextFlow(); HDocument document = textFlow.getDocument(); HProjectIteration projectIteration = document.getProjectIteration(); String iterationSlug = projectIteration.getSlug(); String projectSlug = projectIteration.getProject().getSlug(); ProjectType projectType = projectIteration.getProjectType(); WorkspaceId workspaceId = new WorkspaceId(new ProjectIterationId(projectSlug, iterationSlug, projectType), localeId); Optional<TranslationWorkspace> workspaceOptional = translationWorkspaceManager.tryGetWorkspace(workspaceId); if (!workspaceOptional.isPresent()) { return; } TransUnitTransformer transUnitTransformer = serviceLocator.getInstance(TransUnitTransformer.class); TransUnit transUnit = transUnitTransformer.transform(textFlow, target, target.getLocale()); DocumentId documentId = new DocumentId(document.getId(), document.getDocId()); int wordCount = textFlow.getWordCount().intValue(); TransUnitUpdateInfo updateInfo = createTransUnitUpdateInfo( transUnit, documentId, wordCount, previousVersionNum, previousState); CacheValue context = updateContext.getIfPresent(new CacheKey(transUnit.getId(), transUnit.getLocaleId())); TransUnitUpdated updated; if (context != null) { updated = new TransUnitUpdated(updateInfo, context.editorClientId, context.updateType); log.debug("about to publish trans unit updated event {}", updated); } else if (ServletContexts.instance().getRequest() != null) { String sessionId = ServletContexts.instance().getRequest().getSession().getId(); EditorClientId editorClientId = new EditorClientId(sessionId, -1); updated = new TransUnitUpdated( updateInfo, editorClientId, TransUnitUpdated.UpdateType.NonEditorSave); } else { updated = new TransUnitUpdated( updateInfo, new EditorClientId("unknown", -1), TransUnitUpdated.UpdateType.NonEditorSave); } if (Events.exists()) { Events.instance() .raiseTransactionSuccessEvent( TextFlowTargetUpdatedEvent.EVENT_NAME, new TextFlowTargetUpdatedEvent(workspaceOptional.get(), target.getId(), updated)); } }
@Test public void getDocumentStatisticsWillReturnResult() { HDocument document = new HDocument(); document.setId(1L); when(documentDAO.getByProjectIterationAndDocId("a", "1", "authors")).thenReturn(document); ContainerTranslationStatistics statistics = new ContainerTranslationStatistics(); statistics.addStats(new TranslationStatistics(new TransUnitWords(10, 0, 0, 10, 0), "de")); statistics.addStats(new TranslationStatistics(new TransUnitCount(5, 0, 0, 10, 0), "de")); when(documentDAO.getStatistics(1L, LocaleId.DE)).thenReturn(statistics); Response response = service.getDocumentStatistics("a", "1", "authors", "de"); assertThat(response.getStatus()).isEqualTo(200); }
private void processWebHookDocumentMilestoneEvent( DocumentStatisticUpdatedEvent event, Collection<ContentState> contentStates, String message, int percentMilestone) { HProjectIteration version = projectIterationDAO.findById(event.getProjectIterationId()); HProject project = version.getProject(); if (!project.getWebHooks().isEmpty()) { WordStatistic stats = translationStateCacheImpl.getDocumentStatistics( event.getDocumentId(), event.getLocaleId()); WordStatistic oldStats = StatisticsUtil.copyWordStatistic(stats); if (oldStats != null) { oldStats.decrement(event.getNewState(), event.getWordCount()); oldStats.increment(event.getPreviousState(), event.getWordCount()); boolean shouldPublish = hasContentStateReachedMilestone(oldStats, stats, contentStates, percentMilestone); if (shouldPublish) { HDocument document = documentDAO.getById(event.getDocumentId()); String editorUrl = urlUtil.fullEditorDocumentUrl( project.getSlug(), version.getSlug(), event.getLocaleId(), LocaleId.EN_US, document.getDocId()); DocumentMilestoneEvent milestoneEvent = new DocumentMilestoneEvent( project.getSlug(), version.getSlug(), document.getDocId(), event.getLocaleId(), message, editorUrl); for (WebHook webHook : project.getWebHooks()) { publishDocumentMilestoneEvent(webHook, milestoneEvent); } } } } }
@Test public void checkPositionsNotNull() throws Exception { EntityManager em = getEm(); HIterationProject project = em.find(HIterationProject.class, 1l); // assertThat( project, notNullValue() ); HDocument hdoc = new HDocument("fullpath", ContentType.TextPlain, en_US); hdoc.setProjectIteration(project.getProjectIterations().get(0)); List<HTextFlow> textFlows = hdoc.getTextFlows(); HTextFlow flow1 = new HTextFlow(hdoc, "textflow1", "some content"); HTextFlow flow2 = new HTextFlow(hdoc, "textflow2", "more content"); textFlows.add(flow1); textFlows.add(flow2); em.persist(hdoc); em.flush(); // em.clear(); // hdoc = em.find(HDocument.class, docId); em.refresh(hdoc); List<HTextFlow> textFlows2 = hdoc.getTextFlows(); assertThat(textFlows2.size(), is(2)); flow1 = textFlows2.get(0); assertThat(flow1, notNullValue()); flow2 = textFlows2.get(1); assertThat(flow2, notNullValue()); // TODO: we should automate this... hdoc.incrementRevision(); textFlows2.remove(flow1); flow1.setObsolete(true); dao.syncRevisions(hdoc, flow1); // flow1.setPos(null); em.flush(); em.refresh(hdoc); em.refresh(flow1); em.refresh(flow2); assertThat(hdoc.getTextFlows().size(), is(1)); flow2 = hdoc.getTextFlows().get(0); assertThat(flow2.getResId(), equalTo("textflow2")); flow1 = hdoc.getAllTextFlows().get("textflow1"); // assertThat(flow1.getPos(), nullValue()); assertThat(flow1.isObsolete(), is(true)); assertThat(flow1.getRevision(), is(2)); flow2 = hdoc.getAllTextFlows().get("textflow2"); // assertThat(flow1.getPos(), is(0)); assertThat(flow2.isObsolete(), is(false)); }
@Override public GetDocumentListResult execute(GetDocumentList action, ExecutionContext context) throws ActionException { ZanataIdentity.instance().checkLoggedIn(); ProjectIterationId iterationId = action.getProjectIterationId(); ArrayList<DocumentInfo> docs = new ArrayList<DocumentInfo>(); HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(iterationId.getProjectSlug(), iterationId.getIterationSlug()); Collection<HDocument> hDocs = hProjectIteration.getDocuments().values(); for (HDocument hDoc : hDocs) { DocumentId docId = new DocumentId(hDoc.getId()); DocumentInfo doc = new DocumentInfo(docId, hDoc.getName(), hDoc.getPath()); docs.add(doc); } return new GetDocumentListResult(iterationId, docs); }
@Override @Transactional public void makeObsolete(HDocument document) { // Simply make it obsolete. This method is here in case this logic is // expanded. document.setObsolete(true); documentDAO.makePersistent(document); documentDAO.flush(); clearStatsCacheForUpdatedDocument(document); }
@Override @Transactional public HDocument saveDocument( String projectSlug, String iterationSlug, Resource sourceDoc, Set<String> extensions, boolean copyTrans) { // Only active iterations allow the addition of a document HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(projectSlug, iterationSlug); // Check permission identity.checkPermission(hProjectIteration, "import-template"); String docId = sourceDoc.getName(); HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, docId); HLocale hLocale = this.localeServiceImpl.validateSourceLocale(sourceDoc.getLang()); boolean changed = false; int nextDocRev; if (document == null) { // must be a create operation nextDocRev = 1; changed = true; // TODO check that entity name matches id parameter document = new HDocument(sourceDoc.getName(), sourceDoc.getContentType(), hLocale); document.setProjectIteration(hProjectIteration); hProjectIteration.getDocuments().put(docId, document); document = documentDAO.makePersistent(document); } else if (document.isObsolete()) { // must also be a create operation nextDocRev = document.getRevision() + 1; changed = true; document.setObsolete(false); // not sure if this is needed hProjectIteration.getDocuments().put(docId, document); } else { // must be an update operation nextDocRev = document.getRevision() + 1; } changed |= resourceUtils.transferFromResource(sourceDoc, document, extensions, hLocale, nextDocRev); documentDAO.flush(); long actorId = authenticatedAccount.getPerson().getId(); if (changed) { documentUploadedEvent.fireAfterSuccess( new DocumentUploadedEvent(actorId, document.getId(), true, hLocale.getLocaleId())); clearStatsCacheForUpdatedDocument(document); } if (copyTrans && nextDocRev == 1) { copyTranslations(document); } return document; }
// FIXME this test only works if resources-dev is on the classpath @SuppressWarnings("unchecked") @Test public void ensureHistoryOnTextFlow() { EntityManager em = getEm(); HIterationProject project = em.find(HIterationProject.class, 1l); // assertThat( project, notNullValue() ); HDocument hdoc = new HDocument("fullpath", ContentType.TextPlain, en_US); hdoc.setProjectIteration(project.getProjectIterations().get(0)); List<HTextFlow> textFlows = hdoc.getTextFlows(); HTextFlow flow1 = new HTextFlow(hdoc, "textflow3", "some content"); HTextFlow flow2 = new HTextFlow(hdoc, "textflow4", "more content"); textFlows.add(flow1); textFlows.add(flow2); em.persist(hdoc); em.flush(); hdoc.incrementRevision(); flow1.setContent("nwe content!"); dao.syncRevisions(hdoc, flow1); em.flush(); HTextFlowTarget target = new HTextFlowTarget(flow1, de_DE); target.setContent("hello world"); em.persist(target); em.flush(); target.setContent("h2"); em.flush(); List<HTextFlowTargetHistory> hist = em.createQuery("from HTextFlowTargetHistory h where h.textFlowTarget =:target") .setParameter("target", target) .getResultList(); assertThat(hist, notNullValue()); assertThat(hist.size(), not(0)); }
@Override public ProcessStatus startSourceDocCreation( final String idNoSlash, final String projectSlug, final String iterationSlug, final Resource resource, final Set<String> extensions, final boolean copytrans) { HProjectIteration hProjectIteration = retrieveAndCheckIteration(projectSlug, iterationSlug, true); resourceUtils.validateExtensions(extensions); // gettext, comment HDocument document = documentDAO.getByDocIdAndIteration(hProjectIteration, resource.getName()); // already existing non-obsolete document. if (document != null) { if (!document.isObsolete()) { // updates must happen through PUT on the actual resource ProcessStatus status = new ProcessStatus(); status.setStatusCode(ProcessStatusCode.Failed); status.getMessages().add("A document with name " + resource.getName() + " already exists."); return status; } } String name = "SourceDocCreation: " + projectSlug + "-" + iterationSlug + "-" + idNoSlash; AsyncTaskHandle<HDocument> handle = new AsyncTaskHandle<HDocument>(); Serializable taskId = asyncTaskHandleManager.registerTaskHandle(handle); documentServiceImpl.saveDocumentAsync( projectSlug, iterationSlug, resource, extensions, copytrans, true, handle); return getProcessStatus(taskId.toString()); // TODO Change to return 202 // Accepted, // with a url to get the // progress }
private void signalCopiedTranslation( HTextFlowTarget target, ContentState previousState, Long wordCount) { /* * Using a direct method call instead of an event because it's easier to * read. Since these events are being called synchronously (as opposed * to an 'after Transaction' events), there is no big performance gain * and makes the code easier to read and navigate. */ // TODO how was this not causing duplicate events? Is this bypassing TranslationServiceImpl? // FIXME other observers may not be notified HDocument document = target.getTextFlow().getDocument(); DocumentLocaleKey key = new DocumentLocaleKey(document.getId(), target.getLocaleId()); Map<ContentState, Long> contentStates = Maps.newHashMap(); DocStatsEvent.updateContentStateDeltas( contentStates, target.getState(), previousState, wordCount); DocStatsEvent docEvent = new DocStatsEvent( key, document.getProjectIteration().getId(), contentStates, target.getId()); versionStateCacheImpl.docStatsUpdated(docEvent); }
/** * Half of the HTextFlow will have target in it. * * @return number of textflows that has target in it. */ private static int createSourceAndSomeTargets( HDocument doc, HLocale transLocale, int numOfTextFlows) { HTextFlowBuilder baseBuilder = new HTextFlowBuilder().withDocument(doc).withTargetLocale(transLocale); int targetCount = 0; for (int i = 0; i < numOfTextFlows; i++) { HTextFlowBuilder builder = baseBuilder.withResId("res" + i).withSourceContent("hello world " + i); if (i % 2 == 0) { builder = builder .withTargetState(ContentState.NeedReview) .withTargetContent("previous translation"); targetCount++; } doc.getTextFlows().add(builder.build()); } return targetCount; }
public Integer runCopyTrans( HLocale targetLocale, HCopyTransOptions options, HDocument document, boolean requireTranslationReview, List<HTextFlow> copyTargets) { int numCopied = 0; boolean checkContext = false, checkProject = false, checkDocument = false; // Only outright reject copies if the options say so if (options.getDocIdMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkDocument = true; } if (options.getProjectMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkProject = true; } if (options.getContextMismatchAction() == HCopyTransOptions.ConditionRuleAction.REJECT) { checkContext = true; } Long actorId = authenticatedAccount.getPerson().getId(); for (HTextFlow textFlow : copyTargets) { if (shouldFindMatch(textFlow, targetLocale, requireTranslationReview)) { Optional<HTextFlowTarget> bestMatch = translationFinder.searchBestMatchTransMemory( textFlow, targetLocale.getLocaleId(), document.getLocale().getLocaleId(), checkContext, checkDocument, checkProject); if (bestMatch.isPresent()) { numCopied++; saveCopyTransMatch(actorId, bestMatch.get(), textFlow, options, requireTranslationReview); } } } return numCopied; }
@Override public GetDocumentListResult execute(GetDocumentList action, ExecutionContext context) throws ActionException { identity.checkLoggedIn(); LocaleId localeId = action.getWorkspaceId().getLocaleId(); ProjectIterationId iterationId = action.getProjectIterationId(); ArrayList<DocumentInfo> docs = new ArrayList<DocumentInfo>(); HProjectIteration hProjectIteration = projectIterationDAO.getBySlug(iterationId.getProjectSlug(), iterationId.getIterationSlug()); Collection<HDocument> hDocs = hProjectIteration.getDocuments().values(); for (HDocument hDoc : hDocs) { if (action.getFilters() == null || action.getFilters().isEmpty() || action.getFilters().contains(hDoc.getPath() + hDoc.getName())) { DocumentId docId = new DocumentId(hDoc.getId(), hDoc.getDocId()); TranslationStats stats = documentDAO.getStatistics(hDoc.getId(), localeId); String lastModifiedBy = ""; HPerson person = hDoc.getLastModifiedBy(); if (person != null) { lastModifiedBy = person.getAccount().getUsername(); } Map<String, String> downloadExtensions = new HashMap<String, String>(); downloadExtensions.put(".po", "po?docId=" + hDoc.getDocId()); if (translationFileServiceImpl.hasPersistedDocument( iterationId.getProjectSlug(), iterationId.getIterationSlug(), hDoc.getPath(), hDoc.getName())) { String extension = "." + translationFileServiceImpl.getFileExtension( iterationId.getProjectSlug(), iterationId.getIterationSlug(), hDoc.getPath(), hDoc.getName()); downloadExtensions.put(extension, "baked?docId=" + hDoc.getDocId()); } DocumentInfo doc = new DocumentInfo( docId, hDoc.getName(), hDoc.getPath(), hDoc.getLocale().getLocaleId(), stats, lastModifiedBy, hDoc.getLastChanged(), downloadExtensions); docs.add(doc); } } return new GetDocumentListResult(iterationId, docs); }
@Override public String call() throws Exception { // Needed Components DocumentDAO documentDAO = (DocumentDAO) Component.getInstance(DocumentDAO.class); LocaleDAO localeDAO = (LocaleDAO) Component.getInstance(LocaleDAO.class); ResourceUtils resourceUtils = (ResourceUtils) Component.getInstance(ResourceUtils.class); TextFlowTargetDAO textFlowTargetDAO = (TextFlowTargetDAO) Component.getInstance(TextFlowTargetDAO.class); FileSystemService fileSystemService = (FileSystemService) Component.getInstance(FileSystemServiceImpl.class); ConfigurationService configurationService = (ConfigurationService) Component.getInstance(ConfigurationServiceImpl.class); final String projectDirectory = projectSlug + "-" + iterationSlug + "/"; final HLocale hLocale = localeDAO.findByLocaleId(new LocaleId(localeId)); final String mappedLocale = hLocale.getLocaleId().getId(); final String localeDirectory = projectDirectory + mappedLocale + "/"; final File downloadFile = fileSystemService.createDownloadStagingFile("zip"); final FileOutputStream output = new FileOutputStream(downloadFile); final ZipOutputStream zipOutput = new ZipOutputStream(output); zipOutput.setMethod(ZipOutputStream.DEFLATED); final PoWriter2 poWriter = new PoWriter2(false, !isPoProject); final Set<String> extensions = new HashSet<String>(); extensions.add("gettext"); extensions.add("comment"); // Generate the download descriptor file String downloadId = fileSystemService.createDownloadDescriptorFile( downloadFile, projectSlug + "_" + iterationSlug + "_" + localeId + ".zip", userName); // Add the config file at the root of the project directory String configFilename = projectDirectory + configurationService.getConfigurationFileName(); zipOutput.putNextEntry(new ZipEntry(configFilename)); zipOutput.write( configurationService .getConfigForOfflineTranslation(projectSlug, iterationSlug, hLocale) .getBytes()); zipOutput.closeEntry(); getHandle().increaseProgress(1); final List<HDocument> allIterationDocs = documentDAO.getAllByProjectIteration(projectSlug, iterationSlug); for (HDocument document : allIterationDocs) { // Stop the process if signaled to do so if (getHandle().isCancelled()) { zipOutput.close(); downloadFile.delete(); fileSystemService.deleteDownloadDescriptorFile(downloadId); return null; } TranslationsResource translationResource = new TranslationsResource(); List<HTextFlowTarget> hTargets = textFlowTargetDAO.findTranslations(document, hLocale); resourceUtils.transferToTranslationsResource( translationResource, document, hLocale, extensions, hTargets, Optional.<String>absent()); Resource res = resourceUtils.buildResource(document); String filename = localeDirectory + document.getDocId() + ".po"; zipOutput.putNextEntry(new ZipEntry(filename)); poWriter.writePo(zipOutput, "UTF-8", res, translationResource); zipOutput.closeEntry(); getHandle().increaseProgress(1); } zipOutput.flush(); zipOutput.close(); return downloadId; }
private void clearStatsCacheForUpdatedDocument(HDocument document) { versionStateCacheImpl.clearVersionStatsCache(document.getProjectIteration().getId()); translationStateCacheImpl.clearDocumentStatistics(document.getId()); }
@Override public ContainerTranslationStatistics getStatistics( String projectSlug, String iterationSlug, String docId, boolean includeWordStats, String[] locales) { LocaleId[] localeIds; // if no locales are specified, search in all locales if (locales.length == 0) { List<HLocale> iterationLocales = localeServiceImpl.getSupportedLangugeByProjectIteration(projectSlug, iterationSlug); localeIds = new LocaleId[iterationLocales.size()]; for (int i = 0, iterationLocalesSize = iterationLocales.size(); i < iterationLocalesSize; i++) { HLocale loc = iterationLocales.get(i); localeIds[i] = loc.getLocaleId(); } } else { localeIds = new LocaleId[locales.length]; for (int i = 0; i < locales.length; i++) { localeIds[i] = new LocaleId(locales[i]); } } HDocument document = documentDAO.getByProjectIterationAndDocId(projectSlug, iterationSlug, docId); if (document == null) { throw new NoSuchEntityException(projectSlug + "/" + iterationSlug + "/" + docId); } ContainerTranslationStatistics docStatistics = new ContainerTranslationStatistics(); docStatistics.setId(docId); docStatistics.addRef( new Link(URI.create(zPathService.generatePathForDocument(document)), "statSource", "DOC")); for (LocaleId localeId : localeIds) { ContainerTranslationStatistics docStats = getDocStatistics(document.getId(), localeId); DocumentStatus docStatus = translationStateCacheImpl.getDocumentStatus(document.getId(), localeId); TranslationStatistics docWordStatistic = docStats.getStats(localeId.getId(), StatUnit.WORD); TranslationStatistics docMsgStatistic = docStats.getStats(localeId.getId(), StatUnit.MESSAGE); docMsgStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy()); docMsgStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate()); docMsgStatistic.setLastTranslated( getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy())); docStatistics.addStats(docMsgStatistic); if (includeWordStats) { docWordStatistic.setLastTranslatedBy(docStatus.getLastTranslatedBy()); docWordStatistic.setLastTranslatedDate(docStatus.getLastTranslatedDate()); docWordStatistic.setLastTranslated( getLastTranslated(docStatus.getLastTranslatedDate(), docStatus.getLastTranslatedBy())); docStatistics.addStats(docWordStatistic); } } return docStatistics; }
// @Test(enabled = true, description = "this should only be executed manually in IDE") @Ignore @Test @PerformanceProfiling public void pushTranslation() { EntityMaker entityMaker = EntityMakerBuilder.builder() .addFieldOrPropertyMaker( HProject.class, "sourceViewURL", FixedValueMaker.EMPTY_STRING_MAKER) .build(); HProjectIteration iteration = entityMaker.makeAndPersist(getEm(), HProjectIteration.class); HLocale srcLocale = createAndPersistLocale(LocaleId.EN_US, getEm()); HLocale transLocale = createAndPersistLocale(LocaleId.DE, getEm()); String versionSlug = iteration.getSlug(); String projectSlug = iteration.getProject().getSlug(); HDocument document = new HDocument("message", ContentType.PO, srcLocale); document.setProjectIteration(iteration); getEm().persist(document); getEm().flush(); // adjust this number to suit testing purpose int numOfTextFlows = 50; int numOfTextFlowsHavingTarget = createSourceAndSomeTargets(document, transLocale, numOfTextFlows); getEm().getTransaction().commit(); getEm().getTransaction().begin(); Long targetsCountBefore = getEm() .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class) .setParameter("locale", transLocale) .getSingleResult(); Assertions.assertThat(targetsCountBefore).isEqualTo(numOfTextFlowsHavingTarget); // ============ add targets ========= TranslationsResource translations = new TranslationsResource(); translations.setRevision(1); for (int i = 0; i < numOfTextFlows; i++) { addSampleTranslation(translations, "res" + i); } Monitor mon = MonitorFactory.start(""); log.info("==== start translateAllInDoc"); service.translateAllInDoc( projectSlug, versionSlug, document.getDocId(), transLocale.getLocaleId(), translations, extensions, MergeType.AUTO, false, TranslationSourceType.API_UPLOAD); log.info("==== stop translateAllInDoc: {}", mon.stop()); getEm().getTransaction().commit(); getEm().getTransaction().begin(); Long targetsCount = getEm() .createQuery("select count(*) from HTextFlowTarget where locale = :locale", Long.class) .setParameter("locale", transLocale) .getSingleResult(); Assertions.assertThat(targetsCount).isEqualTo(numOfTextFlows); List<HTextFlowTargetHistory> histories = getEm() .createQuery("from HTextFlowTargetHistory", HTextFlowTargetHistory.class) .getResultList(); Assertions.assertThat(histories).hasSize(numOfTextFlowsHavingTarget); }