@Override public String execute() throws Exception { CategoryOptionGroupSet categoryOptionGroupSet = dataElementCategoryService.getCategoryOptionGroupSet(id); dataElementCategoryService.deleteCategoryOptionGroupSet(categoryOptionGroupSet); return SUCCESS; }
@Override public void setUpTest() { categoryCombo = categoryService.getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME); categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); dataElementIds = new HashSet<>(); indicatorIds = new HashSet<>(); periodIds = new HashSet<>(); organisationUnitIds = new HashSet<>(); }
@Override public DataApprovalLevel getLowestDataApprovalLevel( OrganisationUnit orgUnit, DataElementCategoryOptionCombo attributeOptionCombo) { Set<CategoryOptionGroupSet> cogSets = null; if (attributeOptionCombo != null && attributeOptionCombo != categoryService.getDefaultDataElementCategoryOptionCombo()) { cogSets = new HashSet<>(); for (DataElementCategoryOption option : attributeOptionCombo.getCategoryOptions()) { if (option.getGroupSets() != null) { cogSets.addAll(option.getGroupSets()); } } } int orgUnitLevel = orgUnit.getLevel(); List<DataApprovalLevel> approvalLevels = getDataApprovalLevelsByOrgUnitLevel(orgUnitLevel); for (DataApprovalLevel level : Lists.reverse(approvalLevels)) { if (level.getCategoryOptionGroupSet() == null) { if (cogSets == null) { return level; } } else if (cogSets != null && cogSets.contains(level.getCategoryOptionGroupSet())) { return level; } } return null; }
public String execute() { if (dataSetId != null) { dataSet = dataSetService.getDataSet(dataSetId); operands = new ArrayList<DataElementOperand>( dataElementCategoryService.getFullOperands(dataSet.getDataElements())); } else { operands = new ArrayList<DataElementOperand>(); } if (key != null) { Iterator<DataElementOperand> iterator = operands.iterator(); while (iterator.hasNext()) { DataElementOperand operand = iterator.next(); if (operand.getOperandName().toLowerCase().indexOf(key.toLowerCase()) == -1) { iterator.remove(); } } } Collections.sort(operands, new DataElementOperandNameComparator()); if (usePaging) { this.paging = createPaging(operands.size()); operands = operands.subList(paging.getStartPos(), paging.getEndPos()); } return SUCCESS; }
public String execute() { try { DataElementCategory category = dataElementCategoryService.getDataElementCategory(id); dataElementCategoryService.deleteDataElementCategory(category); } catch (DeleteNotAllowedException ex) { if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) { message = i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage(); return ERROR; } } return SUCCESS; }
@Override protected void importMatching(DataElementCategoryOption object, DataElementCategoryOption match) { match.setCode(object.getCode()); match.setUid(object.getUid()); match.setName(object.getName()); categoryService.updateDataElementCategoryOption(match); }
@Override public String execute() { dataElementCategories = new ArrayList<>(dataElementCategoryService.getAllDataElementCategories()); Collections.sort(dataElementCategories, IdentifiableObjectNameComparator.INSTANCE); return SUCCESS; }
/** * Can the user read from this CategoryOptionGroupSet (COGS)? * * <p>If the COGS is null, then the user must have no dimension constraints. (In other words, the * user must be able to read across all category option groups.) * * <p>If the COGS is not null, then the user must be able to read at least one category option * group from the category option group set. * * @param cogs The category option group set to test * @return true if user can read at least one category option group. */ private boolean canReadCOGS(User user, CategoryOptionGroupSet cogs) { if (cogs == null) { UserCredentials userCredentials = user.getUserCredentials(); return CollectionUtils.isEmpty(userCredentials.getCogsDimensionConstraints()) && CollectionUtils.isEmpty(userCredentials.getCatDimensionConstraints()); } return !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs)); }
public String execute() throws Exception { // --------------------------------------------------------------------- // Prepare values // --------------------------------------------------------------------- code = nullIfEmpty(code); shortName = nullIfEmpty(shortName); description = nullIfEmpty(description); PeriodType periodType = PeriodType.getPeriodTypeByName(frequencySelect); DataSet dataSet = new DataSet(name, shortName, code, periodType); MapLegendSet legendSet = mappingService.getMapLegendSet(selectedLegendSetId); dataSet.setExpiryDays(expiryDays); dataSet.setTimelyDays(timelyDays); dataSet.setSkipAggregation(skipAggregation); for (String id : dataElementsSelectedList) { dataSet.addDataElement(dataElementService.getDataElement(Integer.parseInt(id))); } Set<Indicator> indicators = new HashSet<Indicator>(); for (String id : indicatorsSelectedList) { indicators.add(indicatorService.getIndicator(Integer.parseInt(id))); } if (categoryComboId != null) { dataSet.setCategoryCombo(categoryService.getDataElementCategoryCombo(categoryComboId)); } dataSet.setDescription(description); dataSet.setVersion(1); dataSet.setMobile(false); dataSet.setIndicators(indicators); dataSet.setNotificationRecipients(userGroupService.getUserGroup(notificationRecipients)); dataSet.setAllowFuturePeriods(allowFuturePeriods); dataSet.setFieldCombinationRequired(fieldCombinationRequired); dataSet.setValidCompleteOnly(validCompleteOnly); dataSet.setNotifyCompletingUser(notifyCompletingUser); dataSet.setApproveData(approveData); dataSet.setSkipOffline(skipOffline); dataSet.setDataElementDecoration(dataElementDecoration); dataSet.setRenderAsTabs(renderAsTabs); dataSet.setRenderHorizontally(renderHorizontally); dataSet.setLegendSet(legendSet); dataSetService.addDataSet(dataSet); userService.assignDataSetToUserRole(dataSet); return SUCCESS; }
@Override public String execute() { DataElementCategoryCombo categoryCombo = dataElementCategoryService.getDataElementCategoryCombo(id); DataElementCategoryCombo defaultCategoryCombo = dataElementCategoryService.getDataElementCategoryComboByName( DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME); if (!categoryCombo.equals(defaultCategoryCombo)) { try { dataElementCategoryService.deleteDataElementCategoryCombo(categoryCombo); } catch (DeleteNotAllowedException ex) { if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) { message = i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage(); return ERROR; } } } return SUCCESS; }
@Override public Set<CategoryOptionGroup> getCogDimensionConstraints(UserCredentials userCredentials) { Set<CategoryOptionGroup> groups = null; Set<CategoryOptionGroupSet> cogsConstraints = userCredentials.getCogsDimensionConstraints(); if (cogsConstraints != null && !cogsConstraints.isEmpty()) { groups = new HashSet<>(); for (CategoryOptionGroupSet cogs : cogsConstraints) { groups.addAll(categoryService.getCategoryOptionGroups(cogs)); } } return groups; }
@Override public Set<DataElementCategoryOption> getCoDimensionConstraints(UserCredentials userCredentials) { Set<DataElementCategoryOption> options = null; Set<DataElementCategory> catConstraints = userCredentials.getCatDimensionConstraints(); if (catConstraints != null && !catConstraints.isEmpty()) { options = new HashSet<>(); for (DataElementCategory category : catConstraints) { options.addAll(categoryService.getDataElementCategoryOptions(category)); } } return options; }
public String execute() { if (name != null) { DataElementCategoryOption match = dataElementCategoryService.getDataElementCategoryOptionByName(name); if (match != null && (id == null || match.getId() != id)) { message = i18n.getString("name_in_use"); return ERROR; } } message = "ok"; return SUCCESS; }
@Override public List<DataApprovalLevel> getUserDataApprovalLevels() { UserCredentials userCredentials = currentUserService.getCurrentUser().getUserCredentials(); boolean mayApprove = userCredentials.isAuthorized(DataApproval.AUTH_APPROVE); boolean mayApproveAtLowerLevels = userCredentials.isAuthorized(DataApproval.AUTH_APPROVE_LOWER_LEVELS); boolean mayAcceptAtLowerLevels = userCredentials.isAuthorized(DataApproval.AUTH_ACCEPT_LOWER_LEVELS); if (!mayApprove && !mayApproveAtLowerLevels && !mayAcceptAtLowerLevels) { return new ArrayList<>(); } int lowestNumberOrgUnitLevel = getCurrentUsersLowestNumberOrgUnitLevel(); boolean canSeeAllDimensions = CollectionUtils.isEmpty(userService.getCoDimensionConstraints(userCredentials)) && CollectionUtils.isEmpty(userService.getCogDimensionConstraints(userCredentials)); List<DataApprovalLevel> approvalLevels = getAllDataApprovalLevels(); List<DataApprovalLevel> userDataApprovalLevels = new ArrayList<>(); boolean addLevel = false; for (DataApprovalLevel approvalLevel : approvalLevels) { if (!addLevel && approvalLevel.getOrgUnitLevel() >= lowestNumberOrgUnitLevel) { CategoryOptionGroupSet cogs = approvalLevel.getCategoryOptionGroupSet(); addLevel = securityService.canRead(approvalLevel) && cogs == null ? canSeeAllDimensions : (securityService.canRead(cogs) && !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs))); } if (addLevel) { userDataApprovalLevels.add(approvalLevel); } } return userDataApprovalLevels; }
protected void addDataValue( OrganisationUnit unit, Period period, String expression, String value, Set<DataValue> oldList, Set<DataValue> newList) { // value = value.replaceAll( "\\.", "" ).replace( ",", "." ); DataElementOperand operand = expressionService.getOperandsInExpression(expression).iterator().next(); DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId()); DataElementCategoryOptionCombo optionCombo = categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId()); String storedBy = currentUserService.getCurrentUsername(); DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo); if (dataValue == null) { dataValue = new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo); dataValueService.addDataValue(dataValue); newList.add(dataValue); } else { DataValue backedUpDataValue = new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo); oldList.add(backedUpDataValue); dataValue.setValue(value); dataValue.setTimestamp(new Date()); dataValue.setStoredBy(storedBy); dataValueService.updateDataValue(dataValue); } }
public String execute() { DataElement dataElement = dataElementService.getDataElement(dataElementId); DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo(categoryOptionComboId); Period period = PeriodType.createPeriodExternalId(periodId); OrganisationUnit source = organisationUnitService.getOrganisationUnit(organisationUnitId); DataValue dataValue = dataValueService.getDataValue(source, dataElement, period, categoryOptionCombo); boolean isMarked = dataValue.isFollowup(); dataValue.setFollowup(!isMarked); dataValueService.updateDataValue(dataValue); message = !isMarked ? "marked" : "unmarked"; log.info(!isMarked ? "Data value marked for follow-up" : "Data value unmarked for follow-up"); return SUCCESS; }
@Override public String execute() { optionCombo = categoryService.getDataElementCategoryOptionCombo(id); return SUCCESS; }
@Override protected DataElementCategoryOption getMatching(DataElementCategoryOption object) { return categoryService.getDataElementCategoryOptionByName(object.getName()); }
@RequestMapping(method = RequestMethod.POST, produces = "text/plain") public void saveDataValue( @RequestParam String de, @RequestParam String cc, @RequestParam String pe, @RequestParam String ou, @RequestParam String value, HttpServletResponse response) { DataElement dataElement = dataElementService.getDataElement(de); if (dataElement == null) { ContextUtils.conflictResponse(response, "Illegal data element identifier: " + de); return; } DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo(cc); if (categoryOptionCombo == null) { ContextUtils.conflictResponse(response, "Illegal category option combo identifier: " + cc); return; } Period period = PeriodType.getPeriodFromIsoString(pe); if (period == null) { ContextUtils.conflictResponse(response, "Illegal period identifier: " + pe); return; } OrganisationUnit organisationUnit = organisationUnitService.getOrganisationUnit(ou); if (organisationUnit == null) { ContextUtils.conflictResponse(response, "Illegal organisation unit identifier: " + ou); return; } if (dataSetService.isLocked(dataElement, period, organisationUnit, null)) { ContextUtils.conflictResponse(response, "Data set is locked"); return; } value = StringUtils.trimToNull(value); String storedBy = currentUserService.getCurrentUsername(); Date now = new Date(); DataValue dataValue = dataValueService.getDataValue(organisationUnit, dataElement, period, categoryOptionCombo); if (dataValue == null) { if (value != null) { dataValue = new DataValue( dataElement, period, organisationUnit, value, storedBy, now, null, categoryOptionCombo); dataValueService.addDataValue(dataValue); } } else { dataValue.setValue(value); dataValue.setTimestamp(now); dataValue.setStoredBy(storedBy); dataValueService.updateDataValue(dataValue); } }
@Override public String execute() throws Exception { for (String aggregateValue : aggregateValues) { // ----------------------------------------------------------------- // Get params // ----------------------------------------------------------------- String[] info = aggregateValue.split(SEPERATE_SIGN); int dataElementId = Integer.parseInt(info[0]); int optionComboId = Integer.parseInt(info[1]); String periodIsoId = info[2]; int orgunitId = Integer.parseInt(info[3]); String resultValue = info[4]; // ----------------------------------------------------------------- // Create objects // ----------------------------------------------------------------- DataElement dataElement = dataElementService.getDataElement(dataElementId); DataElementCategoryOptionCombo optionCombo = categoryService.getDataElementCategoryOptionCombo(optionComboId); DataElementCategoryOptionCombo attributeOptioncombo = categoryService.getDefaultDataElementCategoryOptionCombo(); Period period = PeriodType.getPeriodFromIsoString(periodIsoId); OrganisationUnit orgunit = organisationUnitService.getOrganisationUnit(orgunitId); DataValue dataValue = dataValueService.getDataValue( dataElement, period, orgunit, optionCombo, attributeOptioncombo); // ----------------------------------------------------------------- // Save/Update/Delete data-values // ----------------------------------------------------------------- if (resultValue != "0.0") { if (dataValue == null) { dataValue = new DataValue( dataElement, period, orgunit, optionCombo, attributeOptioncombo, "" + resultValue, CaseAggregationCondition.AUTO_STORED_BY, new Date(), null); dataValueService.addDataValue(dataValue); } else { dataValue.setValue(resultValue); dataValue.setLastUpdated(new Date()); dataValue.setStoredBy(CaseAggregationCondition.AUTO_STORED_BY); dataValueService.updateDataValue(dataValue); } } else if (dataValue != null) { dataValueService.deleteDataValue(dataValue); } } return SUCCESS; }
@Transactional public void export( Collection<Integer> dataElementIds, Collection<Integer> indicatorIds, Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, Collection<Integer> organisationUnitGroupIds, TaskId id) { final int cpuCores = SystemUtils.getCpuCores(); Clock clock = new Clock() .startClock() .logTime( "Data mart export process started, number of CPU cores: " + cpuCores + ", " + SystemUtils.getMemoryString()); notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started"); // --------------------------------------------------------------------- // Recreate temporary tables // --------------------------------------------------------------------- dataMartManager.dropTempAggregatedTables(); dataMartManager.createTempAggregatedTables(); clock.logTime("Recreated temporary tables"); // --------------------------------------------------------------------- // Replace null with empty collection // --------------------------------------------------------------------- dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>(); indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>(); periodIds = periodIds != null ? periodIds : new ArrayList<Integer>(); organisationUnitIds = organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>(); organisationUnitGroupIds = organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>(); clock.logTime( "Data elements: " + dataElementIds.size() + ", indicators: " + indicatorIds.size() + ", periods: " + periodIds.size() + ", org units: " + organisationUnitIds.size()); // --------------------------------------------------------------------- // Get objects // --------------------------------------------------------------------- final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds); final Collection<Period> periods = periodService.getPeriods(periodIds); final List<OrganisationUnit> organisationUnits = new ArrayList<OrganisationUnit>( organisationUnitService.getOrganisationUnits(organisationUnitIds)); final Collection<OrganisationUnitGroup> organisationUnitGroups = organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds); final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds); clock.logTime("Retrieved meta-data objects"); notifier.notify(id, DATAMART, "Filtering meta-data"); // --------------------------------------------------------------------- // Filter objects // --------------------------------------------------------------------- organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits); FilterUtils.filter(dataElements, new AggregatableDataElementFilter()); FilterUtils.filter(dataElements, new DataElementWithAggregationFilter()); expressionService.filterInvalidIndicators(indicators); clock.logTime("Filtered objects"); notifier.notify(id, DATAMART, "Loading indicators"); // --------------------------------------------------------------------- // Explode indicator expressions // --------------------------------------------------------------------- for (Indicator indicator : indicators) { indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator())); indicator.setExplodedDenominator( expressionService.explodeExpression(indicator.getDenominator())); } clock.logTime("Exploded indicator expressions"); notifier.notify(id, DATAMART, "Loading data elements"); // --------------------------------------------------------------------- // Get operands // --------------------------------------------------------------------- final Collection<DataElementOperand> dataElementOperands = categoryService.getOperands(dataElements); final List<DataElementOperand> indicatorOperands = new ArrayList<DataElementOperand>( categoryService.populateOperands( expressionService.getOperandsInIndicators(indicators))); Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>(); allOperands.addAll(dataElementOperands); allOperands.addAll(indicatorOperands); clock.logTime("Retrieved operands: " + allOperands.size()); notifier.notify(id, DATAMART, "Loading periods"); // --------------------------------------------------------------------- // Filter out future periods // --------------------------------------------------------------------- FilterUtils.filter(periods, new PastAndCurrentPeriodFilter()); clock.logTime("Number of periods: " + periods.size()); notifier.notify(id, DATAMART, "Filtering data elements without data"); // --------------------------------------------------------------------- // Remove operands without data // --------------------------------------------------------------------- allOperands = dataMartManager.getOperandsWithData(allOperands); indicatorOperands.retainAll(allOperands); clock.logTime( "Number of operands with data: " + allOperands.size() + ", " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Populating crosstabulation table"); // --------------------------------------------------------------------- // Create crosstabtable // --------------------------------------------------------------------- final Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods)); final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods); final Set<Integer> orgUnitChildrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds); final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds); final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds); final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands); Collections.shuffle(operandList); final List<List<DataElementOperand>> operandPages = new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages(); List<Future<?>> crossTabFutures = new ArrayList<Future<?>>(); for (List<DataElementOperand> operandPage : operandPages) { crossTabFutures.add( crossTabService.populateCrossTabTable( operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key)); } ConcurrentUtils.waitForCompletion(crossTabFutures); clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Exporting data element data"); final boolean isDataElements = true; final boolean isIndicators = indicators != null && indicators.size() > 0; final int groupLevel = (Integer) systemSettingManager.getSystemSetting( KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL); final boolean isGroups = organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0; if (isDataElements) { // ----------------------------------------------------------------- // 1. Export data element values // ----------------------------------------------------------------- if (operandList.size() > 0) { final OrganisationUnitHierarchy orgUnitHierarchy = organisationUnitService .getOrganisationUnitHierarchy() .prepareChildren(organisationUnits); List<Future<?>> futures = new ArrayList<Future<?>>(); for (List<DataElementOperand> operandPage : operandPages) { futures.add( dataElementDataMart.exportDataValues( operandPage, periods, organisationUnits, null, periodHierarchy, orgUnitHierarchy, AggregatedDataValueTempBatchHandler.class, key)); } ConcurrentUtils.waitForCompletion(futures); } clock.logTime( "Exported values for data element operands (" + operandList.size() + "), pages: " + operandPages.size() + ", " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Dropping data element index"); // ----------------------------------------------------------------- // 2. Drop data element index // ----------------------------------------------------------------- dataMartManager.dropDataValueIndex(); clock.logTime("Dropped data element index"); notifier.notify(id, DATAMART, "Deleting existing data element data"); // ----------------------------------------------------------------- // 3. Delete existing aggregated data values // ----------------------------------------------------------------- dataMartManager.deleteAggregatedDataValues(periodIds); clock.logTime("Deleted existing data element data"); notifier.notify(id, DATAMART, "Copying data element data from temporary table"); // ----------------------------------------------------------------- // 4. Copy aggregated data values from temporary table // ----------------------------------------------------------------- dataMartManager.copyAggregatedDataValuesFromTemp(); clock.logTime("Copied data element data from temporary table"); notifier.notify(id, DATAMART, "Creating data element index"); // ----------------------------------------------------------------- // 5. Create data element index // ----------------------------------------------------------------- dataMartManager.createDataValueIndex(); clock.logTime("Created data element index"); } if (isGroups && isDataElements) { // ----------------------------------------------------------------- // 1. Export data element values // ----------------------------------------------------------------- notifier.notify(id, DATAMART, "Exporting data element org unit data"); Collection<OrganisationUnit> groupOrganisationUnits = new HashSet<OrganisationUnit>(organisationUnits); FilterUtils.filter( groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel)); if (operandList.size() > 0) { final OrganisationUnitHierarchy orgUnitHierarchy = organisationUnitService .getOrganisationUnitHierarchy() .prepareChildren(groupOrganisationUnits, organisationUnitGroups); List<Future<?>> futures = new ArrayList<Future<?>>(); for (List<DataElementOperand> operandPage : operandPages) { futures.add( dataElementDataMart.exportDataValues( operandPage, periods, groupOrganisationUnits, organisationUnitGroups, periodHierarchy, orgUnitHierarchy, AggregatedOrgUnitDataValueTempBatchHandler.class, key)); } ConcurrentUtils.waitForCompletion(futures); } clock.logTime( "Exported values for data element operands (" + operandList.size() + "), pages: " + operandPages.size() + ", " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Dropping data element data indexes"); // ----------------------------------------------------------------- // 2. Drop data element index // ----------------------------------------------------------------- dataMartManager.dropOrgUnitDataValueIndex(); clock.logTime("Dropped org unit data element index"); notifier.notify(id, DATAMART, "Deleting existing org unit data element data"); // ----------------------------------------------------------------- // 3. Delete existing aggregated data values // ----------------------------------------------------------------- dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds); clock.logTime("Deleted existing aggregated org unit datavalues"); notifier.notify(id, DATAMART, "Copying org unit data element data"); // ----------------------------------------------------------------- // 4. Copy aggregated org unit data values from temporary table // ----------------------------------------------------------------- dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp(); clock.logTime("Copied org unit data element data from temporary table"); notifier.notify(id, DATAMART, "Creating org unit data element index"); // ----------------------------------------------------------------- // 5. Create org unit data element index // ----------------------------------------------------------------- dataMartManager.createOrgUnitDataValueIndex(); clock.logTime("Created org unit data element index"); notifier.notify(id, DATAMART, "Exporting data for org unit indicator data"); } crossTabService.dropCrossTabTable(key); List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>(organisationUnits) .setNumberOfPages(cpuCores) .getPages(); if (isIndicators) { // ----------------------------------------------------------------- // 1. Create and populate aggregated data cache // ----------------------------------------------------------------- notifier.notify(id, DATAMART, "Populating aggregated data cache"); crossTabService.createAggregatedDataCache(indicatorOperands, key); List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>(); for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) { aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache( indicatorOperands, periods, organisationUnitPage, key)); } ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures); clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size()); notifier.notify(id, DATAMART, "Exporting indicator data"); // ----------------------------------------------------------------- // 2. Export indicator values // ----------------------------------------------------------------- List<Future<?>> futures = new ArrayList<Future<?>>(); for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) { futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, null, indicatorOperands, AggregatedIndicatorValueTempBatchHandler.class, key)); } ConcurrentUtils.waitForCompletion(futures); clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Dropping indicator index"); // ----------------------------------------------------------------- // 3. Drop aggregated data cache and indicator index // ----------------------------------------------------------------- crossTabService.dropAggregatedDataCache(key); dataMartManager.dropIndicatorValueIndex(); clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Deleting existing indicator data"); // ----------------------------------------------------------------- // 4. Delete existing aggregated indicator values // ----------------------------------------------------------------- dataMartManager.deleteAggregatedIndicatorValues(periodIds); clock.logTime("Deleted existing indicator data"); notifier.notify(id, DATAMART, "Copying indicator data from temporary table"); // ----------------------------------------------------------------- // 5. Copy aggregated data values from temporary table // ----------------------------------------------------------------- dataMartManager.copyAggregatedIndicatorValuesFromTemp(); clock.logTime("Copied indicator data from temporary table"); notifier.notify(id, DATAMART, "Creating indicator index"); // ----------------------------------------------------------------- // 6. Create indicator index // ----------------------------------------------------------------- dataMartManager.createIndicatorValueIndex(); clock.logTime("Created indicator index"); } if (isGroups && isIndicators) { // ----------------------------------------------------------------- // 1. Create aggregated data cache // ----------------------------------------------------------------- notifier.notify(id, DATAMART, "Populating aggregated data cache"); crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key); List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>(); for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) { aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache( indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key)); } ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures); clock.logTime("Created aggregated org unit data cache"); notifier.notify(id, DATAMART, "Exporting org unit indicator data"); // ----------------------------------------------------------------- // 2. Export indicator values // ----------------------------------------------------------------- List<Future<?>> futures = new ArrayList<Future<?>>(); for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) { futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueTempBatchHandler.class, key)); } ConcurrentUtils.waitForCompletion(futures); clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Dropping org unit indicator index"); // ----------------------------------------------------------------- // 3. Drop aggregated data cache and indicator index // ----------------------------------------------------------------- crossTabService.dropAggregatedOrgUnitDataCache(key); dataMartManager.dropOrgUnitIndicatorValueIndex(); clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString()); notifier.notify(id, DATAMART, "Deleting existing org unit indicator data"); // ----------------------------------------------------------------- // 4. Delete existing aggregated indicator values // ----------------------------------------------------------------- dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds); clock.logTime("Deleted existing aggregated org unit indicatorvalues"); notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table"); // ----------------------------------------------------------------- // 5. Copy aggregated org unit indicator values from temp table // ----------------------------------------------------------------- dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp(); clock.logTime("Copied org unit indicator data from temporary table"); notifier.notify(id, DATAMART, "Creating org unit indicator indexes"); // ----------------------------------------------------------------- // 6. Create org unit indicator index // ----------------------------------------------------------------- dataMartManager.createOrgUnitIndicatorValueIndex(); clock.logTime("Created org unit indicator index"); } // --------------------------------------------------------------------- // Drop temporary tables // --------------------------------------------------------------------- dataMartManager.dropTempAggregatedTables(); clock.logTime("Dropped crosstab table"); clock.logTime("Data mart export process completed"); notifier.notify(id, DATAMART, INFO, "Data mart process completed", true); }
@Override public void setUpTest() { mockDataValueBatchHandler = new MockBatchHandler<>(); mockBatchHandlerFactory = new MockBatchHandlerFactory(); mockBatchHandlerFactory.registerBatchHandler( DataValueBatchHandler.class, mockDataValueBatchHandler); setDependency(dataValueSetService, "batchHandlerFactory", mockBatchHandlerFactory); categoryOptionA = createCategoryOption('A'); categoryOptionB = createCategoryOption('B'); categoryA = createDataElementCategory('A', categoryOptionA, categoryOptionB); categoryComboA = createCategoryCombo('A', categoryA); ocDef = categoryService.getDefaultDataElementCategoryOptionCombo(); ocA = createCategoryOptionCombo(categoryComboA, categoryOptionA); ocB = createCategoryOptionCombo(categoryComboA, categoryOptionB); deA = createDataElement('A'); deB = createDataElement('B'); deC = createDataElement('C'); deD = createDataElement('D'); dsA = createDataSet('A', new MonthlyPeriodType()); ouA = createOrganisationUnit('A'); ouB = createOrganisationUnit('B'); ouC = createOrganisationUnit('C'); peA = createPeriod( PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME), getDate(2012, 1, 1), getDate(2012, 1, 31)); peB = createPeriod( PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME), getDate(2012, 2, 1), getDate(2012, 2, 29)); ocA.setUid("kjuiHgy67hg"); ocB.setUid("Gad33qy67g5"); deA.setUid("f7n9E0hX8qk"); deB.setUid("Ix2HsbDMLea"); deC.setUid("eY5ehpbEsB7"); dsA.setUid("pBOMPrpg1QX"); ouA.setUid("DiszpKrYNg8"); ouB.setUid("BdfsJfj87js"); ouC.setUid("j7Hg26FpoIa"); ocA.setCode("OC_A"); ocB.setCode("OC_B"); deA.setCode("DE_A"); deB.setCode("DE_B"); deC.setCode("DE_C"); deD.setCode("DE_D"); dsA.setCode("DS_A"); ouA.setCode("OU_A"); ouB.setCode("OU_B"); ouC.setCode("OU_C"); categoryService.addDataElementCategoryOption(categoryOptionA); categoryService.addDataElementCategoryOption(categoryOptionB); categoryService.addDataElementCategory(categoryA); categoryService.addDataElementCategoryCombo(categoryComboA); categoryService.addDataElementCategoryOptionCombo(ocA); categoryService.addDataElementCategoryOptionCombo(ocB); dataElementService.addDataElement(deA); dataElementService.addDataElement(deB); dataElementService.addDataElement(deC); dataElementService.addDataElement(deD); dataSetService.addDataSet(dsA); organisationUnitService.addOrganisationUnit(ouA); organisationUnitService.addOrganisationUnit(ouB); organisationUnitService.addOrganisationUnit(ouC); periodService.addPeriod(peA); periodService.addPeriod(peB); user = createUser('A'); user.setOrganisationUnits(Sets.newHashSet(ouA, ouB)); CurrentUserService currentUserService = new MockCurrentUserService(user); setDependency(dataValueSetService, "currentUserService", currentUserService); }